diff --git a/.sampo/changesets/prompt-config.md b/.sampo/changesets/prompt-config.md new file mode 100644 index 00000000..85525261 --- /dev/null +++ b/.sampo/changesets/prompt-config.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +feat(ai): `Prompts.get(..., with_metadata=True)` results now include `config`, the JSON object of model parameters or agent configuration stored with the prompt version in PostHog prompt management (`None` when the version has none). Config is carried through the client-side cache and the stale-cache fallback. The hardcoded `fallback` string has no config, so use defensive access like `(result.config or {}).get("temperature", 0)`. diff --git a/posthog/ai/prompts.py b/posthog/ai/prompts.py index dd35a2e3..0e5ef38b 100644 --- a/posthog/ai/prompts.py +++ b/posthog/ai/prompts.py @@ -4,6 +4,7 @@ Fetch and compile LLM prompts from PostHog with caching and fallback support. """ +import copy import logging import re import time @@ -32,6 +33,11 @@ class PromptResult: ``label`` is the label the prompt resolved through, populated from the API response when fetching with the ``label`` option; ``None`` otherwise. + + ``config`` is the JSON object of model parameters or agent configuration + stored with the prompt version, or ``None`` when the version has none + (including on ``code_fallback`` results). Use defensive access, e.g. + ``(result.config or {}).get("temperature", 0)``. """ source: PromptSource @@ -39,6 +45,7 @@ class PromptResult: name: Optional[str] = None version: Optional[int] = None label: Optional[str] = None + config: Optional[Dict[str, Any]] = None class CachedPrompt: @@ -51,12 +58,14 @@ def __init__( name: str, version: int, label: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, ): self.prompt = prompt self.fetched_at = fetched_at self.name = name self.version = version self.label = label + self.config = config def _cache_key( @@ -83,6 +92,12 @@ def _prompt_reference( return reference +def _extract_config(data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Read config from an API response, tolerating servers that don't send it.""" + config = data.get("config") + return config if isinstance(config, dict) else None + + def _is_prompt_api_response(data: Any) -> bool: """Check if the response is a valid prompt API response.""" return ( @@ -225,9 +240,9 @@ def get( Fetch a prompt by name from the PostHog API. When ``with_metadata`` is ``True``, returns a :class:`PromptResult` - with ``source``, ``name``, and ``version`` metadata. When omitted or - ``False``, returns a plain string (deprecated -- will be removed in a - future major version). + with ``source``, ``name``, ``version``, and ``config`` metadata. When + omitted or ``False``, returns a plain string (deprecated -- will be + removed in a future major version). Args: name: The name of the prompt to fetch @@ -319,6 +334,9 @@ def _get_internal( name=cached.name, version=cached.version, label=cached.label, + # Copied so a caller mutating result.config can't pollute the + # cache entry that later cache hits are served from. + config=copy.deepcopy(cached.config), ) # Try to fetch from API @@ -337,6 +355,8 @@ def _get_internal( data.get("label"), ) + config = _extract_config(data) + # Update cache self._cache[cache_key] = CachedPrompt( prompt=data["prompt"], @@ -344,6 +364,7 @@ def _get_internal( name=data["name"], version=data["version"], label=data.get("label"), + config=config, ) return PromptResult( @@ -352,6 +373,7 @@ def _get_internal( name=data["name"], version=data["version"], label=data.get("label"), + config=copy.deepcopy(config), ) except Exception as error: @@ -371,6 +393,7 @@ def _get_internal( name=cached.name, version=cached.version, label=cached.label, + config=copy.deepcopy(cached.config), ) raise diff --git a/posthog/test/ai/test_prompts.py b/posthog/test/ai/test_prompts.py index c1d0a0fe..9a860f38 100644 --- a/posthog/test/ai/test_prompts.py +++ b/posthog/test/ai/test_prompts.py @@ -769,6 +769,97 @@ def test_share_cache_with_non_metadata_calls(self, mock_get_session): self.assertEqual(mock_get.call_count, 1) +class TestPromptsConfig(TestPrompts): + """Tests for the config object attached to prompt versions.""" + + mock_config = {"model": "gpt-4o", "temperature": 0.2} + + @patch("posthog.ai.prompts._get_session") + def test_config_flows_through_api_and_cache_results(self, mock_get_session): + """Config from the API response must survive both the fresh fetch and a cache hit.""" + mock_get = mock_get_session.return_value.get + mock_get.return_value = MockResponse( + json_data={**self.mock_prompt_response, "config": self.mock_config} + ) + + prompts = Prompts(self.create_mock_posthog()) + + api_result = prompts.get("test-prompt", with_metadata=True) + cached_result = prompts.get("test-prompt", with_metadata=True) + + self.assertEqual(api_result.source, "api") + self.assertEqual(api_result.config, self.mock_config) + self.assertEqual(cached_result.source, "cache") + self.assertEqual(cached_result.config, self.mock_config) + self.assertEqual(mock_get.call_count, 1) + + @patch("posthog.ai.prompts._get_session") + @patch("posthog.ai.prompts.time.time") + def test_stale_cache_result_keeps_config(self, mock_time, mock_get_session): + """A fetch failure served from stale cache must not lose the config.""" + mock_get = mock_get_session.return_value.get + mock_get.side_effect = [ + MockResponse( + json_data={**self.mock_prompt_response, "config": self.mock_config} + ), + Exception("Network error"), + ] + mock_time.return_value = 1000.0 + + prompts = Prompts(self.create_mock_posthog()) + prompts.get("test-prompt", with_metadata=True, cache_ttl_seconds=60) + mock_time.return_value = 1061.0 + + result = prompts.get("test-prompt", with_metadata=True, cache_ttl_seconds=60) + + self.assertEqual(result.source, "stale_cache") + self.assertEqual(result.config, self.mock_config) + + @patch("posthog.ai.prompts._get_session") + def test_mutating_a_result_config_does_not_pollute_the_cache( + self, mock_get_session + ): + """Callers often mutate config before spreading it into an LLM call; that must not leak into later cache hits.""" + mock_get = mock_get_session.return_value.get + mock_get.return_value = MockResponse( + json_data={**self.mock_prompt_response, "config": self.mock_config} + ) + + prompts = Prompts(self.create_mock_posthog()) + + first = prompts.get("test-prompt", with_metadata=True) + assert first.config is not None + first.config["temperature"] = 0.9 + first.config.pop("model") + + second = prompts.get("test-prompt", with_metadata=True) + + self.assertEqual(second.source, "cache") + self.assertEqual(second.config, self.mock_config) + + @parameterized.expand( + [ + ("absent", {}), + ("null", {"config": None}), + ("non_dict", {"config": "gpt-4o"}), + ] + ) + @patch("posthog.ai.prompts._get_session") + def test_missing_or_invalid_config_reads_as_none( + self, _scenario, extra, mock_get_session + ): + """Servers that omit config or send unexpected shapes read as None.""" + mock_get = mock_get_session.return_value.get + mock_get.return_value = MockResponse( + json_data={**self.mock_prompt_response, **extra} + ) + + prompts = Prompts(self.create_mock_posthog()) + result = prompts.get("test-prompt", with_metadata=True) + + self.assertIsNone(result.config) + + class TestPromptsGetDeprecationWarning(TestPrompts): """Tests for the deprecation warning when with_metadata is not passed."""