From 6e4c1076f6d32037119fa670ee43f14f2a22a5b2 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Mon, 8 Sep 2025 14:56:48 +0200 Subject: [PATCH 1/3] Handle errors in cost calculation in InstrumentedModel --- .../pydantic_ai/models/instrumented.py | 13 +++- pydantic_ai_slim/pyproject.toml | 2 +- tests/models/test_instrumented.py | 70 +++++++++++++++++++ uv.lock | 10 +-- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/models/instrumented.py b/pydantic_ai_slim/pydantic_ai/models/instrumented.py index 45a2cbd91a..7bbe82355b 100644 --- a/pydantic_ai_slim/pydantic_ai/models/instrumented.py +++ b/pydantic_ai_slim/pydantic_ai/models/instrumented.py @@ -420,10 +420,17 @@ def _record_metrics(): return self.instrumentation_settings.handle_messages(messages, response, system, span) + + cost_attributes = {} try: cost_attributes = {'operation.cost': float(response.cost().total_price)} except LookupError: - cost_attributes = {} + pass + except Exception as e: + warnings.warn( + f'Failed to get cost from response: {type(e).__name__}: {e}', CostCalculationFailedWarning + ) + span.set_attributes( { **response.usage.opentelemetry_attributes(), @@ -478,3 +485,7 @@ def serialize_any(value: Any) -> str: return str(value) except Exception as e: return f'Unable to serialize: {e}' + + +class CostCalculationFailedWarning(Warning): + """Warning raised when cost calculation fails.""" diff --git a/pydantic_ai_slim/pyproject.toml b/pydantic_ai_slim/pyproject.toml index bf35813b45..5f7cc61914 100644 --- a/pydantic_ai_slim/pyproject.toml +++ b/pydantic_ai_slim/pyproject.toml @@ -61,7 +61,7 @@ dependencies = [ "exceptiongroup; python_version < '3.11'", "opentelemetry-api>=1.28.0", "typing-inspection>=0.4.0", - "genai-prices>=0.0.22", + "genai-prices>=0.0.23", ] [tool.hatch.metadata.hooks.uv-dynamic-versioning.optional-dependencies] diff --git a/tests/models/test_instrumented.py b/tests/models/test_instrumented.py index f9265171dc..e2ed20a526 100644 --- a/tests/models/test_instrumented.py +++ b/tests/models/test_instrumented.py @@ -7,6 +7,7 @@ import pytest from inline_snapshot import snapshot +from inline_snapshot.extra import warns from logfire_api import DEFAULT_LOGFIRE_INSTANCE from opentelemetry._events import NoOpEventLoggerProvider from opentelemetry.trace import NoOpTracerProvider @@ -1274,3 +1275,72 @@ def test_deprecated_event_mode_warning(): assert settings.event_mode == 'logs' assert settings.version == 1 assert InstrumentationSettings().version == 2 + + +async def test_response_cost_error(capfire: CaptureLogfire, monkeypatch: pytest.MonkeyPatch): + model = InstrumentedModel(MyModel()) + + messages: list[ModelMessage] = [ModelRequest(parts=[UserPromptPart('user_prompt')])] + monkeypatch.setattr(ModelResponse, 'cost', None) + + with warns( + snapshot( + [ + "CostCalculationFailedWarning: Failed to get cost from response: TypeError: 'NoneType' object is not callable" + ] + ) + ): + await model.request(messages, model_settings=ModelSettings(), model_request_parameters=ModelRequestParameters()) + + assert capfire.exporter.exported_spans_as_dict(parse_json_attributes=True) == snapshot( + [ + { + 'name': 'chat gpt-4o', + 'context': {'trace_id': 1, 'span_id': 1, 'is_remote': False}, + 'parent': None, + 'start_time': 1000000000, + 'end_time': 2000000000, + 'attributes': { + 'gen_ai.operation.name': 'chat', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4o', + 'server.address': 'example.com', + 'server.port': 8000, + 'model_request_parameters': { + 'function_tools': [], + 'builtin_tools': [], + 'output_mode': 'text', + 'output_object': None, + 'output_tools': [], + 'allow_text_output': True, + }, + 'logfire.span_type': 'span', + 'logfire.msg': 'chat gpt-4o', + 'gen_ai.input.messages': [{'role': 'user', 'parts': [{'type': 'text', 'content': 'user_prompt'}]}], + 'gen_ai.output.messages': [ + { + 'role': 'assistant', + 'parts': [ + {'type': 'text', 'content': 'text1'}, + {'type': 'tool_call', 'id': 'tool_call_1', 'name': 'tool1', 'arguments': 'args1'}, + {'type': 'tool_call', 'id': 'tool_call_2', 'name': 'tool2', 'arguments': {'args2': 3}}, + {'type': 'text', 'content': 'text2'}, + ], + 'finish_reason': 'stop', + } + ], + 'logfire.json_schema': { + 'type': 'object', + 'properties': { + 'gen_ai.input.messages': {'type': 'array'}, + 'gen_ai.output.messages': {'type': 'array'}, + 'model_request_parameters': {'type': 'object'}, + }, + }, + 'gen_ai.usage.input_tokens': 100, + 'gen_ai.usage.output_tokens': 200, + 'gen_ai.response.model': 'gpt-4o-2024-11-20', + }, + } + ] + ) diff --git a/uv.lock b/uv.lock index c2ef3431c9..ae724b6ccc 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.13' and platform_python_implementation == 'PyPy'", @@ -1157,16 +1157,16 @@ http = [ [[package]] name = "genai-prices" -version = "0.0.22" +version = "0.0.25" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "eval-type-backport", marker = "python_full_version < '3.11'" }, { name = "httpx" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/c5/0aa155ac23a17eb6de36f0611d8595fc49861bdb0a5f302133b7b4f68f5b/genai_prices-0.0.22.tar.gz", hash = "sha256:5e743424d40176ea04de7b74d1ad3a41801390439f4404d4593b82218f2c0c04", size = 44125, upload-time = "2025-08-12T12:04:52.265Z" } +sdist = { url = "https://files.pythonhosted.org/packages/02/9e/f292acaf69bd209b354ef835cab4ebe845eced05c4db85e3b31585429806/genai_prices-0.0.25.tar.gz", hash = "sha256:caf5fe2fd2248e87f70b2b44bbf8b3b52871abfc078a5e35372c40aca4cc4450", size = 44693, upload-time = "2025-09-01T17:30:42.185Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/1c/313541ea19144a7e5b0ac32dec3dd026de719c5b15147b043d52006dbef7/genai_prices-0.0.22-py3-none-any.whl", hash = "sha256:1ae496bdf517047bc489421c1eff653872e5d456d5eb86d113dfb49d2977a041", size = 46445, upload-time = "2025-08-12T12:04:50.884Z" }, + { url = "https://files.pythonhosted.org/packages/86/12/41fcfba4ae0f6b4805f09d11f0e6d6417df2572cea13208c0f439170ee0c/genai_prices-0.0.25-py3-none-any.whl", hash = "sha256:47b412e6927787caa00717a5d99b2e4c0858bed507bb16473b1bcaff48d5aae9", size = 47002, upload-time = "2025-09-01T17:30:41.012Z" }, ] [[package]] @@ -3192,7 +3192,7 @@ requires-dist = [ { name = "eval-type-backport", specifier = ">=0.2.0" }, { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "fasta2a", marker = "extra == 'a2a'", specifier = ">=0.4.1" }, - { name = "genai-prices", specifier = ">=0.0.22" }, + { name = "genai-prices", specifier = ">=0.0.23" }, { name = "google-auth", marker = "extra == 'vertexai'", specifier = ">=2.36.0" }, { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.31.0" }, { name = "griffe", specifier = ">=1.3.2" }, From bf17a8d0c4e781d8b5b77ce55b145fa3d2d6e5f0 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Mon, 8 Sep 2025 15:00:08 +0200 Subject: [PATCH 2/3] comments --- pydantic_ai_slim/pydantic_ai/models/instrumented.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pydantic_ai_slim/pydantic_ai/models/instrumented.py b/pydantic_ai_slim/pydantic_ai/models/instrumented.py index 7bbe82355b..f7046bc63f 100644 --- a/pydantic_ai_slim/pydantic_ai/models/instrumented.py +++ b/pydantic_ai_slim/pydantic_ai/models/instrumented.py @@ -425,6 +425,7 @@ def _record_metrics(): try: cost_attributes = {'operation.cost': float(response.cost().total_price)} except LookupError: + # The cost of this provider/model is unknown, which is common. pass except Exception as e: warnings.warn( From e9003ec9abd09388a0ebc43e9396b79088c7753e Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Mon, 8 Sep 2025 15:07:36 +0200 Subject: [PATCH 3/3] Update test_anthropic for updated genai-prices --- tests/models/test_anthropic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index a9c849b4a2..e2b49dd675 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -252,7 +252,7 @@ async def test_async_request_prompt_caching(allow_model_requests: None): ) last_message = result.all_messages()[-1] assert isinstance(last_message, ModelResponse) - assert last_message.cost().total_price == snapshot(Decimal('0.00003488')) + assert last_message.cost().total_price == snapshot(Decimal('0.00002688')) async def test_async_request_text_response(allow_model_requests: None):