Skip to content

Commit 971347b

Browse files
RobertCraigieRobertCraigie
committed
chore(tests): add responses output_text test
1 parent f03a03d commit 971347b

File tree

4 files changed

+58
-2
lines changed

4 files changed

+58
-2
lines changed

‎tests/lib/chat/test_completions.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pyte
4343
content_snapshot=snapshot(
4444
'{"id": "chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY", "object": "chat.completion", "created": 1727346142, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "I\'m unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend checking a reliable weather website or app like the Weather Channel or a local news station.", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 14, "completion_tokens": 37, "total_tokens": 51, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
4545
),
46+
path="/chat/completions",
4647
mock_client=client,
4748
respx_mock=respx_mock,
4849
)
@@ -112,6 +113,7 @@ class Location(BaseModel):
112113
content_snapshot=snapshot(
113114
'{"id": "chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99", "object": "chat.completion", "created": 1727346143, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
114115
),
116+
path="/chat/completions",
115117
mock_client=client,
116118
respx_mock=respx_mock,
117119
)
@@ -182,6 +184,7 @@ class Location(BaseModel):
182184
content_snapshot=snapshot(
183185
'{"id": "chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b", "object": "chat.completion", "created": 1727346144, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 88, "completion_tokens": 14, "total_tokens": 102, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
184186
),
187+
path="/chat/completions",
185188
mock_client=client,
186189
respx_mock=respx_mock,
187190
)
@@ -256,6 +259,7 @@ class ColorDetection(BaseModel):
256259
content_snapshot=snapshot(
257260
'{"id": "chatcmpl-ABfvjIatz0zrZu50gRbMtlp0asZpz", "object": "chat.completion", "created": 1727346151, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"color\\":\\"red\\",\\"hex_color_code\\":\\"#FF0000\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 109, "completion_tokens": 14, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
258261
),
262+
path="/chat/completions",
259263
mock_client=client,
260264
respx_mock=respx_mock,
261265
)
@@ -305,6 +309,7 @@ class Location(BaseModel):
305309
content_snapshot=snapshot(
306310
'{"id": "chatcmpl-ABfvp8qzboW92q8ONDF4DPHlI7ckC", "object": "chat.completion", "created": 1727346157, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":64,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 1, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 2, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":63.0,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 44, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
307311
),
312+
path="/chat/completions",
308313
mock_client=client,
309314
respx_mock=respx_mock,
310315
)
@@ -385,6 +390,7 @@ class CalendarEvent:
385390
content_snapshot=snapshot(
386391
'{"id": "chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8", "object": "chat.completion", "created": 1727346158, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"name\\":\\"Science Fair\\",\\"date\\":\\"Friday\\",\\"participants\\":[\\"Alice\\",\\"Bob\\"]}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 92, "completion_tokens": 17, "total_tokens": 109, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
387392
),
393+
path="/chat/completions",
388394
mock_client=client,
389395
respx_mock=respx_mock,
390396
)
@@ -449,6 +455,7 @@ def test_pydantic_tool_model_all_types(client: OpenAI, respx_mock: MockRouter, m
449455
content_snapshot=snapshot(
450456
'{"id": "chatcmpl-ABfvtNiaTNUF6OymZUnEFc9lPq9p1", "object": "chat.completion", "created": 1727346161, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_NKpApJybW1MzOjZO2FzwYw0d", "type": "function", "function": {"name": "Query", "arguments": "{\\"name\\":\\"May 2022 Fulfilled Orders Not Delivered on Time\\",\\"table_name\\":\\"orders\\",\\"columns\\":[\\"id\\",\\"status\\",\\"expected_delivery_date\\",\\"delivered_at\\",\\"shipped_at\\",\\"ordered_at\\",\\"canceled_at\\"],\\"conditions\\":[{\\"column\\":\\"ordered_at\\",\\"operator\\":\\">=\\",\\"value\\":\\"2022-05-01\\"},{\\"column\\":\\"ordered_at\\",\\"operator\\":\\"<=\\",\\"value\\":\\"2022-05-31\\"},{\\"column\\":\\"status\\",\\"operator\\":\\"=\\",\\"value\\":\\"fulfilled\\"},{\\"column\\":\\"delivered_at\\",\\"operator\\":\\">\\",\\"value\\":{\\"column_name\\":\\"expected_delivery_date\\"}}],\\"order_by\\":\\"asc\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 512, "completion_tokens": 132, "total_tokens": 644, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
451457
),
458+
path="/chat/completions",
452459
mock_client=client,
453460
respx_mock=respx_mock,
454461
)
@@ -534,6 +541,7 @@ class Location(BaseModel):
534541
content_snapshot=snapshot(
535542
'{"id": "chatcmpl-ABfvvX7eB1KsfeZj8VcF3z7G7SbaA", "object": "chat.completion", "created": 1727346163, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"", "refusal": null}, "logprobs": null, "finish_reason": "length"}], "usage": {"prompt_tokens": 79, "completion_tokens": 1, "total_tokens": 80, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
536543
),
544+
path="/chat/completions",
537545
mock_client=client,
538546
respx_mock=respx_mock,
539547
)
@@ -560,6 +568,7 @@ class Location(BaseModel):
560568
content_snapshot=snapshot(
561569
'{"id": "chatcmpl-ABfvwoKVWPQj2UPlAcAKM7s40GsRx", "object": "chat.completion", "created": 1727346164, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": "I\'m very sorry, but I can\'t assist with that."}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 12, "total_tokens": 91, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
562570
),
571+
path="/chat/completions",
563572
mock_client=client,
564573
respx_mock=respx_mock,
565574
)
@@ -610,6 +619,7 @@ class GetWeatherArgs(BaseModel):
610619
content_snapshot=snapshot(
611620
'{"id": "chatcmpl-ABfvx6Z4dchiW2nya1N8KMsHFrQRE", "object": "chat.completion", "created": 1727346165, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_Y6qJ7ofLgOrBnMD5WbVAeiRV", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\":\\"Edinburgh\\",\\"country\\":\\"UK\\",\\"units\\":\\"c\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 76, "completion_tokens": 24, "total_tokens": 100, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_e45dabd248"}'
612621
),
622+
path="/chat/completions",
613623
mock_client=client,
614624
respx_mock=respx_mock,
615625
)
@@ -683,6 +693,7 @@ class GetStockPrice(BaseModel):
683693
content_snapshot=snapshot(
684694
'{"id": "chatcmpl-ABfvyvfNWKcl7Ohqos4UFrmMs1v4C", "object": "chat.completion", "created": 1727346166, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_fdNz3vOBKYgOIpMdWotB9MjY", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\": \\"Edinburgh\\", \\"country\\": \\"GB\\", \\"units\\": \\"c\\"}"}}, {"id": "call_h1DWI1POMJLb0KwIyQHWXD4p", "type": "function", "function": {"name": "get_stock_price", "arguments": "{\\"ticker\\": \\"AAPL\\", \\"exchange\\": \\"NASDAQ\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 149, "completion_tokens": 60, "total_tokens": 209, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
685695
),
696+
path="/chat/completions",
686697
mock_client=client,
687698
respx_mock=respx_mock,
688699
)
@@ -765,6 +776,7 @@ def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch:
765776
content_snapshot=snapshot(
766777
'{"id": "chatcmpl-ABfvzdvCI6RaIkiEFNjqGXCSYnlzf", "object": "chat.completion", "created": 1727346167, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_CUdUoJpsWWVdxXntucvnol1M", "type": "function", "function": {"name": "get_weather", "arguments": "{\\"city\\":\\"San Francisco\\",\\"state\\":\\"CA\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 48, "completion_tokens": 19, "total_tokens": 67, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
767778
),
779+
path="/chat/completions",
768780
mock_client=client,
769781
respx_mock=respx_mock,
770782
)
@@ -842,6 +854,7 @@ class Location(BaseModel):
842854
content_snapshot=snapshot(
843855
'{"id": "chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT", "object": "chat.completion", "created": 1727389540, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":58,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
844856
),
857+
path="/chat/completions",
845858
mock_client=client,
846859
respx_mock=respx_mock,
847860
)
@@ -918,6 +931,7 @@ class Location(BaseModel):
918931
content_snapshot=snapshot(
919932
'{"id": "chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq", "object": "chat.completion", "created": 1727389532, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
920933
),
934+
path="/chat/completions",
921935
mock_client=async_client,
922936
respx_mock=respx_mock,
923937
)

‎tests/lib/responses/__init__.py

Whitespace-only changes.

‎tests/lib/responses/test_responses.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
from __future__ import annotations
2+
3+
from typing_extensions import TypeVar
4+
5+
import pytest
6+
from respx import MockRouter
7+
from inline_snapshot import snapshot
8+
9+
from openai import OpenAI
10+
11+
from ...conftest import base_url
12+
from ..snapshots import make_snapshot_request
13+
14+
_T = TypeVar("_T")
15+
16+
# all the snapshots in this file are auto-generated from the live API
17+
#
18+
# you can update them with
19+
#
20+
# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
21+
22+
23+
@pytest.mark.respx(base_url=base_url)
24+
def test_output_text(client: OpenAI, respx_mock: MockRouter) -> None:
25+
response = make_snapshot_request(
26+
lambda c: c.responses.create(
27+
model="gpt-4o-mini",
28+
input="What's the weather like in SF?",
29+
),
30+
content_snapshot=snapshot(
31+
'{"id": "resp_689a0b2545288193953c892439b42e2800b2e36c65a1fd4b", "object": "response", "created_at": 1754925861, "status": "completed", "background": false, "error": null, "incomplete_details": null, "instructions": null, "max_output_tokens": null, "max_tool_calls": null, "model": "gpt-4o-mini-2024-07-18", "output": [{"id": "msg_689a0b2637b08193ac478e568f49e3f900b2e36c65a1fd4b", "type": "message", "status": "completed", "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "I can\'t provide real-time updates, but you can easily check the current weather in San Francisco using a weather website or app. Typically, San Francisco has cool, foggy summers and mild winters, so it\'s good to be prepared for variable weather!"}], "role": "assistant"}], "parallel_tool_calls": true, "previous_response_id": null, "prompt_cache_key": null, "reasoning": {"effort": null, "summary": null}, "safety_identifier": null, "service_tier": "default", "store": true, "temperature": 1.0, "text": {"format": {"type": "text"}, "verbosity": "medium"}, "tool_choice": "auto", "tools": [], "top_logprobs": 0, "top_p": 1.0, "truncation": "disabled", "usage": {"input_tokens": 14, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 50, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 64}, "user": null, "metadata": {}}'
32+
),
33+
path="/responses",
34+
mock_client=client,
35+
respx_mock=respx_mock,
36+
)
37+
38+
assert response.output_text == snapshot(
39+
"I can't provide real-time updates, but you can easily check the current weather in San Francisco using a weather website or app. Typically, San Francisco has cool, foggy summers and mild winters, so it's good to be prepared for variable weather!"
40+
)

‎tests/lib/snapshots.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ def make_snapshot_request(
2121
content_snapshot: Any,
2222
respx_mock: MockRouter,
2323
mock_client: OpenAI,
24+
path: str,
2425
) -> _T:
2526
live = os.environ.get("OPENAI_LIVE") == "1"
2627
if live:
@@ -39,7 +40,7 @@ def _on_response(response: httpx.Response) -> None:
3940
)
4041
)
4142
else:
42-
respx_mock.post("/chat/completions").mock(
43+
respx_mock.post(path).mock(
4344
return_value=httpx.Response(
4445
200,
4546
content=get_snapshot_value(content_snapshot),
@@ -63,6 +64,7 @@ async def make_async_snapshot_request(
6364
content_snapshot: Any,
6465
respx_mock: MockRouter,
6566
mock_client: AsyncOpenAI,
67+
path: str,
6668
) -> _T:
6769
live = os.environ.get("OPENAI_LIVE") == "1"
6870
if live:
@@ -81,7 +83,7 @@ async def _on_response(response: httpx.Response) -> None:
8183
)
8284
)
8385
else:
84-
respx_mock.post("/chat/completions").mock(
86+
respx_mock.post(path).mock(
8587
return_value=httpx.Response(
8688
200,
8789
content=get_snapshot_value(content_snapshot),

0 commit comments

Comments
 (0)