109fe03ad1
Co-authored-by: Chang Su <chang.s.su@oracle.com> Co-authored-by: key4ng <rukeyang@gmail.com>
290 lines
10 KiB
Python
290 lines
10 KiB
Python
"""Structured output tests for Response API.
|
|
|
|
Tests for text.format field with json_object and json_schema formats.
|
|
|
|
Source: Migrated from e2e_response_api/features/test_structured_output.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
|
|
import pytest
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# =============================================================================
|
|
# Cloud Backend Tests (OpenAI)
|
|
# =============================================================================
|
|
|
|
|
|
@pytest.mark.parametrize("setup_backend", ["openai"], indirect=True)
|
|
class TestStructuredOutputCloud:
|
|
"""Structured output tests against cloud APIs."""
|
|
|
|
def test_structured_output_json_schema(self, setup_backend):
|
|
"""Test structured output with json_schema format."""
|
|
_, model, client, gateway = setup_backend
|
|
|
|
params = {
|
|
"model": model,
|
|
"input": [
|
|
{
|
|
"role": "system",
|
|
"content": "You are a helpful math tutor. Guide the user through the solution step by step.",
|
|
},
|
|
{"role": "user", "content": "how can I solve 8x + 7 = -23"},
|
|
],
|
|
"text": {
|
|
"format": {
|
|
"type": "json_schema",
|
|
"name": "math_reasoning",
|
|
"schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"steps": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"explanation": {"type": "string"},
|
|
"output": {"type": "string"},
|
|
},
|
|
"required": ["explanation", "output"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
"final_answer": {"type": "string"},
|
|
},
|
|
"required": ["steps", "final_answer"],
|
|
"additionalProperties": False,
|
|
},
|
|
"strict": True,
|
|
}
|
|
},
|
|
}
|
|
|
|
create_resp = client.responses.create(**params)
|
|
assert create_resp.error is None
|
|
assert create_resp.id is not None
|
|
assert create_resp.output is not None
|
|
assert create_resp.text is not None
|
|
|
|
# Verify text format was echoed back correctly
|
|
assert create_resp.text.format is not None
|
|
assert create_resp.text.format.type == "json_schema"
|
|
assert create_resp.text.format.name == "math_reasoning"
|
|
assert create_resp.text.format.schema_ is not None
|
|
assert create_resp.text.format.strict
|
|
|
|
# Find the message output
|
|
output_text = next(
|
|
(
|
|
content.text
|
|
for item in create_resp.output
|
|
if item.type == "message"
|
|
for content in item.content
|
|
if content.type == "output_text"
|
|
),
|
|
None,
|
|
)
|
|
|
|
assert output_text is not None, "No output_text found in response"
|
|
assert output_text.strip(), "output_text is empty"
|
|
|
|
# Parse JSON output
|
|
output_json = json.loads(output_text)
|
|
|
|
# Verify schema structure
|
|
assert "steps" in output_json
|
|
assert "final_answer" in output_json
|
|
assert isinstance(output_json["steps"], list)
|
|
assert len(output_json["steps"]) > 0
|
|
|
|
# Verify each step has required fields
|
|
for step in output_json["steps"]:
|
|
assert "explanation" in step
|
|
assert "output" in step
|
|
|
|
|
|
# =============================================================================
|
|
# Local Backend Tests (gRPC with Harmony model - complex schema)
|
|
# =============================================================================
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.model("gpt-oss")
|
|
@pytest.mark.gateway(
|
|
extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]
|
|
)
|
|
@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
|
|
class TestStructuredOutputHarmony:
|
|
"""Structured output tests against local gRPC backend with Harmony model."""
|
|
|
|
def test_structured_output_json_schema(self, setup_backend):
|
|
"""Test structured output with json_schema format."""
|
|
_, model, client, gateway = setup_backend
|
|
|
|
params = {
|
|
"model": model,
|
|
"input": [
|
|
{
|
|
"role": "system",
|
|
"content": "You are a helpful math tutor. Guide the user through the solution step by step.",
|
|
},
|
|
{"role": "user", "content": "how can I solve 8x + 7 = -23"},
|
|
],
|
|
"text": {
|
|
"format": {
|
|
"type": "json_schema",
|
|
"name": "math_reasoning",
|
|
"schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"steps": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"explanation": {"type": "string"},
|
|
"output": {"type": "string"},
|
|
},
|
|
"required": ["explanation", "output"],
|
|
"additionalProperties": False,
|
|
},
|
|
},
|
|
"final_answer": {"type": "string"},
|
|
},
|
|
"required": ["steps", "final_answer"],
|
|
"additionalProperties": False,
|
|
},
|
|
"strict": True,
|
|
}
|
|
},
|
|
}
|
|
|
|
create_resp = client.responses.create(**params)
|
|
assert create_resp.error is None
|
|
assert create_resp.id is not None
|
|
assert create_resp.output is not None
|
|
assert create_resp.text is not None
|
|
|
|
# Verify text format was echoed back correctly
|
|
assert create_resp.text.format is not None
|
|
assert create_resp.text.format.type == "json_schema"
|
|
assert create_resp.text.format.name == "math_reasoning"
|
|
assert create_resp.text.format.schema_ is not None
|
|
assert create_resp.text.format.strict
|
|
|
|
# Find the message output (output[0] may be reasoning, output[1] is message)
|
|
output_text = next(
|
|
(
|
|
content.text
|
|
for item in create_resp.output
|
|
if item.type == "message"
|
|
for content in item.content
|
|
if content.type == "output_text"
|
|
),
|
|
None,
|
|
)
|
|
|
|
assert output_text is not None, "No output_text found in response"
|
|
assert output_text.strip(), "output_text is empty"
|
|
|
|
# Parse JSON output
|
|
output_json = json.loads(output_text)
|
|
|
|
# Verify schema structure
|
|
assert "steps" in output_json
|
|
assert "final_answer" in output_json
|
|
assert isinstance(output_json["steps"], list)
|
|
assert len(output_json["steps"]) > 0
|
|
|
|
# Verify each step has required fields
|
|
for step in output_json["steps"]:
|
|
assert "explanation" in step
|
|
assert "output" in step
|
|
|
|
|
|
# =============================================================================
|
|
# Local Backend Tests (gRPC with Qwen model - simple schema)
|
|
# =============================================================================
|
|
|
|
|
|
@pytest.mark.e2e
|
|
@pytest.mark.model("qwen-14b")
|
|
@pytest.mark.gateway(
|
|
extra_args=["--tool-call-parser", "qwen", "--history-backend", "memory"]
|
|
)
|
|
@pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True)
|
|
class TestSimpleSchemaStructuredOutput:
|
|
"""Structured output tests with simpler schema for models that don't
|
|
handle complex schemas well.
|
|
"""
|
|
|
|
def test_structured_output_json_schema(self, setup_backend):
|
|
"""Test structured output with simple json_schema format."""
|
|
_, model, client, gateway = setup_backend
|
|
|
|
params = {
|
|
"model": model,
|
|
"input": [
|
|
{
|
|
"role": "system",
|
|
"content": "You are a math solver. Return ONLY a JSON object that matches the schema-no extra text.",
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": "What is 1 + 1?",
|
|
},
|
|
],
|
|
"text": {
|
|
"format": {
|
|
"type": "json_schema",
|
|
"name": "math_answer",
|
|
"schema": {
|
|
"type": "object",
|
|
"properties": {"answer": {"type": "string"}},
|
|
"required": ["answer"],
|
|
},
|
|
}
|
|
},
|
|
}
|
|
|
|
create_resp = client.responses.create(**params)
|
|
assert create_resp.error is None
|
|
assert create_resp.id is not None
|
|
assert create_resp.output is not None
|
|
assert create_resp.text is not None
|
|
|
|
# Verify text format was echoed back correctly
|
|
assert create_resp.text.format is not None
|
|
assert create_resp.text.format.type == "json_schema"
|
|
assert create_resp.text.format.name == "math_answer"
|
|
assert create_resp.text.format.schema_ is not None
|
|
|
|
# Find the message output
|
|
output_text = next(
|
|
(
|
|
content.text
|
|
for item in create_resp.output
|
|
if item.type == "message"
|
|
for content in item.content
|
|
if content.type == "output_text"
|
|
),
|
|
None,
|
|
)
|
|
|
|
assert output_text is not None, "No output_text found in response"
|
|
assert output_text.strip(), "output_text is empty"
|
|
|
|
# Parse JSON output
|
|
output_json = json.loads(output_text)
|
|
|
|
# Verify simple schema structure (just answer field)
|
|
assert "answer" in output_json
|
|
assert isinstance(output_json["answer"], str)
|
|
assert output_json["answer"], "Answer is empty"
|