add reasoning_tokens usage test for tool call (#18022)
This commit is contained in:
@@ -23,6 +23,7 @@ class ToolCallTestParams:
|
||||
test_strict: bool = True
|
||||
test_multiturn: bool = True
|
||||
test_thinking: bool = False # model-specific, e.g. DeepSeek
|
||||
test_reasoning_usage: bool = False # verify usage.reasoning_tokens > 0
|
||||
test_parallel: bool = True
|
||||
|
||||
|
||||
@@ -225,6 +226,29 @@ def _test_thinking(client, model):
|
||||
assert "8" in content, f"expected '8' in content, got: {content}"
|
||||
|
||||
|
||||
def _test_reasoning_usage(client, model):
|
||||
"""With thinking enabled, usage.reasoning_tokens should be reported as > 0."""
|
||||
thinking_body = {"thinking": {"type": "enabled", "budget_tokens": 1024}}
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is 3 + 5?"}],
|
||||
tools=[ADD_TOOL_STRICT],
|
||||
tool_choice="required",
|
||||
temperature=0.1,
|
||||
extra_body=thinking_body,
|
||||
)
|
||||
usage = response.usage
|
||||
assert usage is not None, "usage should not be None"
|
||||
assert (
|
||||
usage.reasoning_tokens and usage.reasoning_tokens > 0
|
||||
), f"expected reasoning_tokens > 0, got {usage.reasoning_tokens}"
|
||||
if usage.completion_tokens_details:
|
||||
detail_reasoning = usage.completion_tokens_details.get("reasoning_tokens", 0)
|
||||
assert (
|
||||
detail_reasoning > 0
|
||||
), f"expected completion_tokens_details.reasoning_tokens > 0, got {detail_reasoning}"
|
||||
|
||||
|
||||
def _test_parallel(client, model):
|
||||
"""Single request should return multiple tool calls."""
|
||||
response = _call(
|
||||
@@ -246,6 +270,7 @@ _TESTS = [
|
||||
("strict", _test_strict, "test_strict"),
|
||||
("multiturn", _test_multiturn, "test_multiturn"),
|
||||
("thinking", _test_thinking, "test_thinking"),
|
||||
("reasoning_usage", _test_reasoning_usage, "test_reasoning_usage"),
|
||||
("parallel", _test_parallel, "test_parallel"),
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user