Add tool call tests for DeepSeek V3.2 in nightly CI (#17951)
This commit is contained in:
Hudson Xing
@@ -14,6 +14,11 @@ from sglang.test.performance_test_runner import (
|
||||
run_performance_test,
|
||||
)
|
||||
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings, is_in_ci
|
||||
from sglang.test.tool_call_test_runner import (
|
||||
ToolCallTestParams,
|
||||
ToolCallTestResult,
|
||||
run_tool_call_test,
|
||||
)
|
||||
|
||||
|
||||
def run_combined_tests(
|
||||
@@ -23,8 +28,9 @@ def run_combined_tests(
|
||||
is_vlm: bool = False,
|
||||
accuracy_params: Optional[AccuracyTestParams] = None,
|
||||
performance_params: Optional[PerformanceTestParams] = None,
|
||||
tool_call_params: Optional[ToolCallTestParams] = None,
|
||||
) -> dict:
|
||||
"""Run performance and/or accuracy tests for a list of models.
|
||||
"""Run performance, accuracy, and/or tool call tests for a list of models.
|
||||
|
||||
Args:
|
||||
models: List of ModelLaunchSettings to test
|
||||
@@ -33,6 +39,7 @@ def run_combined_tests(
|
||||
is_vlm: Whether these are VLM models (affects defaults)
|
||||
accuracy_params: Parameters for accuracy tests (None to skip accuracy)
|
||||
performance_params: Parameters for performance tests (None to skip perf)
|
||||
tool_call_params: Parameters for tool call tests (None to skip tool call)
|
||||
|
||||
Returns:
|
||||
dict with test results:
|
||||
@@ -52,6 +59,7 @@ def run_combined_tests(
|
||||
base_url = base_url or DEFAULT_URL_FOR_TEST
|
||||
run_perf = performance_params is not None
|
||||
run_accuracy = accuracy_params is not None
|
||||
run_tool_call = tool_call_params is not None
|
||||
|
||||
# Print test header
|
||||
print("\n" + "=" * 80)
|
||||
@@ -61,6 +69,8 @@ def run_combined_tests(
|
||||
print(f" Accuracy dataset: {accuracy_params.dataset}")
|
||||
if run_perf:
|
||||
print(f" Performance batches: {performance_params.batch_sizes}")
|
||||
if run_tool_call:
|
||||
print(" Tool call tests: enabled")
|
||||
print("=" * 80)
|
||||
|
||||
# Set up performance parameters
|
||||
@@ -96,6 +106,7 @@ def run_combined_tests(
|
||||
"model": model.model_path,
|
||||
"perf_result": None,
|
||||
"accuracy_result": None,
|
||||
"tool_call_result": None,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
@@ -136,6 +147,21 @@ def run_combined_tests(
|
||||
print("\nWaiting 20 seconds for resource cleanup...")
|
||||
time.sleep(20)
|
||||
|
||||
# Run tool call test
|
||||
if run_tool_call:
|
||||
tc_result: ToolCallTestResult = run_tool_call_test(
|
||||
model=model,
|
||||
params=tool_call_params,
|
||||
base_url=base_url,
|
||||
)
|
||||
model_result["tool_call_result"] = tc_result
|
||||
if not tc_result.passed:
|
||||
all_passed = False
|
||||
model_result["errors"].extend(tc_result.failures)
|
||||
|
||||
print("\nWaiting 20 seconds for resource cleanup...")
|
||||
time.sleep(20)
|
||||
|
||||
all_results.append(model_result)
|
||||
|
||||
# Write performance report if we ran perf tests
|
||||
@@ -180,6 +206,11 @@ def run_combined_tests(
|
||||
print(f" Accuracy: {'PASS' if acc.passed else 'FAIL'}")
|
||||
if acc.score is not None:
|
||||
print(f" Score: {acc.score:.3f}")
|
||||
if run_tool_call and model_result["tool_call_result"]:
|
||||
tc = model_result["tool_call_result"]
|
||||
print(
|
||||
f" Tool Call: {'PASS' if tc.passed else 'FAIL'} ({tc.num_passed}/{tc.num_total})"
|
||||
)
|
||||
if model_result["errors"]:
|
||||
print(f" Errors: {model_result['errors']}")
|
||||
|
||||
|
||||
320
python/sglang/test/tool_call_test_runner.py
Normal file
320
python/sglang/test/tool_call_test_runner.py
Normal file
@@ -0,0 +1,320 @@
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
|
||||
import openai
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
ModelLaunchSettings,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallTestParams:
|
||||
test_basic: bool = True
|
||||
test_streaming: bool = True
|
||||
test_required: bool = True
|
||||
test_none: bool = True
|
||||
test_specific: bool = True
|
||||
test_strict: bool = True
|
||||
test_multiturn: bool = True
|
||||
test_thinking: bool = False # model-specific, e.g. DeepSeek
|
||||
test_parallel: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallTestResult:
|
||||
model: str
|
||||
passed: bool
|
||||
num_passed: int
|
||||
num_total: int
|
||||
failures: List[str]
|
||||
variant: Optional[str] = None
|
||||
|
||||
|
||||
# ---- tool definitions ----
|
||||
|
||||
ADD_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "add",
|
||||
"description": "Compute the sum of two integers",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": {"type": "integer", "description": "First integer"},
|
||||
"b": {"type": "integer", "description": "Second integer"},
|
||||
},
|
||||
"required": ["a", "b"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
WEATHER_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a city",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string", "description": "City name"},
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ADD_TOOL_STRICT = {
|
||||
"type": "function",
|
||||
"function": {**ADD_TOOL["function"], "strict": True},
|
||||
}
|
||||
|
||||
|
||||
def _call(client, model, content, tools=None, tool_choice="required", **kwargs):
|
||||
"""Single-turn tool call request. Defaults to ADD_TOOL_STRICT + required."""
|
||||
return client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": content}],
|
||||
tools=tools or [ADD_TOOL_STRICT],
|
||||
tool_choice=tool_choice,
|
||||
temperature=0.1,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ---- test cases ----
|
||||
|
||||
|
||||
def _test_basic_format(client, model):
|
||||
"""Format + field placement: tool_calls present, content empty, valid JSON args."""
|
||||
response = _call(client, model, "Compute 3 + 5")
|
||||
msg = response.choices[0].message
|
||||
assert msg.tool_calls and len(msg.tool_calls) > 0
|
||||
assert not msg.content, f"content should be empty, got: {msg.content}"
|
||||
tc = msg.tool_calls[0]
|
||||
assert tc.function.name == "add", f"expected 'add', got '{tc.function.name}'"
|
||||
assert isinstance(json.loads(tc.function.arguments), dict)
|
||||
assert response.choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
def _test_streaming(client, model):
|
||||
"""Streaming chunks should concatenate to valid JSON."""
|
||||
response = _call(client, model, "Compute 5 + 7", stream=True)
|
||||
chunks = list(response)
|
||||
assert len(chunks) > 0
|
||||
arg_fragments = []
|
||||
name = None
|
||||
for chunk in chunks:
|
||||
if chunk.choices[0].delta.tool_calls:
|
||||
tc = chunk.choices[0].delta.tool_calls[0]
|
||||
name = tc.function.name or name
|
||||
if tc.function.arguments:
|
||||
arg_fragments.append(tc.function.arguments)
|
||||
assert name == "add", f"expected 'add', got '{name}'"
|
||||
args = json.loads("".join(arg_fragments))
|
||||
assert "a" in args and "b" in args
|
||||
assert chunks[-1].choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
def _test_required(client, model):
|
||||
"""tool_choice='required' must return a tool call even for unrelated queries."""
|
||||
response = _call(
|
||||
client,
|
||||
model,
|
||||
"What is the capital of France?",
|
||||
tools=[ADD_TOOL, WEATHER_TOOL],
|
||||
)
|
||||
assert response.choices[0].message.tool_calls
|
||||
|
||||
|
||||
def _test_none(client, model):
|
||||
"""tool_choice='none' must not return any tool call."""
|
||||
response = _call(client, model, "What is 1+1?", tool_choice="none")
|
||||
assert response.choices[0].message.tool_calls is None
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
def _test_specific(client, model):
|
||||
"""Specifying a function name should return that function."""
|
||||
response = _call(
|
||||
client,
|
||||
model,
|
||||
"What is the capital of France?",
|
||||
tools=[ADD_TOOL, WEATHER_TOOL],
|
||||
tool_choice={"type": "function", "function": {"name": "get_weather"}},
|
||||
)
|
||||
tc = response.choices[0].message.tool_calls
|
||||
assert tc and tc[0].function.name == "get_weather"
|
||||
|
||||
|
||||
def _test_strict(client, model):
|
||||
"""strict: true should enforce schema on arguments."""
|
||||
response = _call(client, model, "Compute 5 - 7")
|
||||
args = json.loads(response.choices[0].message.tool_calls[0].function.arguments)
|
||||
assert "a" in args and "b" in args
|
||||
|
||||
|
||||
def _test_multiturn(client, model):
|
||||
"""Pass tool result back, model should reply based on it."""
|
||||
# turn 1: get tool call
|
||||
messages = [{"role": "user", "content": "What is 3 + 5?"}]
|
||||
r1 = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=[ADD_TOOL_STRICT],
|
||||
tool_choice="required",
|
||||
temperature=0.1,
|
||||
)
|
||||
tc = r1.choices[0].message.tool_calls[0]
|
||||
# turn 2: pass result back
|
||||
messages.append(r1.choices[0].message)
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.id,
|
||||
"content": "8",
|
||||
"name": tc.function.name,
|
||||
}
|
||||
)
|
||||
r2 = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=[ADD_TOOL],
|
||||
temperature=0.1,
|
||||
)
|
||||
assert "8" in (r2.choices[0].message.content or "")
|
||||
|
||||
|
||||
def _test_thinking(client, model):
|
||||
"""After tool result with thinking enabled, output should be in content not reasoning_content."""
|
||||
thinking_body = {"thinking": {"type": "enabled", "budget_tokens": 1024}}
|
||||
# turn 1
|
||||
messages = [{"role": "user", "content": "What is 3 + 5?"}]
|
||||
r1 = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=[ADD_TOOL_STRICT],
|
||||
tool_choice="required",
|
||||
temperature=0.1,
|
||||
extra_body=thinking_body,
|
||||
)
|
||||
tc = r1.choices[0].message.tool_calls[0]
|
||||
# turn 2
|
||||
messages.append(r1.choices[0].message)
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.id,
|
||||
"content": "8",
|
||||
"name": tc.function.name,
|
||||
}
|
||||
)
|
||||
r2 = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=[ADD_TOOL],
|
||||
temperature=0.1,
|
||||
extra_body=thinking_body,
|
||||
)
|
||||
content = r2.choices[0].message.content or ""
|
||||
assert "8" in content, f"expected '8' in content, got: {content}"
|
||||
|
||||
|
||||
def _test_parallel(client, model):
|
||||
"""Single request should return multiple tool calls."""
|
||||
response = _call(
|
||||
client,
|
||||
model,
|
||||
"What is 3+5 and what is the weather in Tokyo?",
|
||||
tools=[ADD_TOOL, WEATHER_TOOL],
|
||||
)
|
||||
tc = response.choices[0].message.tool_calls
|
||||
assert tc and len(tc) >= 2, f"expected >= 2 tool calls, got {len(tc) if tc else 0}"
|
||||
|
||||
|
||||
_TESTS = [
|
||||
("basic_format", _test_basic_format, "test_basic"),
|
||||
("streaming", _test_streaming, "test_streaming"),
|
||||
("required", _test_required, "test_required"),
|
||||
("none", _test_none, "test_none"),
|
||||
("specific", _test_specific, "test_specific"),
|
||||
("strict", _test_strict, "test_strict"),
|
||||
("multiturn", _test_multiturn, "test_multiturn"),
|
||||
("thinking", _test_thinking, "test_thinking"),
|
||||
("parallel", _test_parallel, "test_parallel"),
|
||||
]
|
||||
|
||||
|
||||
# ---- runner ----
|
||||
|
||||
|
||||
def run_tool_call_test(
|
||||
model: ModelLaunchSettings,
|
||||
params: ToolCallTestParams,
|
||||
base_url: Optional[str] = None,
|
||||
) -> ToolCallTestResult:
|
||||
"""Launch server, run enabled test cases, return results."""
|
||||
base_url = base_url or DEFAULT_URL_FOR_TEST
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Running TOOL CALL test for {model.model_path}")
|
||||
if model.variant:
|
||||
print(f" Variant: {model.variant}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
process = None
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
model.model_path,
|
||||
base_url,
|
||||
other_args=model.extra_args,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
env=model.env,
|
||||
)
|
||||
client = openai.Client(api_key="sk-test", base_url=base_url + "/v1")
|
||||
|
||||
passed_list = []
|
||||
failed_list = []
|
||||
|
||||
for name, fn, flag in _TESTS:
|
||||
if not getattr(params, flag):
|
||||
continue
|
||||
try:
|
||||
fn(client, model.model_path)
|
||||
passed_list.append(name)
|
||||
print(f" PASS: {name}")
|
||||
except Exception as e:
|
||||
failed_list.append(f"{name}: {e}")
|
||||
print(f" FAIL: {name}: {e}")
|
||||
|
||||
total = len(passed_list) + len(failed_list)
|
||||
print(f"\n Result: {len(passed_list)}/{total} passed")
|
||||
|
||||
return ToolCallTestResult(
|
||||
model=model.model_path,
|
||||
passed=len(failed_list) == 0,
|
||||
num_passed=len(passed_list),
|
||||
num_total=total,
|
||||
failures=failed_list,
|
||||
variant=model.variant,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f" Server launch failed: {e}")
|
||||
return ToolCallTestResult(
|
||||
model=model.model_path,
|
||||
passed=False,
|
||||
num_passed=0,
|
||||
num_total=0,
|
||||
failures=[f"Server launch failed: {e}"],
|
||||
variant=model.variant,
|
||||
)
|
||||
|
||||
finally:
|
||||
if process:
|
||||
kill_process_tree(process.pid)
|
||||
Reference in New Issue
Block a user