diff --git a/python/sglang/test/run_combined_tests.py b/python/sglang/test/run_combined_tests.py index 07dc59976..f6f630db5 100644 --- a/python/sglang/test/run_combined_tests.py +++ b/python/sglang/test/run_combined_tests.py @@ -14,6 +14,11 @@ from sglang.test.performance_test_runner import ( run_performance_test, ) from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings, is_in_ci +from sglang.test.tool_call_test_runner import ( + ToolCallTestParams, + ToolCallTestResult, + run_tool_call_test, +) def run_combined_tests( @@ -23,8 +28,9 @@ def run_combined_tests( is_vlm: bool = False, accuracy_params: Optional[AccuracyTestParams] = None, performance_params: Optional[PerformanceTestParams] = None, + tool_call_params: Optional[ToolCallTestParams] = None, ) -> dict: - """Run performance and/or accuracy tests for a list of models. + """Run performance, accuracy, and/or tool call tests for a list of models. Args: models: List of ModelLaunchSettings to test @@ -33,6 +39,7 @@ def run_combined_tests( is_vlm: Whether these are VLM models (affects defaults) accuracy_params: Parameters for accuracy tests (None to skip accuracy) performance_params: Parameters for performance tests (None to skip perf) + tool_call_params: Parameters for tool call tests (None to skip tool call) Returns: dict with test results: @@ -52,6 +59,7 @@ def run_combined_tests( base_url = base_url or DEFAULT_URL_FOR_TEST run_perf = performance_params is not None run_accuracy = accuracy_params is not None + run_tool_call = tool_call_params is not None # Print test header print("\n" + "=" * 80) @@ -61,6 +69,8 @@ def run_combined_tests( print(f" Accuracy dataset: {accuracy_params.dataset}") if run_perf: print(f" Performance batches: {performance_params.batch_sizes}") + if run_tool_call: + print(" Tool call tests: enabled") print("=" * 80) # Set up performance parameters @@ -96,6 +106,7 @@ def run_combined_tests( "model": model.model_path, "perf_result": None, "accuracy_result": None, + "tool_call_result": None, "errors": [], } @@ -136,6 +147,21 @@ def run_combined_tests( print("\nWaiting 20 seconds for resource cleanup...") time.sleep(20) + # Run tool call test + if run_tool_call: + tc_result: ToolCallTestResult = run_tool_call_test( + model=model, + params=tool_call_params, + base_url=base_url, + ) + model_result["tool_call_result"] = tc_result + if not tc_result.passed: + all_passed = False + model_result["errors"].extend(tc_result.failures) + + print("\nWaiting 20 seconds for resource cleanup...") + time.sleep(20) + all_results.append(model_result) # Write performance report if we ran perf tests @@ -180,6 +206,11 @@ def run_combined_tests( print(f" Accuracy: {'PASS' if acc.passed else 'FAIL'}") if acc.score is not None: print(f" Score: {acc.score:.3f}") + if run_tool_call and model_result["tool_call_result"]: + tc = model_result["tool_call_result"] + print( + f" Tool Call: {'PASS' if tc.passed else 'FAIL'} ({tc.num_passed}/{tc.num_total})" + ) if model_result["errors"]: print(f" Errors: {model_result['errors']}") diff --git a/python/sglang/test/tool_call_test_runner.py b/python/sglang/test/tool_call_test_runner.py new file mode 100644 index 000000000..ffa9c3048 --- /dev/null +++ b/python/sglang/test/tool_call_test_runner.py @@ -0,0 +1,320 @@ +import json +from dataclasses import dataclass +from typing import List, Optional + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + ModelLaunchSettings, + popen_launch_server, +) + + +@dataclass +class ToolCallTestParams: + test_basic: bool = True + test_streaming: bool = True + test_required: bool = True + test_none: bool = True + test_specific: bool = True + test_strict: bool = True + test_multiturn: bool = True + test_thinking: bool = False # model-specific, e.g. DeepSeek + test_parallel: bool = True + + +@dataclass +class ToolCallTestResult: + model: str + passed: bool + num_passed: int + num_total: int + failures: List[str] + variant: Optional[str] = None + + +# ---- tool definitions ---- + +ADD_TOOL = { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two integers", + "parameters": { + "type": "object", + "properties": { + "a": {"type": "integer", "description": "First integer"}, + "b": {"type": "integer", "description": "Second integer"}, + }, + "required": ["a", "b"], + }, + }, +} + +WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + }, + "required": ["city"], + }, + }, +} + +ADD_TOOL_STRICT = { + "type": "function", + "function": {**ADD_TOOL["function"], "strict": True}, +} + + +def _call(client, model, content, tools=None, tool_choice="required", **kwargs): + """Single-turn tool call request. Defaults to ADD_TOOL_STRICT + required.""" + return client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": content}], + tools=tools or [ADD_TOOL_STRICT], + tool_choice=tool_choice, + temperature=0.1, + **kwargs, + ) + + +# ---- test cases ---- + + +def _test_basic_format(client, model): + """Format + field placement: tool_calls present, content empty, valid JSON args.""" + response = _call(client, model, "Compute 3 + 5") + msg = response.choices[0].message + assert msg.tool_calls and len(msg.tool_calls) > 0 + assert not msg.content, f"content should be empty, got: {msg.content}" + tc = msg.tool_calls[0] + assert tc.function.name == "add", f"expected 'add', got '{tc.function.name}'" + assert isinstance(json.loads(tc.function.arguments), dict) + assert response.choices[0].finish_reason == "tool_calls" + + +def _test_streaming(client, model): + """Streaming chunks should concatenate to valid JSON.""" + response = _call(client, model, "Compute 5 + 7", stream=True) + chunks = list(response) + assert len(chunks) > 0 + arg_fragments = [] + name = None + for chunk in chunks: + if chunk.choices[0].delta.tool_calls: + tc = chunk.choices[0].delta.tool_calls[0] + name = tc.function.name or name + if tc.function.arguments: + arg_fragments.append(tc.function.arguments) + assert name == "add", f"expected 'add', got '{name}'" + args = json.loads("".join(arg_fragments)) + assert "a" in args and "b" in args + assert chunks[-1].choices[0].finish_reason == "tool_calls" + + +def _test_required(client, model): + """tool_choice='required' must return a tool call even for unrelated queries.""" + response = _call( + client, + model, + "What is the capital of France?", + tools=[ADD_TOOL, WEATHER_TOOL], + ) + assert response.choices[0].message.tool_calls + + +def _test_none(client, model): + """tool_choice='none' must not return any tool call.""" + response = _call(client, model, "What is 1+1?", tool_choice="none") + assert response.choices[0].message.tool_calls is None + assert response.choices[0].finish_reason == "stop" + + +def _test_specific(client, model): + """Specifying a function name should return that function.""" + response = _call( + client, + model, + "What is the capital of France?", + tools=[ADD_TOOL, WEATHER_TOOL], + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + ) + tc = response.choices[0].message.tool_calls + assert tc and tc[0].function.name == "get_weather" + + +def _test_strict(client, model): + """strict: true should enforce schema on arguments.""" + response = _call(client, model, "Compute 5 - 7") + args = json.loads(response.choices[0].message.tool_calls[0].function.arguments) + assert "a" in args and "b" in args + + +def _test_multiturn(client, model): + """Pass tool result back, model should reply based on it.""" + # turn 1: get tool call + messages = [{"role": "user", "content": "What is 3 + 5?"}] + r1 = client.chat.completions.create( + model=model, + messages=messages, + tools=[ADD_TOOL_STRICT], + tool_choice="required", + temperature=0.1, + ) + tc = r1.choices[0].message.tool_calls[0] + # turn 2: pass result back + messages.append(r1.choices[0].message) + messages.append( + { + "role": "tool", + "tool_call_id": tc.id, + "content": "8", + "name": tc.function.name, + } + ) + r2 = client.chat.completions.create( + model=model, + messages=messages, + tools=[ADD_TOOL], + temperature=0.1, + ) + assert "8" in (r2.choices[0].message.content or "") + + +def _test_thinking(client, model): + """After tool result with thinking enabled, output should be in content not reasoning_content.""" + thinking_body = {"thinking": {"type": "enabled", "budget_tokens": 1024}} + # turn 1 + messages = [{"role": "user", "content": "What is 3 + 5?"}] + r1 = client.chat.completions.create( + model=model, + messages=messages, + tools=[ADD_TOOL_STRICT], + tool_choice="required", + temperature=0.1, + extra_body=thinking_body, + ) + tc = r1.choices[0].message.tool_calls[0] + # turn 2 + messages.append(r1.choices[0].message) + messages.append( + { + "role": "tool", + "tool_call_id": tc.id, + "content": "8", + "name": tc.function.name, + } + ) + r2 = client.chat.completions.create( + model=model, + messages=messages, + tools=[ADD_TOOL], + temperature=0.1, + extra_body=thinking_body, + ) + content = r2.choices[0].message.content or "" + assert "8" in content, f"expected '8' in content, got: {content}" + + +def _test_parallel(client, model): + """Single request should return multiple tool calls.""" + response = _call( + client, + model, + "What is 3+5 and what is the weather in Tokyo?", + tools=[ADD_TOOL, WEATHER_TOOL], + ) + tc = response.choices[0].message.tool_calls + assert tc and len(tc) >= 2, f"expected >= 2 tool calls, got {len(tc) if tc else 0}" + + +_TESTS = [ + ("basic_format", _test_basic_format, "test_basic"), + ("streaming", _test_streaming, "test_streaming"), + ("required", _test_required, "test_required"), + ("none", _test_none, "test_none"), + ("specific", _test_specific, "test_specific"), + ("strict", _test_strict, "test_strict"), + ("multiturn", _test_multiturn, "test_multiturn"), + ("thinking", _test_thinking, "test_thinking"), + ("parallel", _test_parallel, "test_parallel"), +] + + +# ---- runner ---- + + +def run_tool_call_test( + model: ModelLaunchSettings, + params: ToolCallTestParams, + base_url: Optional[str] = None, +) -> ToolCallTestResult: + """Launch server, run enabled test cases, return results.""" + base_url = base_url or DEFAULT_URL_FOR_TEST + + print(f"\n{'='*60}") + print(f"Running TOOL CALL test for {model.model_path}") + if model.variant: + print(f" Variant: {model.variant}") + print(f"{'='*60}\n") + + process = None + try: + process = popen_launch_server( + model.model_path, + base_url, + other_args=model.extra_args, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + env=model.env, + ) + client = openai.Client(api_key="sk-test", base_url=base_url + "/v1") + + passed_list = [] + failed_list = [] + + for name, fn, flag in _TESTS: + if not getattr(params, flag): + continue + try: + fn(client, model.model_path) + passed_list.append(name) + print(f" PASS: {name}") + except Exception as e: + failed_list.append(f"{name}: {e}") + print(f" FAIL: {name}: {e}") + + total = len(passed_list) + len(failed_list) + print(f"\n Result: {len(passed_list)}/{total} passed") + + return ToolCallTestResult( + model=model.model_path, + passed=len(failed_list) == 0, + num_passed=len(passed_list), + num_total=total, + failures=failed_list, + variant=model.variant, + ) + + except Exception as e: + print(f" Server launch failed: {e}") + return ToolCallTestResult( + model=model.model_path, + passed=False, + num_passed=0, + num_total=0, + failures=[f"Server launch failed: {e}"], + variant=model.variant, + ) + + finally: + if process: + kill_process_tree(process.pid) diff --git a/test/registered/8-gpu-models/test_deepseek_v32.py b/test/registered/8-gpu-models/test_deepseek_v32.py index 5bf73bb74..6fcca87c8 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32.py +++ b/test/registered/8-gpu-models/test_deepseek_v32.py @@ -5,6 +5,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.performance_test_runner import PerformanceTestParams from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system +from sglang.test.tool_call_test_runner import ToolCallTestParams register_cuda_ci(est_time=5400, suite="nightly-8-gpu-common", nightly=True) @@ -16,6 +17,11 @@ BASE_ARGS = [ '{"enable_multithread_load": true}', ] +TOOL_CALL_ARGS = [ + "--tool-call-parser=deepseekv32", + "--reasoning-parser=deepseek-v3", +] + DP_ARGS = [ "--tp=8", "--dp=8", @@ -54,28 +60,28 @@ class TestDeepseekV32(unittest.TestCase): ModelLaunchSettings( DEEPSEEK_V32_MODEL_PATH, tp_size=8, - extra_args=BASE_ARGS + DP_ARGS, + extra_args=BASE_ARGS + DP_ARGS + TOOL_CALL_ARGS, variant="DP8", ), # Variant: "dp+mtp" - DP + EAGLE speculative decoding ModelLaunchSettings( DEEPSEEK_V32_MODEL_PATH, tp_size=8, - extra_args=BASE_ARGS + DP_ARGS + MTP_ARGS, + extra_args=BASE_ARGS + DP_ARGS + TOOL_CALL_ARGS + MTP_ARGS, variant="DP8+MTP", ), # Variant: "tp" - Pure TP=8 only ModelLaunchSettings( DEEPSEEK_V32_MODEL_PATH, tp_size=8, - extra_args=BASE_ARGS + TP_ARGS, + extra_args=BASE_ARGS + TP_ARGS + TOOL_CALL_ARGS, variant="TP8", ), # Variant: "tp+mtp" - Pure TP=8 + EAGLE speculative decoding ModelLaunchSettings( DEEPSEEK_V32_MODEL_PATH, tp_size=8, - extra_args=BASE_ARGS + TP_ARGS + MTP_ARGS, + extra_args=BASE_ARGS + TP_ARGS + TOOL_CALL_ARGS + MTP_ARGS, variant="TP8+MTP", ), ] @@ -90,6 +96,7 @@ class TestDeepseekV32(unittest.TestCase): batch_sizes=[1, 8, 16, 64], profile_dir="performance_profiles_deepseek_v32", ), + tool_call_params=ToolCallTestParams(test_thinking=True), ) @unittest.skipIf(is_blackwell_system(), "Requires H200 system")