chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 15 Packages Projects Releases Wiki Activity

[router] add basic ci tests for gpt-oss model support (#12651)

Browse Source 
Co-authored-by: Chang Su <chang.s.su@oracle.com>
This commit is contained in:
Keyang Ru
2025-11-05 15:45:11 -08:00
committed by GitHub
co-authored by Chang Su
parent b0476a0642
commit 7ef1964a66
21 changed files with 1008 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py
+275
View File
@@ -0,0 +1,275 @@
"""
gRPC backend tests for Response API (including Harmony).
Run with:
python3 -m pytest py_test/e2e_response_api/backends/test_grpc_backend.py -v
python3 -m unittest e2e_response_api.backends.test_grpc_backend.TestGrpcBackend
"""
import sys
import unittest
from pathlib import Path
# Add e2e_response_api directory for imports
_TEST_DIR = Path(__file__).parent.parent
sys.path.insert(0, str(_TEST_DIR))
# Import local modules
from mixins.function_call import FunctionCallingBaseTest
from mixins.mcp import MCPTests
from mixins.state_management import StateManagementTests
from router_fixtures import popen_launch_workers_and_router
from util import kill_process_tree
class TestGrpcBackend(StateManagementTests, MCPTests):
"""End to end tests for gRPC backend."""
@classmethod
def setUpClass(cls):
cls.model = "/home/ubuntu/models/meta-llama/Llama-3.1-8B-Instruct"
cls.base_url_port = "http://127.0.0.1:30030"
cls.cluster = popen_launch_workers_and_router(
cls.model,
cls.base_url_port,
timeout=90,
num_workers=1,
tp_size=2,
policy="round_robin",
router_args=["--history-backend", "memory"],
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
for worker in cls.cluster.get("workers", []):
kill_process_tree(worker.pid)
@unittest.skip(
"TODO: transport error, details: [], metadata: MetadataMap { headers: {} }"
)
def test_previous_response_id_chaining(self):
super().test_previous_response_id_chaining()
@unittest.skip("TODO: return 501 Not Implemented")
def test_conversation_with_multiple_turns(self):
super().test_conversation_with_multiple_turns()
@unittest.skip("TODO: decode error message")
def test_mutually_exclusive_parameters(self):
super().test_mutually_exclusive_parameters()
@unittest.skip(
"TODO: Pipeline execution failed: Pipeline stage WorkerSelection failed"
)
def test_mcp_basic_tool_call(self):
super().test_mcp_basic_tool_call()
@unittest.skip("TODO: no event fields")
def test_mcp_basic_tool_call_streaming(self):
return super().test_mcp_basic_tool_call_streaming()
class TestHarmonyBackend(StateManagementTests, MCPTests, FunctionCallingBaseTest):
"""End to end tests for Harmony backend."""
@classmethod
def setUpClass(cls):
cls.model = "/home/ubuntu/models/openai/gpt-oss-20b"
cls.base_url_port = "http://127.0.0.1:30030"
cls.cluster = popen_launch_workers_and_router(
cls.model,
cls.base_url_port,
timeout=90,
num_workers=1,
tp_size=2,
policy="round_robin",
router_args=["--history-backend", "memory"],
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
for worker in cls.cluster.get("workers", []):
kill_process_tree(worker.pid)
def test_previous_response_id_chaining(self):
super().test_previous_response_id_chaining()
@unittest.skip(
"TODO: fix requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)"
)
def test_mutually_exclusive_parameters(self):
super().test_mutually_exclusive_parameters()
def test_mcp_basic_tool_call(self):
"""Test basic MCP tool call (non-streaming)."""
tools = [
{
"type": "mcp",
"server_label": "deepwiki",
"server_url": "https://mcp.deepwiki.com/mcp",
"require_approval": "never",
}
]
resp = self.create_response(
"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?",
tools=tools,
stream=False,
)
# Should successfully make the request
self.assertEqual(resp.status_code, 200)
data = resp.json()
# Basic response structure
self.assertIn("id", data)
self.assertIn("status", data)
self.assertEqual(data["status"], "completed")
self.assertIn("output", data)
self.assertIn("model", data)
# Verify output array is not empty
output = data["output"]
self.assertIsInstance(output, list)
self.assertGreater(len(output), 0)
# Check for MCP-specific output types
output_types = [item.get("type") for item in output]
# Should have mcp_list_tools - tools are listed before calling
self.assertIn(
"mcp_list_tools", output_types, "Response should contain mcp_list_tools"
)
# Should have at least one mcp_call
mcp_calls = [item for item in output if item.get("type") == "mcp_call"]
self.assertGreater(
len(mcp_calls), 0, "Response should contain at least one mcp_call"
)
# Verify mcp_call structure
for mcp_call in mcp_calls:
self.assertIn("id", mcp_call)
self.assertIn("status", mcp_call)
self.assertEqual(mcp_call["status"], "completed")
self.assertIn("server_label", mcp_call)
self.assertEqual(mcp_call["server_label"], "deepwiki")
self.assertIn("name", mcp_call)
self.assertIn("arguments", mcp_call)
self.assertIn("output", mcp_call)
def test_mcp_basic_tool_call_streaming(self):
"""Test basic MCP tool call (streaming)."""
tools = [
{
"type": "mcp",
"server_label": "deepwiki",
"server_url": "https://mcp.deepwiki.com/mcp",
"require_approval": "never",
}
]
resp = self.create_response(
"What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?",
tools=tools,
stream=True,
)
# Should successfully make the request
self.assertEqual(resp.status_code, 200)
events = self.parse_sse_events(resp)
self.assertGreater(len(events), 0)
event_types = [e.get("event") for e in events]
# Check for lifecycle events
self.assertIn(
"response.created", event_types, "Should have response.created event"
)
self.assertIn(
"response.completed", event_types, "Should have response.completed event"
)
# Check for MCP list tools events
self.assertIn(
"response.output_item.added",
event_types,
"Should have output_item.added events",
)
self.assertIn(
"response.mcp_list_tools.in_progress",
event_types,
"Should have mcp_list_tools.in_progress event",
)
self.assertIn(
"response.mcp_list_tools.completed",
event_types,
"Should have mcp_list_tools.completed event",
)
# Check for MCP call events
self.assertIn(
"response.mcp_call.in_progress",
event_types,
"Should have mcp_call.in_progress event",
)
self.assertIn(
"response.mcp_call_arguments.delta",
event_types,
"Should have mcp_call_arguments.delta event",
)
self.assertIn(
"response.mcp_call_arguments.done",
event_types,
"Should have mcp_call_arguments.done event",
)
self.assertIn(
"response.mcp_call.completed",
event_types,
"Should have mcp_call.completed event",
)
# Verify final completed event has full response
completed_events = [e for e in events if e.get("event") == "response.completed"]
self.assertEqual(len(completed_events), 1)
final_response = completed_events[0].get("data", {}).get("response", {})
self.assertIn("id", final_response)
self.assertEqual(final_response.get("status"), "completed")
self.assertIn("output", final_response)
# Verify final output contains expected items
final_output = final_response.get("output", [])
final_output_types = [item.get("type") for item in final_output]
self.assertIn("mcp_list_tools", final_output_types)
self.assertIn("mcp_call", final_output_types)
# Verify mcp_call items in final output
mcp_calls = [item for item in final_output if item.get("type") == "mcp_call"]
self.assertGreater(len(mcp_calls), 0)
for mcp_call in mcp_calls:
self.assertEqual(mcp_call.get("status"), "completed")
self.assertEqual(mcp_call.get("server_label"), "deepwiki")
self.assertIn("name", mcp_call)
self.assertIn("arguments", mcp_call)
self.assertIn("output", mcp_call)
@unittest.skip("TODO: 501 Not Implemented")
def test_conversation_with_multiple_turns(self):
super().test_conversation_with_multiple_turns()
if __name__ == "__main__":
unittest.main()
sgl-router/py_test/e2e_response_api/backends/test_http_backend.py
+82
View File
@@ -0,0 +1,82 @@
"""
HTTP backend tests for Response API (OpenAI and XAI).
Run with:
export OPENAI_API_KEY=your_key
export XAI_API_KEY=your_key
python3 -m pytest py_test/e2e_response_api/backends/test_http_backend.py -v
python3 -m unittest e2e_response_api.backends.test_http_backend.TestOpenaiBackend
"""
import os
import sys
import unittest
from pathlib import Path
# Add e2e_response_api directory for imports
_TEST_DIR = Path(__file__).parent.parent
sys.path.insert(0, str(_TEST_DIR))
# Import local modules
from mixins.basic_crud import ConversationCRUDBaseTest, ResponseCRUDBaseTest
from mixins.function_call import FunctionCallingBaseTest
from mixins.mcp import MCPTests
from mixins.state_management import StateManagementTests
from router_fixtures import popen_launch_openai_xai_router
from util import kill_process_tree
class TestOpenaiBackend(
ResponseCRUDBaseTest,
ConversationCRUDBaseTest,
StateManagementTests,
MCPTests,
FunctionCallingBaseTest,
):
"""End to end tests for OpenAI backend."""
api_key = os.environ.get("OPENAI_API_KEY")
@classmethod
def setUpClass(cls):
cls.model = "gpt-5-nano"
cls.base_url_port = "http://127.0.0.1:30010"
cls.cluster = popen_launch_openai_xai_router(
backend="openai",
base_url=cls.base_url_port,
history_backend="memory",
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
class TestXaiBackend(StateManagementTests):
"""End to end tests for XAI backend."""
api_key = os.environ.get("XAI_API_KEY")
@classmethod
def setUpClass(cls):
cls.model = "grok-4-fast"
cls.base_url_port = "http://127.0.0.1:30023"
cls.cluster = popen_launch_openai_xai_router(
backend="xai",
base_url=cls.base_url_port,
history_backend="memory",
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
if __name__ == "__main__":
unittest.main()
sgl-router/py_test/e2e_response_api/conftest.py
+2
View File
@@ -18,6 +18,7 @@ def pytest_collection_modifyitems(config, items):
- ConversationCRUDBaseTest
- MCPTests
- StateManagementTests
- FunctionCallingBaseTest
"""
base_class_names = {
"StateManagementBaseTest",
@@ -25,6 +26,7 @@ def pytest_collection_modifyitems(config, items):
"ConversationCRUDBaseTest",
"MCPTests",
"StateManagementTests",
"FunctionCallingBaseTest",
}
# Filter out tests from base classes
sgl-router/py_test/e2e_response_api/base.py → sgl-router/py_test/e2e_response_api/mixins/basic_crud.py
View File
sgl-router/py_test/e2e_response_api/mixins/function_call.py
+202
View File
@@ -0,0 +1,202 @@
"""
Base test class for function calling tests.
This module provides test cases for function calling functionality
across different backends.
"""
import json
import sys
from pathlib import Path
# Add current directory for local imports
_TEST_DIR = Path(__file__).parent
sys.path.insert(0, str(_TEST_DIR))
from util import CustomTestCase
class ResponseAPIBaseTest(CustomTestCase):
"""Base class for Response API tests with common utilities."""
# To be set by subclasses
base_url: str = None
api_key: str = None
model: str = None
def make_request(
self,
endpoint: str,
method: str = "POST",
json_data: dict = None,
params: dict = None,
):
"""
Make HTTP request to router.
This is a minimal implementation - subclasses should import from basic_crud.
"""
import requests
url = f"{self.base_url}{endpoint}"
headers = {"Content-Type": "application/json"}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
if method == "POST":
resp = requests.post(url, json=json_data, headers=headers, params=params)
elif method == "GET":
resp = requests.get(url, headers=headers, params=params)
elif method == "DELETE":
resp = requests.delete(url, headers=headers, params=params)
else:
raise ValueError(f"Unsupported method: {method}")
return resp
class FunctionCallingBaseTest(ResponseAPIBaseTest):
"""Base class for function calling tests."""
def test_basic_function_call(self):
"""
Test basic function calling workflow.
This test follows the pattern from function_call_test.py:
1. Define a function tool (get_horoscope)
2. Send user message asking for horoscope
3. Model should return function_call
4. Execute function locally and provide output
5. Model should generate final response using the function output
"""
# 1. Define a list of callable tools for the model
tools = [
{
"type": "function",
"name": "get_horoscope",
"description": "Get today's horoscope for an astrological sign.",
"parameters": {
"type": "object",
"properties": {
"sign": {
"type": "string",
"description": "An astrological sign like Taurus or Aquarius",
},
},
"required": ["sign"],
},
},
]
# Create a running input list we will add to over time
input_list = [
{"role": "user", "content": "What is my horoscope? I am an Aquarius."}
]
# 2. Prompt the model with tools defined
resp = self.make_request(
"/v1/responses",
"POST",
{
"model": self.model,
"tools": tools,
"input": input_list,
},
)
# Should successfully make the request
self.assertEqual(resp.status_code, 200)
data = resp.json()
# Basic response structure
self.assertIn("id", data)
self.assertIn("status", data)
self.assertEqual(data["status"], "completed")
self.assertIn("output", data)
# Verify output array is not empty
output = data["output"]
self.assertIsInstance(output, list)
self.assertGreater(len(output), 0)
# Check for function_call in output
function_calls = [
item for item in output if item.get("type") == "function_call"
]
self.assertGreater(
len(function_calls), 0, "Response should contain at least one function_call"
)
# Verify function_call structure
function_call = function_calls[0]
self.assertIn("call_id", function_call)
self.assertIn("name", function_call)
self.assertEqual(function_call["name"], "get_horoscope")
self.assertIn("arguments", function_call)
# Parse arguments
args = json.loads(function_call["arguments"])
self.assertIn("sign", args)
self.assertEqual(args["sign"].lower(), "aquarius")
# 3. Save function call outputs for subsequent requests
input_list += output
# 4. Execute the function logic for get_horoscope
horoscope = f"{args['sign']}: Next Tuesday you will befriend a baby otter."
# 5. Provide function call results to the model
input_list.append(
{
"type": "function_call_output",
"call_id": function_call["call_id"],
"output": json.dumps({"horoscope": horoscope}),
}
)
# 6. Make second request with function output
resp2 = self.make_request(
"/v1/responses",
"POST",
{
"model": self.model,
"instructions": "Respond only with a horoscope generated by a tool.",
"tools": tools,
"input": input_list,
},
)
self.assertEqual(resp2.status_code, 200)
data2 = resp2.json()
self.assertEqual(data2["status"], "completed")
# The model should be able to give a response using the function output
output2 = data2["output"]
self.assertGreater(len(output2), 0)
# Find message output
messages = [item for item in output2 if item.get("type") == "message"]
self.assertGreater(
len(messages), 0, "Response should contain at least one message"
)
# Verify message contains the horoscope
message = messages[0]
self.assertIn("content", message)
content_parts = message["content"]
self.assertGreater(len(content_parts), 0)
# Get text from content
text_parts = [
part.get("text", "")
for part in content_parts
if part.get("type") == "output_text"
]
full_text = " ".join(text_parts).lower()
# Should mention the horoscope or baby otter
self.assertTrue(
"baby otter" in full_text or "aquarius" in full_text,
"Response should reference the horoscope content",
)
sgl-router/py_test/e2e_response_api/mcp.py → sgl-router/py_test/e2e_response_api/mixins/mcp.py
+1 -1
View File
@@ -5,7 +5,7 @@ Tests MCP tool calling in both streaming and non-streaming modes.
These tests should work across all backends that support MCP (OpenAI, XAI).
"""
from base import ResponseAPIBaseTest
from basic_crud import ResponseAPIBaseTest
class MCPTests(ResponseAPIBaseTest):
sgl-router/py_test/e2e_response_api/state_management.py → sgl-router/py_test/e2e_response_api/mixins/state_management.py
+1 -1
View File
@@ -7,7 +7,7 @@ These tests should work across all backends (OpenAI, XAI, gRPC).
import unittest
from base import ResponseAPIBaseTest
from basic_crud import ResponseAPIBaseTest
class StateManagementTests(ResponseAPIBaseTest):
sgl-router/py_test/e2e_response_api/persistence/test_oracle_store.py
+49
View File
@@ -0,0 +1,49 @@
"""
Oracle database storage backend tests for Response API.
Run with:
export OPENAI_API_KEY=your_key
python3 -m pytest py_test/e2e_response_api/persistence/test_oracle_store.py -v
python3 -m unittest e2e_response_api.persistence.test_oracle_store.TestOracleStore
"""
import os
import sys
import unittest
from pathlib import Path
# Add e2e_response_api directory for imports
_TEST_DIR = Path(__file__).parent.parent
sys.path.insert(0, str(_TEST_DIR))
# Import local modules
from mixins.basic_crud import ConversationCRUDBaseTest, ResponseCRUDBaseTest
from router_fixtures import popen_launch_openai_xai_router
from util import kill_process_tree
class TestOracleStore(ResponseCRUDBaseTest, ConversationCRUDBaseTest):
"""End to end tests for Oracle database storage backend."""
api_key = os.environ.get("OPENAI_API_KEY")
@classmethod
def setUpClass(cls):
cls.model = "gpt-5-nano"
cls.base_url_port = "http://127.0.0.1:30040"
cls.cluster = popen_launch_openai_xai_router(
backend="openai",
base_url=cls.base_url_port,
history_backend="oracle",
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
if __name__ == "__main__":
unittest.main()
sgl-router/py_test/e2e_response_api/router_fixtures.py
-2
View File
@@ -425,8 +425,6 @@ def popen_launch_workers_and_router(
"--grpc-mode", # Enable gRPC for this worker
"--mem-fraction-static",
"0.8",
"--attention-backend",
"fa3",
]
# Add TP size
sgl-router/py_test/e2e_response_api/test_response_api.py
-153
View File
@@ -1,153 +0,0 @@
"""
OpenAI backend tests for Response API.
Run with:
export OPENAI_API_KEY=your_key
python3 -m pytest py_test/e2e_response_api/test_openai_backend.py -v
python3 -m unittest e2e_response_api.test_openai_backend.TestOpenAIStateManagement
"""
import os
import sys
import unittest
from pathlib import Path
# Add current directory for imports
_TEST_DIR = Path(__file__).parent
sys.path.insert(0, str(_TEST_DIR))
# Import local modules
from base import ConversationCRUDBaseTest, ResponseCRUDBaseTest
from mcp import MCPTests
from router_fixtures import (
popen_launch_openai_xai_router,
popen_launch_workers_and_router,
)
from state_management import StateManagementTests
from util import kill_process_tree
class TestOpenaiBackend(
ResponseCRUDBaseTest, ConversationCRUDBaseTest, StateManagementTests, MCPTests
):
"""End to end tests for OpenAI backend."""
api_key = os.environ.get("OPENAI_API_KEY")
@classmethod
def setUpClass(cls):
cls.model = "gpt-5-nano"
cls.base_url_port = "http://127.0.0.1:30010"
cls.cluster = popen_launch_openai_xai_router(
backend="openai",
base_url=cls.base_url_port,
history_backend="memory",
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
class TestXaiBackend(StateManagementTests):
"""End to end tests for XAI backend."""
api_key = os.environ.get("XAI_API_KEY")
@classmethod
def setUpClass(cls):
cls.model = "grok-4-fast"
cls.base_url_port = "http://127.0.0.1:30023"
cls.cluster = popen_launch_openai_xai_router(
backend="xai",
base_url=cls.base_url_port,
history_backend="memory",
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
class TestOracleStore(ResponseCRUDBaseTest, ConversationCRUDBaseTest):
"""End to end tests for Oracle database storage backend."""
api_key = os.environ.get("OPENAI_API_KEY")
@classmethod
def setUpClass(cls):
cls.model = "gpt-5-nano"
cls.base_url_port = "http://127.0.0.1:30040"
cls.cluster = popen_launch_openai_xai_router(
backend="openai",
base_url=cls.base_url_port,
history_backend="oracle",
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
class TestGrpcBackend(StateManagementTests, MCPTests):
"""End to end tests for gRPC backend."""
@classmethod
def setUpClass(cls):
cls.model = "/home/ubuntu/models/meta-llama/Llama-3.1-8B-Instruct"
cls.base_url_port = "http://127.0.0.1:30030"
cls.cluster = popen_launch_workers_and_router(
cls.model,
cls.base_url_port,
timeout=90,
num_workers=1,
tp_size=2,
policy="round_robin",
router_args=["--history-backend", "memory"],
)
cls.base_url = cls.cluster["base_url"]
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.cluster["router"].pid)
for worker in cls.cluster.get("workers", []):
kill_process_tree(worker.pid)
@unittest.skip(
"TODO: transport error, details: [], metadata: MetadataMap { headers: {} }"
)
def test_previous_response_id_chaining(self):
super().test_previous_response_id_chaining()
@unittest.skip("TODO: return 501 Not Implemented")
def test_conversation_with_multiple_turns(self):
super().test_conversation_with_multiple_turns()
@unittest.skip("TODO: decode error message")
def test_mutually_exclusive_parameters(self):
super().test_mutually_exclusive_parameters()
@unittest.skip(
"TODO: Pipeline execution failed: Pipeline stage WorkerSelection failed"
)
def test_mcp_basic_tool_call(self):
super().test_mcp_basic_tool_call()
@unittest.skip("TODO: no event fields")
def test_mcp_basic_tool_call_streaming(self):
return super().test_mcp_basic_tool_call_streaming()
if __name__ == "__main__":
unittest.main()