diff --git a/sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py b/sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py index 4070aff7b..c7634da8a 100644 --- a/sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py +++ b/sgl-router/py_test/e2e_response_api/backends/test_grpc_backend.py @@ -25,11 +25,11 @@ from util import kill_process_tree class TestGrpcBackend(StateManagementTests, MCPTests, StructuredOutputBaseTest): - """End to end tests for gRPC backend (Regular backend with Llama).""" + """End to end tests for gRPC backend (Regular backend with Qwen2.5).""" @classmethod def setUpClass(cls): - cls.model = "/home/ubuntu/models/meta-llama/Llama-3.1-8B-Instruct" + cls.model = "/home/ubuntu/models/Qwen/Qwen2.5-14B-Instruct" cls.base_url_port = "http://127.0.0.1:30030" cls.cluster = popen_launch_workers_and_router( @@ -46,7 +46,7 @@ class TestGrpcBackend(StateManagementTests, MCPTests, StructuredOutputBaseTest): "--history-backend", "memory", "--tool-call-parser", - "llama", + "qwen", ], ) @@ -152,7 +152,7 @@ class TestGrpcHarmonyBackend( cls.base_url_port, timeout=90, num_workers=1, - tp_size=2, + tp_size=4, policy="round_robin", worker_args=[ "--reasoning-parser=gpt-oss", diff --git a/sgl-router/py_test/e2e_response_api/mixins/function_call.py b/sgl-router/py_test/e2e_response_api/mixins/function_call.py index 77c2b09cb..3bb92e687 100644 --- a/sgl-router/py_test/e2e_response_api/mixins/function_call.py +++ b/sgl-router/py_test/e2e_response_api/mixins/function_call.py @@ -86,10 +86,16 @@ class FunctionCallingBaseTest(ResponseAPIBaseTest): }, }, ] + system_prompt = ( + "You are a helpful assistant that can call functions. " + "When a user asks for horoscope information, call the function. " + "IMPORTANT: Don't reply directly to the user, only call the function. " + ) # Create a running input list we will add to over time input_list = [ - {"role": "user", "content": "What is my horoscope? I am an Aquarius."} + {"role": "system", "content": system_prompt}, + {"role": "user", "content": "What is my horoscope? I am an Aquarius."}, ] # 2. Prompt the model with tools defined