895e56097c
Co-authored-by: cy <chenyang08056032@163.com> Co-authored-by: Cherry_ming <136634645@qq.com>
733 lines
28 KiB
Python
733 lines
28 KiB
Python
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import unittest
|
|
|
|
import requests
|
|
from transformers import AutoTokenizer
|
|
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.utils import kill_process_tree
|
|
from sglang.test.ascend.test_ascend_utils import LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH
|
|
from sglang.test.ci.ci_register import register_npu_ci
|
|
from sglang.test.test_utils import (
|
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
DEFAULT_URL_FOR_TEST,
|
|
CustomTestCase,
|
|
popen_launch_server,
|
|
)
|
|
|
|
# Global variables: Manage server process and initialization status
|
|
GLOBAL_SERVER_PROCESS = None
|
|
GLOBAL_SERVER_INITIALIZED = False
|
|
OUTPUT_DIR = "./profiler_dir"
|
|
|
|
register_npu_ci(est_time=1600, suite="nightly-npu-a3-merged", nightly=True)
|
|
|
|
|
|
class TestNpuApi(CustomTestCase):
|
|
"""Testcase: Verify that the basic functions of the API interfaces work properly and the returned parameters are consistent with the configurations.
|
|
|
|
[Test Category] Interface
|
|
[Test Target] /health; /health_generate; /ping; /model_info; /server_info; /get_load; /v1/models; /v1/models/{model:path}; /generate
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
global GLOBAL_SERVER_PROCESS, GLOBAL_SERVER_INITIALIZED
|
|
# Start server only if not initialized
|
|
if not GLOBAL_SERVER_INITIALIZED:
|
|
cls.model = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH
|
|
other_args = [
|
|
"--attention-backend",
|
|
"ascend",
|
|
"--enable-return-hidden-states",
|
|
]
|
|
# Start server and save to global variable
|
|
GLOBAL_SERVER_PROCESS = popen_launch_server(
|
|
cls.model,
|
|
DEFAULT_URL_FOR_TEST,
|
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
other_args=other_args,
|
|
)
|
|
GLOBAL_SERVER_INITIALIZED = True
|
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
# First class does not terminate server
|
|
pass
|
|
|
|
def test_api_health(self):
|
|
response = requests.get(f"{self.base_url}/health")
|
|
self.assertEqual(response.status_code, 200)
|
|
|
|
def test_api_health_generate(self):
|
|
response = requests.get(f"{self.base_url}/health_generate")
|
|
self.assertEqual(response.status_code, 200)
|
|
|
|
def test_api_ping(self):
|
|
response = requests.get(f"{self.base_url}/ping")
|
|
self.assertEqual(response.status_code, 200)
|
|
|
|
def test_api_model_info(self):
|
|
response = requests.get(f"{self.base_url}/model_info")
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(response.json()["model_path"], self.model)
|
|
self.assertEqual(response.json()["tokenizer_path"], self.model)
|
|
self.assertTrue(response.json()["is_generation"])
|
|
self.assertIsNone(response.json()["preferred_sampling_params"])
|
|
self.assertEqual(response.json()["weight_version"], "default")
|
|
self.assertFalse(response.json()["has_image_understanding"])
|
|
self.assertFalse(response.json()["has_audio_understanding"])
|
|
self.assertEqual(response.json()["model_type"], "llama")
|
|
self.assertEqual(response.json()["architectures"][0], "LlamaForCausalLM")
|
|
|
|
def test_api_server_info(self):
|
|
response = requests.get(f"{self.base_url}/server_info")
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(response.json()["model_path"], self.model)
|
|
self.assertEqual(response.json()["tokenizer_path"], self.model)
|
|
|
|
def test_api_get_load(self):
|
|
response = requests.get(f"{self.base_url}/get_load")
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIsNone(response.json()[0]["rid"])
|
|
self.assertIsNone(response.json()[0]["http_worker_ipc"])
|
|
self.assertIsNone(response.json()[0]["dp_rank"])
|
|
self.assertGreaterEqual(response.json()[0]["num_reqs"], 0)
|
|
self.assertGreaterEqual(response.json()[0]["num_waiting_reqs"], 0)
|
|
self.assertGreaterEqual(response.json()[0]["num_tokens"], 0)
|
|
|
|
def test_api_v1_models(self):
|
|
response = requests.get(f"{self.base_url}/v1/models")
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(response.json()["data"][0]["id"], self.model)
|
|
self.assertEqual(response.json()["data"][0]["object"], "model")
|
|
self.assertEqual(response.json()["data"][0]["owned_by"], "sglang")
|
|
self.assertEqual(response.json()["data"][0]["root"], self.model)
|
|
self.assertEqual(response.json()["data"][0]["max_model_len"], 131072)
|
|
|
|
def test_api_v1_models_path(self):
|
|
response = requests.get(f"{self.base_url}/v1/models/{self.model}")
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(response.json()["id"], self.model)
|
|
self.assertEqual(response.json()["object"], "model")
|
|
self.assertEqual(response.json()["owned_by"], "sglang")
|
|
self.assertEqual(response.json()["root"], self.model)
|
|
self.assertEqual(response.json()["max_model_len"], 131072)
|
|
|
|
def test_api_generate_single_text(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/generate",
|
|
json={
|
|
"rid": "req_001",
|
|
"text": "The capital of France is",
|
|
"sampling_params": {
|
|
"temperature": 0,
|
|
"max_new_tokens": 20,
|
|
},
|
|
"return_logprob": True,
|
|
"stream": False,
|
|
"return_hidden_states": True,
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
meta_info_keys = response.json()["meta_info"].keys()
|
|
self.assertEqual("req_001", response.json()["meta_info"]["id"])
|
|
self.assertIn("Paris", response.json()["text"])
|
|
self.assertEqual(20, response.json()["meta_info"]["completion_tokens"])
|
|
self.assertIn("input_token_logprobs", meta_info_keys)
|
|
self.assertIn("output_token_logprobs", meta_info_keys)
|
|
self.assertIn("hidden_states", meta_info_keys)
|
|
|
|
def test_api_generate_batch_texts(self):
|
|
rids = ["req_1", "req_2"]
|
|
texts = [
|
|
"The capital of France is",
|
|
"What is the best time of year to visit Japan for cherry blossoms?",
|
|
]
|
|
response = requests.post(
|
|
f"{self.base_url}/generate",
|
|
json={
|
|
"rid": rids,
|
|
"text": texts,
|
|
"sampling_params": {
|
|
"temperature": 0,
|
|
"max_new_tokens": 20,
|
|
},
|
|
"return_logprob": False,
|
|
"stream": False,
|
|
"return_hidden_states": False,
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual("req_1", response.json()[0]["meta_info"]["id"])
|
|
self.assertIn("Paris", response.json()[0]["text"])
|
|
self.assertEqual("req_2", response.json()[1]["meta_info"]["id"])
|
|
self.assertIn("Japan", response.json()[1]["text"])
|
|
|
|
def test_api_generate_temperature(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/generate",
|
|
json={
|
|
"text": "The capital of France is",
|
|
"sampling_params": {
|
|
"temperature": 5,
|
|
"max_new_tokens": 20,
|
|
},
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
text1 = response.json()["text"]
|
|
response = requests.post(
|
|
f"{self.base_url}/generate",
|
|
json={
|
|
"text": "The capital of France is",
|
|
"sampling_params": {
|
|
"temperature": 5,
|
|
"max_new_tokens": 20,
|
|
},
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
text2 = response.json()["text"]
|
|
self.assertNotEqual(text2, text1)
|
|
|
|
def test_api_generate_input_ids(self):
|
|
text = "The capital of France is"
|
|
tokenizer = AutoTokenizer.from_pretrained(self.model)
|
|
input_ids = tokenizer(text, return_tensors="pt")["input_ids"][0].tolist()
|
|
response = requests.post(
|
|
f"{self.base_url}/generate",
|
|
json={
|
|
"rid": "req_002",
|
|
"input_ids": input_ids,
|
|
"sampling_params": {
|
|
"temperature": 0,
|
|
"max_new_tokens": 10,
|
|
},
|
|
"return_logprob": False,
|
|
"stream": True,
|
|
"return_hidden_states": False,
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
lines = response.text.strip().split("\n")
|
|
self.assertGreaterEqual(len(lines), 10)
|
|
json_data = lines[-3][6:]
|
|
data = json.loads(json_data)
|
|
meta_info_keys = data["meta_info"].keys()
|
|
self.assertEqual("req_002", data["meta_info"]["id"])
|
|
self.assertIn("Paris", data["text"])
|
|
self.assertEqual(10, data["meta_info"]["completion_tokens"])
|
|
self.assertNotIn("input_token_logprobs", meta_info_keys)
|
|
self.assertNotIn("output_token_logprobs", meta_info_keys)
|
|
self.assertNotIn("hidden_states", meta_info_keys)
|
|
|
|
|
|
class TestChatCompletionsInterface(CustomTestCase):
|
|
"""Testcase: The test is to verify whether the functions of each parameter of the v1/chat/completions interface are normal.
|
|
|
|
[Test Category] Interface
|
|
[Test Target] v1/chat/completions
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
# Skip initialization, directly reuse global server
|
|
cls.model = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH
|
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
cls.additional_chat_kwargs = {}
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
# Do not terminate server
|
|
pass
|
|
|
|
def test_model_and_messages(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
data = response.json()
|
|
self.assertEqual(data["model"], self.model)
|
|
self.assertIsNotNone(data["choices"][0]["message"]["reasoning_content"])
|
|
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
data = response.json()
|
|
self.assertEqual(data["model"], "default")
|
|
self.assertIsNotNone(data["choices"][0]["message"]["reasoning_content"])
|
|
|
|
def test_max_completion_tokens(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
"max_completion_tokens": 1,
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
self.assertEqual(response.json()["choices"][0]["finish_reason"], "length")
|
|
|
|
def test_stream(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
"stream": True,
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
has_reasoning = False
|
|
has_content = False
|
|
|
|
for line in response.iter_lines():
|
|
if line:
|
|
line = line.decode("utf-8")
|
|
if line.startswith("data:") and not line.startswith("data: [DONE]"):
|
|
data = json.loads(line[6:])
|
|
if "choices" in data and len(data["choices"]) > 0:
|
|
delta = data["choices"][0].get("delta", {})
|
|
if "reasoning_content" in delta and delta["reasoning_content"]:
|
|
has_reasoning = True
|
|
if "content" in delta and delta["content"]:
|
|
has_content = True
|
|
|
|
self.assertTrue(
|
|
has_reasoning, "Reasoning content not included in stream response"
|
|
)
|
|
self.assertTrue(has_content, "Normal content not included in stream response")
|
|
|
|
def test_temperature(self):
|
|
response1 = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "Please write a five-character quatrain for me.",
|
|
}
|
|
],
|
|
"temperature": 0,
|
|
},
|
|
)
|
|
self.assertEqual(response1.status_code, 200, f"Failed with: {response1.text}")
|
|
content1 = response1.json()["choices"][0]["message"]["content"]
|
|
|
|
response2 = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "Please write a five-character quatrain for me.",
|
|
}
|
|
],
|
|
"temperature": 0,
|
|
},
|
|
)
|
|
self.assertEqual(response2.status_code, 200, f"Failed with: {response2.text}")
|
|
content2 = response2.json()["choices"][0]["message"]["content"]
|
|
self.assertEqual(content1, content2)
|
|
|
|
response3 = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "Please write a five-character quatrain for me.",
|
|
}
|
|
],
|
|
"temperature": 2,
|
|
},
|
|
)
|
|
self.assertEqual(response3.status_code, 200, f"Failed with: {response3.text}")
|
|
content3 = response3.json()["choices"][0]["message"]["content"]
|
|
|
|
response4 = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "Please write a five-character quatrain for me.",
|
|
}
|
|
],
|
|
"temperature": 2,
|
|
},
|
|
)
|
|
self.assertEqual(response4.status_code, 200, f"Failed with: {response4.text}")
|
|
content4 = response4.json()["choices"][0]["message"]["content"]
|
|
self.assertNotEqual(content3, content4)
|
|
|
|
def test_return_hidden_states(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
"return_hidden_states": True,
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
self.assertIn("hidden_states", response.json()["choices"][0])
|
|
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
self.assertNotIn("hidden_states", response.json()["choices"][0])
|
|
|
|
def test_top_k(self):
|
|
response1 = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "Please write a five-character quatrain for me.",
|
|
}
|
|
],
|
|
"top_k": 20,
|
|
},
|
|
)
|
|
self.assertEqual(response1.status_code, 200, f"Failed with: {response1.text}")
|
|
content1 = response1.json()["choices"][0]["message"]["content"]
|
|
|
|
response2 = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "Please write a five-character quatrain for me.",
|
|
}
|
|
],
|
|
"top_k": 20,
|
|
},
|
|
)
|
|
self.assertEqual(response2.status_code, 200, f"Failed with: {response2.text}")
|
|
content2 = response2.json()["choices"][0]["message"]["content"]
|
|
self.assertNotEqual(content1, content2)
|
|
|
|
def test_stop_token_ids(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
"stop_token_ids": [1, 13],
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
self.assertEqual(response.json()["choices"][0]["matched_stop"], 13)
|
|
|
|
def test_rid(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/chat/completions",
|
|
json={
|
|
"model": self.model,
|
|
"messages": [{"role": "user", "content": "Hello"}],
|
|
"rid": "sssss",
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
self.assertEqual(response.json()["id"], "sssss")
|
|
|
|
|
|
class TestEnableThinking(CustomTestCase):
|
|
"""Testcase: The test is to verify whether the functions of each parameter of the v1/completions interface are normal.
|
|
|
|
[Test Category] Interface
|
|
[Test Target] v1/completions
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
# Skip initialization, directly reuse global server
|
|
cls.model = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH
|
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
cls.additional_chat_kwargs = {}
|
|
logging.basicConfig(level=logging.INFO) # Initialize logging
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
# Do not terminate server
|
|
pass
|
|
|
|
def test_model_parameters_model(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"model": self.model, "prompt": "who are you?"},
|
|
)
|
|
logging.info(f"response.json:{response.json()}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
data = response.json()
|
|
self.assertEqual(data["model"], self.model)
|
|
|
|
def test_model_parameters_prompt(self):
|
|
# str format
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?"},
|
|
)
|
|
logging.info(f"response.json:{response.json()}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
|
|
# list[int] format
|
|
list_int = [1, 2, 3, 4]
|
|
response1 = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": list_int},
|
|
)
|
|
logging.info(f"response1.json:{response1.json()}")
|
|
self.assertEqual(response1.status_code, 200, f"Failed with: {response1.text}")
|
|
|
|
# list[str] format
|
|
list_str = ["who is you", "hello world", "ABChello"]
|
|
response2 = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": list_str},
|
|
)
|
|
logging.info(f"response2.json:{response2.json()}")
|
|
self.assertEqual(response2.status_code, 200, f"Failed with: {response2.text}")
|
|
|
|
# list[list[int]] format
|
|
list_list_int = [[14990], [1350, 445, 14990, 1879, 899], [14623, 525, 498, 30]]
|
|
response3 = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": list_list_int},
|
|
)
|
|
logging.info(f"response3.json:{response3.json()}")
|
|
self.assertEqual(response3.status_code, 200, f"Failed with: {response3.text}")
|
|
|
|
def test_model_parameters_max_tokens(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "max_tokens": 1},
|
|
)
|
|
logging.info(f"response.json:{response.json()}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
logging.info(f"finish_reason:{response.json()['choices'][0]['finish_reason']}")
|
|
self.assertEqual(response.json()["choices"][0]["finish_reason"], "length")
|
|
|
|
def test_model_parameters_stream(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "stream": True},
|
|
)
|
|
logging.info(f"response.text:{response.text}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
|
|
has_text = False
|
|
logging.info("\n=== Stream With Reasoning ===")
|
|
for line in response.iter_lines():
|
|
if line:
|
|
line = line.decode("utf-8")
|
|
if line.startswith("data:") and not line.startswith("data: [DONE]"):
|
|
data = json.loads(line[6:])
|
|
if "choices" in data and len(data["choices"]) > 0:
|
|
if "text" in data["choices"][0]:
|
|
has_text = True
|
|
self.assertTrue(has_text, "Text content not included in stream response")
|
|
|
|
def test_model_parameters_temperature(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "temperature": 0},
|
|
)
|
|
logging.info(f"response.json:{response.json()}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
|
|
response1 = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "temperature": 0},
|
|
)
|
|
logging.info(f"response1.json:{response1.json()}")
|
|
self.assertEqual(response1.status_code, 200, f"Failed with: {response1.text}")
|
|
self.assertEqual(
|
|
response.json()["choices"][0]["text"],
|
|
response1.json()["choices"][0]["text"],
|
|
)
|
|
|
|
response2 = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "temperature": 2},
|
|
)
|
|
logging.info(f"response2.json:{response2.json()}")
|
|
self.assertEqual(response2.status_code, 200, f"Failed with: {response2.text}")
|
|
|
|
response3 = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "temperature": 2},
|
|
)
|
|
logging.info(f"response3.json:{response3.json()}")
|
|
self.assertEqual(response3.status_code, 200, f"Failed with: {response3.text}")
|
|
self.assertNotEqual(
|
|
response2.json()["choices"][0]["text"],
|
|
response3.json()["choices"][0]["text"],
|
|
)
|
|
|
|
def test_model_parameters_hidden_states(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "return_hidden_states": True},
|
|
)
|
|
logging.info(f"response.json:{response.json()}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
self.assertIn("hidden_states", response.json()["choices"][0])
|
|
|
|
def test_model_parameters_top_k(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "top_k": 20},
|
|
)
|
|
logging.info(f"response.json:{response.json()}")
|
|
logging.info(f"response.text:{response.text}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
|
|
response1 = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "top_k": 20},
|
|
)
|
|
logging.info(f"response1.json:{response1.json()}")
|
|
logging.info(f"response1.text:{response1.text}")
|
|
self.assertEqual(response1.status_code, 200, f"Failed with: {response1.text}")
|
|
self.assertNotEqual(
|
|
response.json()["choices"][0]["text"],
|
|
response1.json()["choices"][0]["text"],
|
|
)
|
|
|
|
def test_model_parameters_stop_token_ids(self):
|
|
list_ids = [13]
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={
|
|
"prompt": "who are you?",
|
|
"stop_token_ids": list_ids,
|
|
"max_tokens": 1024,
|
|
},
|
|
)
|
|
logging.info(f"response.json:{response.json()}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
self.assertEqual(response.json()["choices"][0]["matched_stop"], 13)
|
|
|
|
def test_model_parameters_rid(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/v1/completions",
|
|
json={"prompt": "who are you?", "rid": "10086"},
|
|
)
|
|
logging.info(f"response.json:{response.json()}")
|
|
self.assertEqual(response.status_code, 200, f"Failed with: {response.text}")
|
|
self.assertEqual(response.json()["id"], "10086")
|
|
|
|
|
|
class TestStartProfile(CustomTestCase):
|
|
"""Testcase: Verify the correctness of /start_profile API with different parameter combinations (start_step/num_steps) on Ascend NPU backend.
|
|
|
|
[Test Category] Interface
|
|
[Test Target] /start_profile
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
# Skip initialization, reuse global server + configure profiler directory
|
|
envs.SGLANG_TORCH_PROFILER_DIR.set(OUTPUT_DIR)
|
|
cls.model = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH
|
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
|
cls.additional_chat_kwargs = {}
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
# Terminate server in last class
|
|
global GLOBAL_SERVER_PROCESS
|
|
if GLOBAL_SERVER_PROCESS:
|
|
kill_process_tree(GLOBAL_SERVER_PROCESS.pid)
|
|
GLOBAL_SERVER_PROCESS = None
|
|
|
|
def setUp(self):
|
|
self._clear_profile_dir()
|
|
|
|
def test_start_profile_1(self):
|
|
self._start_profile(start_step="15", num_steps=5)
|
|
self._post_request()
|
|
self._check_non_empty_profile_dir()
|
|
|
|
def test_start_profile_2(self):
|
|
self._clear_profile_dir()
|
|
self._check_empty_profile_dir()
|
|
self._start_profile()
|
|
self._post_request()
|
|
requests.post(f"{self.base_url}/stop_profile")
|
|
self._check_non_empty_profile_dir()
|
|
|
|
def test_start_profile_3(self):
|
|
self._start_profile(num_steps=5)
|
|
self._post_request()
|
|
self._check_non_empty_profile_dir()
|
|
|
|
def _start_profile(self, **kwargs):
|
|
response = requests.post(
|
|
f"{self.base_url}/start_profile",
|
|
json=kwargs if kwargs else None,
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
return response
|
|
|
|
def _post_request(self):
|
|
response = requests.post(
|
|
f"{self.base_url}/generate",
|
|
json={
|
|
"text": "The capital of France is",
|
|
"sampling_params": {
|
|
"temperature": 0,
|
|
"max_new_tokens": 32,
|
|
},
|
|
},
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
|
|
def _clear_profile_dir(self):
|
|
if os.path.isdir(OUTPUT_DIR):
|
|
shutil.rmtree(OUTPUT_DIR)
|
|
|
|
def _check_non_empty_profile_dir(self):
|
|
self.assertTrue(os.path.isdir(OUTPUT_DIR), "Profiler directory does not exist")
|
|
self.assertNotEqual(
|
|
len(os.listdir(OUTPUT_DIR)), 0, "Profiler directory is empty"
|
|
)
|
|
|
|
def _check_empty_profile_dir(self):
|
|
if os.path.isdir(OUTPUT_DIR):
|
|
self.assertEqual(
|
|
len(os.listdir(OUTPUT_DIR)), 0, "Profiler directory is not empty"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|