From c29b98e04393aa73680e6376bfc8774f4081eb35 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Fri, 15 Nov 2024 05:33:43 -0800 Subject: [PATCH] Fix json benchmark (#2043) --- benchmark/json_schema/bench_sglang.py | 12 ++++++++++-- python/sglang/srt/server.py | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/benchmark/json_schema/bench_sglang.py b/benchmark/json_schema/bench_sglang.py index 05c9783eb..72e2f0b3d 100644 --- a/benchmark/json_schema/bench_sglang.py +++ b/benchmark/json_schema/bench_sglang.py @@ -7,6 +7,8 @@ import jsonschema from datasets import load_dataset import sglang as sgl +from sglang.global_config import global_config +from sglang.srt.hf_transformers_utils import get_tokenizer from sglang.test.test_utils import ( add_common_sglang_args_and_parse, select_sglang_backend, @@ -103,7 +105,6 @@ def bench_schema(args): print(e) indexs.append(i) - assert len(indexs) == 0, f"Invalid json outputs: {indexs}" return states, latency @@ -111,11 +112,18 @@ def main(args): states, latency = bench_schema(args) # Compute accuracy + tokenizer = get_tokenizer( + global_config.default_backend.get_server_args()["tokenizer_path"] + ) + output_jsons = [state["json_output"] for state in states] + num_output_tokens = sum(len(tokenizer.encode(x)) for x in output_jsons) print(f"Latency: {latency:.3f}") + print(f"Output throughput: {num_output_tokens / latency:.3f} token/s") + print(f"#output tokens: {num_output_tokens}") # Write results dump_state_text(f"tmp_output_{args.backend}.txt", states) - with open(f"{args.backend}.json", "w") as fout: + with open(f"{args.backend}.jsonl", "w") as fout: for state in states: fout.write(state["json_output"] + "\n") diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py index e4798877a..1ebaf16d9 100644 --- a/python/sglang/srt/server.py +++ b/python/sglang/srt/server.py @@ -139,6 +139,7 @@ async def get_model_info(): """Get the model information.""" result = { "model_path": tokenizer_manager.model_path, + "tokenizer_path": tokenizer_manager.server_args.tokenizer_path, "is_generation": tokenizer_manager.is_generation, } return result