From 1d811094f8580aa1f76ed2e8959b8e0aa240408e Mon Sep 17 00:00:00 2001
From: elvischenv <219235043+elvischenv@users.noreply.github.com>
Date: Tue, 13 Jan 2026 23:34:29 +0800
Subject: [PATCH] [Misc] Auto download question file for benchmark/mtbench
 (#17019)

---
 benchmark/mtbench/bench_other.py        | 9 ++++++++-
 benchmark/mtbench/bench_sglang.py       | 9 ++++++++-
 benchmark/mtbench/bench_sglang_eagle.py | 9 ++++++++-
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/benchmark/mtbench/bench_other.py b/benchmark/mtbench/bench_other.py
index 5e579e9a6..f50ed6fc9 100644
--- a/benchmark/mtbench/bench_other.py
+++ b/benchmark/mtbench/bench_other.py
@@ -9,6 +9,7 @@ from fastchat.model import get_conversation_template
 from tqdm import tqdm
 
 from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate
+from sglang.utils import download_and_cache_file
 
 
 def load_questions(filename):
@@ -37,7 +38,13 @@ def write_answers(filename, model_id, questions, answers):
 
 
 def main(args):
-    questions = load_questions(args.question_file)
+    # Download question file if not exist
+    question_file = args.question_file
+    url = "https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/llm_judge/data/mt_bench/question.jsonl"
+    if not os.path.isfile(question_file):
+        question_file = download_and_cache_file(url)
+
+    questions = load_questions(question_file)
     questions = (questions * 10)[: args.num_questions]
     max_tokens = 256
     model_id = "llama-2-chat"
diff --git a/benchmark/mtbench/bench_sglang.py b/benchmark/mtbench/bench_sglang.py
index 0d0545b3a..04b3fbcd9 100644
--- a/benchmark/mtbench/bench_sglang.py
+++ b/benchmark/mtbench/bench_sglang.py
@@ -9,6 +9,7 @@ from sglang.test.test_utils import (
     add_common_sglang_args_and_parse,
     select_sglang_backend,
 )
+from sglang.utils import download_and_cache_file
 
 
 def load_questions(filename):
@@ -46,8 +47,14 @@ def answer_mt_bench(s, question_1, question_2):
 
 
 def main(args):
+    # Download question file if not exist
+    question_file = args.question_file
+    url = "https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/llm_judge/data/mt_bench/question.jsonl"
+    if not os.path.isfile(question_file):
+        question_file = download_and_cache_file(url)
+
     # Construct prompts
-    questions = load_questions(args.question_file)[: args.num_questions]
+    questions = load_questions(question_file)[: args.num_questions]
     arguments = [
         {"question_1": q["turns"][0], "question_2": q["turns"][1]} for q in questions
     ]
diff --git a/benchmark/mtbench/bench_sglang_eagle.py b/benchmark/mtbench/bench_sglang_eagle.py
index 3eb6036c7..62b99bbb2 100644
--- a/benchmark/mtbench/bench_sglang_eagle.py
+++ b/benchmark/mtbench/bench_sglang_eagle.py
@@ -18,6 +18,7 @@ from sglang.test.test_utils import (
     add_common_sglang_args_and_parse,
     select_sglang_backend,
 )
+from sglang.utils import download_and_cache_file
 
 
 def load_questions(filename):
@@ -57,8 +58,14 @@ def answer_mt_bench(s, question_1, question_2):
 
 
 def main(args):
+    # Download question file if not exist
+    question_file = args.question_file
+    url = "https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/llm_judge/data/mt_bench/question.jsonl"
+    if not os.path.isfile(question_file):
+        question_file = download_and_cache_file(url)
+
     # Construct prompts
-    questions = load_questions(args.question_file)[: args.num_questions]
+    questions = load_questions(question_file)[: args.num_questions]
     arguments = [
         {"question_1": q["turns"][0], "question_2": q["turns"][1]} for q in questions
     ]