Add --thinking-mode to run_eval (#11189)

Signed-off-by: Hao Lu <14827759+hlu1@users.noreply.github.com>

python/sglang/test/run_eval.py

@@ -16,13 +16,29 @@ from sglang.test.simple_eval_common import (
 )
 
 
+def get_thinking_kwargs(args):
+    if args.thinking_mode in THINKING_MODE_CHOICES:
+        thinking_param = (
+            "thinking" if args.thinking_mode == "deepseek-v3" else "enable_thinking"
+        )
+        return {
+            "chat_template_kwargs": {thinking_param: True},
+            "separate_reasoning": True,
+        }
+    return {}
+
+
 def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
+    # Get thinking kwargs based on user's choice
+    thinking_kwargs = get_thinking_kwargs(args)
+
     sampler = ChatCompletionSampler(
         model=args.model,
         max_tokens=getattr(args, "max_tokens", 2048),
         base_url=base_url,
         temperature=getattr(args, "temperature", 0.0),
         reasoning_effort=getattr(args, "reasoning_effort", None),
+        extra_body=thinking_kwargs,
     )
 
     # Run eval
@@ -136,6 +152,8 @@ def run_eval(args):
     return metrics
 
 
+THINKING_MODE_CHOICES = ["deepseek-r1", "deepseek-v3", "qwen3"]
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -166,6 +184,13 @@ if __name__ == "__main__":
     parser.add_argument("--max-tokens", type=int, default=2048)
     parser.add_argument("--temperature", type=float, default=0.0)
     parser.add_argument("--reasoning-effort", type=str)
+    parser.add_argument(
+        "--thinking-mode",
+        default=None,
+        type=str,
+        choices=THINKING_MODE_CHOICES,
+        help="Enable thinking mode in Deepseek R1, V3.1/3.2, or Qwen3",
+    )
     args = parser.parse_args()
 
     run_eval(args)