diff --git a/python/sglang/test/simple_eval_common.py b/python/sglang/test/simple_eval_common.py index 53243fda9..434c10412 100644 --- a/python/sglang/test/simple_eval_common.py +++ b/python/sglang/test/simple_eval_common.py @@ -148,7 +148,7 @@ class ChatCompletionSampler(SamplerBase): reasoning_effort=self.reasoning_effort, extra_body=self.extra_body, ) - return response.choices[0].message.content + return response.choices[0].message.content or "" # NOTE: BadRequestError is triggered once for MMMU, please uncomment if you are rerunning MMMU except openai.BadRequestError as e: print("Bad Request Error", e) @@ -161,7 +161,9 @@ class ChatCompletionSampler(SamplerBase): ) time.sleep(exception_backoff) trial += 1 - # unknown error shall throw exception + # If all retries are exhausted, return empty string instead of None + print(f"All retry attempts exhausted for request. Returning empty response.") + return "" QUERY_TEMPLATE_MULTICHOICE = """ @@ -261,7 +263,7 @@ def format_multichoice_question(row): def check_equality(sampler: SamplerBase, expr1: str, expr2: str): prompt = EQUALITY_TEMPLATE % {"expression1": expr1, "expression2": expr2} response = sampler([dict(content=prompt, role="user")]) - return response.lower().strip() == "yes" + return (response or "").lower().strip() == "yes" def _compute_stat(values: list, stat: str): diff --git a/python/sglang/test/simple_eval_humaneval.py b/python/sglang/test/simple_eval_humaneval.py index efd03af38..aa77b4d99 100644 --- a/python/sglang/test/simple_eval_humaneval.py +++ b/python/sglang/test/simple_eval_humaneval.py @@ -80,6 +80,7 @@ class HumanEval(Eval): instruction = "Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n" def find_code(completion): + completion = completion or "" pattern = re.compile(r"```python\n(.*?)```", re.DOTALL) matches = pattern.findall(completion) extracted_answer = matches[0] if len(matches) >= 1 else completion diff --git a/python/sglang/test/simple_eval_math.py b/python/sglang/test/simple_eval_math.py index 74c49abe5..37d4b120b 100644 --- a/python/sglang/test/simple_eval_math.py +++ b/python/sglang/test/simple_eval_math.py @@ -54,6 +54,7 @@ class MathEval(Eval): sampler._pack_message(content=QUERY_TEMPLATE.format(**row), role="user") ] response_text = sampler(prompt_messages) + response_text = response_text or "" match = re.search(ANSWER_PATTERN, response_text) extracted_answer = match.group(1) if match else None score = float( diff --git a/python/sglang/test/simple_eval_mmlu.py b/python/sglang/test/simple_eval_mmlu.py index 36a5c7fe3..a68dbb935 100644 --- a/python/sglang/test/simple_eval_mmlu.py +++ b/python/sglang/test/simple_eval_mmlu.py @@ -101,6 +101,7 @@ class MMLUEval(Eval): ) ] response_text = sampler(prompt_messages) + response_text = response_text or "" match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) extracted_answer = match.group(1) if match else None score = 1.0 if extracted_answer == row["Answer"] else 0.0 diff --git a/python/sglang/test/simple_eval_mmmu_vlm.py b/python/sglang/test/simple_eval_mmmu_vlm.py index 2f64df004..f13cfd687 100644 --- a/python/sglang/test/simple_eval_mmmu_vlm.py +++ b/python/sglang/test/simple_eval_mmmu_vlm.py @@ -204,6 +204,7 @@ class MMMUVLMEval(Eval): # Sample response_text = sampler(prompt_messages) + response_text = response_text or "" # Parse and score gold = sample["answer"]