diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 41e371efe..667a1dedd 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -1786,17 +1786,19 @@ def check_evaluation_test_results( model_count=None, ): """ - results: list of tuple of (model_path, accuracy, latency) + results: list of tuple of (model_path, accuracy, latency) or (model_path, accuracy, latency, error) """ failed_models = [] if model_latency_thresholds is not None: - summary = " | model | status | score | score_threshold | latency | latency_threshold | \n" - summary += "| ----- | ------ | ----- | --------------- | ------- | ----------------- | \n" + summary = " | model | status | score | score_threshold | latency | latency_threshold | error | \n" + summary += "| ----- | ------ | ----- | --------------- | ------- | ----------------- | ----- | \n" else: - summary = " | model | status | score | score_threshold | \n" - summary += "| ----- | ------ | ----- | --------------- | \n" + summary = " | model | status | score | score_threshold | error | \n" + summary += "| ----- | ------ | ----- | --------------- | ----- | \n" - results_dict = {res[0]: (res[1], res[2]) for res in results} + results_dict = { + res[0]: (res[1], res[2], res[3] if len(res) == 4 else None) for res in results + } for model, accuracy_threshold in sorted(model_accuracy_thresholds.items()): latency_threshold = ( @@ -1805,8 +1807,15 @@ def check_evaluation_test_results( else 1e9 ) - if model in results_dict: - accuracy, latency = results_dict[model] + # check for error here + error = ( + results_dict.get(model, (None, None, None))[2] + if model in results_dict + else None + ) + + if model in results_dict and error is None: + accuracy, latency, _ = results_dict[model] is_success = accuracy >= accuracy_threshold and latency <= latency_threshold status_emoji = "✅" if is_success else "❌" @@ -1823,18 +1832,17 @@ def check_evaluation_test_results( ) if model_latency_thresholds is not None: - line = f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold} | {latency} | {latency_threshold}\n" + line = f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold} | {latency} | {latency_threshold} | - |\n" else: - line = ( - f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold}\n" - ) + line = f"| {model} | {status_emoji} | {accuracy} | {accuracy_threshold} | - |\n" else: status_emoji = "❌" + error_display = error if error else "Model not evaluated" failed_models.append(f"Model failed to launch or be evaluated: {model}") if model_latency_thresholds is not None: - line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold} | N/A | {latency_threshold}\n" + line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold} | N/A | {latency_threshold} | {error_display} |\n" else: - line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold}\n" + line = f"| {model} | {status_emoji} | N/A | {accuracy_threshold} | {error_display} |\n" summary += line diff --git a/test/nightly/test_text_models_gsm8k_eval.py b/test/nightly/test_text_models_gsm8k_eval.py index 8cd62e604..c0bf11cea 100644 --- a/test/nightly/test_text_models_gsm8k_eval.py +++ b/test/nightly/test_text_models_gsm8k_eval.py @@ -69,6 +69,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): for model_setup in self.models: with self.subTest(model=model_setup.model_path): other_args = list(model_setup.extra_args) + error_message = None if model_setup.model_path == "meta-llama/Llama-3.1-70B-Instruct": other_args.extend(["--mem-fraction-static", "0.9"]) @@ -99,8 +100,18 @@ class TestNightlyGsm8KEval(unittest.TestCase): ) is_first = False - # 0.0 for empty latency - all_results.append((model_setup.model_path, metrics["score"], 0.0)) + # 0.0 for empty latency, None for no error + all_results.append( + (model_setup.model_path, metrics["score"], 0.0, error_message) + ) + except Exception as e: + # Capture error message for the summary table + error_message = str(e) + # Still append result with error info (use None for N/A metrics to match else clause) + all_results.append( + (model_setup.model_path, None, None, error_message) + ) + print(f"Error evaluating {model_setup.model_path}: {error_message}") finally: kill_process_tree(process.pid) diff --git a/test/nightly/test_vlms_mmmu_eval.py b/test/nightly/test_vlms_mmmu_eval.py index 4b9a19eb8..7f9b5aae4 100644 --- a/test/nightly/test_vlms_mmmu_eval.py +++ b/test/nightly/test_vlms_mmmu_eval.py @@ -67,6 +67,7 @@ class TestNightlyVLMMmmuEval(unittest.TestCase): for model in self.models: model_path = model.model_path + error_message = None with self.subTest(model=model_path): process = popen_launch_server( model=model_path, @@ -98,8 +99,19 @@ class TestNightlyVLMMmmuEval(unittest.TestCase): is_first = False all_results.append( - (model_path, metrics["score"], metrics["latency"]) + ( + model_path, + metrics["score"], + metrics["latency"], + error_message, + ) ) + except Exception as e: + # Capture error message for the summary table + error_message = str(e) + # Still append result with error info (use None for N/A metrics to match else clause) + all_results.append((model_path, None, None, error_message)) + print(f"Error evaluating {model_path}: {error_message}") finally: kill_process_tree(process.pid)