ci: adding errors to Github summary (#14778)

test/nightly/test_text_models_gsm8k_eval.py

@@ -69,6 +69,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
         for model_setup in self.models:
             with self.subTest(model=model_setup.model_path):
                 other_args = list(model_setup.extra_args)
+                error_message = None
 
                 if model_setup.model_path == "meta-llama/Llama-3.1-70B-Instruct":
                     other_args.extend(["--mem-fraction-static", "0.9"])
@@ -99,8 +100,18 @@ class TestNightlyGsm8KEval(unittest.TestCase):
                     )
                     is_first = False
 
-                    # 0.0 for empty latency
-                    all_results.append((model_setup.model_path, metrics["score"], 0.0))
+                    # 0.0 for empty latency, None for no error
+                    all_results.append(
+                        (model_setup.model_path, metrics["score"], 0.0, error_message)
+                    )
+                except Exception as e:
+                    # Capture error message for the summary table
+                    error_message = str(e)
+                    # Still append result with error info (use None for N/A metrics to match else clause)
+                    all_results.append(
+                        (model_setup.model_path, None, None, error_message)
+                    )
+                    print(f"Error evaluating {model_setup.model_path}: {error_message}")
                 finally:
                     kill_process_tree(process.pid)
 

test/nightly/test_vlms_mmmu_eval.py

@@ -67,6 +67,7 @@ class TestNightlyVLMMmmuEval(unittest.TestCase):
 
         for model in self.models:
             model_path = model.model_path
+            error_message = None
             with self.subTest(model=model_path):
                 process = popen_launch_server(
                     model=model_path,
@@ -98,8 +99,19 @@ class TestNightlyVLMMmmuEval(unittest.TestCase):
                     is_first = False
 
                     all_results.append(
-                        (model_path, metrics["score"], metrics["latency"])
+                        (
+                            model_path,
+                            metrics["score"],
+                            metrics["latency"],
+                            error_message,
+                        )
                     )
+                except Exception as e:
+                    # Capture error message for the summary table
+                    error_message = str(e)
+                    # Still append result with error info (use None for N/A metrics to match else clause)
+                    all_results.append((model_path, None, None, error_message))
+                    print(f"Error evaluating {model_path}: {error_message}")
                 finally:
                     kill_process_tree(process.pid)