Add unified metrics collection framework (v1) (#16064)

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

python/sglang/test/test_utils.py

@@ -1952,3 +1952,165 @@ def intel_amx_benchmark(extra_args=None, min_throughput=None):
         return wrapper
 
     return decorator
+
+
+def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None):
+    """
+    Output test metric to JSONL and stdout for CI performance tracking.
+
+    Schema (v1):
+      - Required: filename, test_case, metric_name, value
+      - Optional fields supported: ts, labels
+        - ts is emitted by default for convenience
+        - labels preferred as dict; if not JSON-serializable, stored as string
+
+    Value types (v1 contract):
+      - Supported: int, float, str
+      - Input may be bool (will be coerced to int: True=1, False=0)
+      - Others: best-effort conversion to float, fallback to str
+
+    Output channels:
+      - JSONL: ${SGLANG_TEST_METRICS_OUTPUT}.${pid}.jsonl (if env var set)
+      - stdout: [METRIC] metric_name=value [labels=...]
+
+    This function never fails tests - all errors are silently caught.
+
+    Args:
+        metric_name: Metric name (e.g., "gsm8k_accuracy", "cache_hit_rate")
+        value: Metric value
+        labels: Optional label dict (e.g., {"backend": "fa3"})
+    """
+    try:
+        # 1. Capture test context
+        filename, test_case = _get_test_context()
+
+        # 2. Convert value to int/float/str
+        # First unwrap numpy/torch scalars
+        if hasattr(value, "item"):
+            value = value.item()
+
+        # Now convert, ensuring no bool in final result
+        if isinstance(value, bool):
+            converted_value = int(value)  # True->1, False->0
+        elif isinstance(value, (int, float, str)):
+            converted_value = value
+        else:
+            try:
+                converted_value = float(value)
+            except (ValueError, TypeError):
+                converted_value = str(value)
+
+        # 3. Build record
+        record = {
+            "filename": filename,
+            "test_case": test_case,
+            "metric_name": metric_name,
+            "value": converted_value,
+            "ts": time.time(),
+        }
+
+        # 4. Handle labels (best-effort JSON serialization)
+        labels_for_output = None
+        if labels:
+            try:
+                json.dumps(labels, ensure_ascii=False)  # Test serializability
+                record["labels"] = labels
+                labels_for_output = labels
+            except (TypeError, ValueError):
+                # If not serializable, stringify
+                stringified = str(labels)
+                record["labels"] = stringified
+                labels_for_output = stringified
+
+        # 5. Write JSONL
+        base_path = os.getenv("SGLANG_TEST_METRICS_OUTPUT")
+        if base_path:
+            try:
+                jsonl_path = f"{base_path}.{os.getpid()}.jsonl"
+                with open(jsonl_path, "a", encoding="utf-8") as f:
+                    f.write(json.dumps(record, ensure_ascii=False) + "\n")
+            except Exception as e:
+                logging.warning(
+                    f"sglang.test.dump_metric: failed to write to {jsonl_path}: {e}"
+                )
+
+        # 6. Output to stdout (use same labels as record)
+        if labels_for_output:
+            if isinstance(labels_for_output, str):
+                labels_str = f" labels='{labels_for_output}'"
+            else:
+                labels_str = (
+                    f" labels={json.dumps(labels_for_output, ensure_ascii=False)}"
+                )
+        else:
+            labels_str = ""
+        print(f"[METRIC] {metric_name}={converted_value}{labels_str}")
+
+    except Exception as e:
+        # Silent failure - never break tests
+        logging.warning(
+            f"sglang.test.dump_metric: failed to dump metric '{metric_name}': {e}",
+            exc_info=True,
+        )
+
+
+def _get_test_context() -> tuple[str, str]:
+    """
+    Get current test's filename and test_case.
+
+    Tries PYTEST_CURRENT_TEST first, falls back to inspect.stack().
+    """
+    # 1. Try parsing PYTEST_CURRENT_TEST
+    pytest_current = os.getenv("PYTEST_CURRENT_TEST")
+    if pytest_current:
+        # Format: "path/to/test.py::TestClass::test_method (call)"
+        parts = pytest_current.split(" ")[0].split("::", 1)
+        if len(parts) == 2:
+            filename = _repo_relative_path(parts[0])
+            test_case = parts[1].replace("::", ".")
+            return filename, test_case
+
+    # 2. Fallback to inspect
+    import inspect
+
+    frame = inspect.currentframe()
+    # Assumes direct callsite: frame -> _get_test_context -> dump_metric -> caller
+    # If dump_metric gets wrapped, may need to scan upward
+    if frame and frame.f_back and frame.f_back.f_back:
+        caller = frame.f_back.f_back
+        filename = _repo_relative_path(caller.f_code.co_filename)
+
+        # Try to get class name from self
+        test_self = caller.f_locals.get("self")
+        if test_self and hasattr(test_self, "__class__"):
+            test_case = f"{test_self.__class__.__name__}.{caller.f_code.co_name}"
+        else:
+            test_case = caller.f_code.co_name
+
+        return filename, test_case
+
+    return "unknown.py", "unknown_test"
+
+
+def _repo_relative_path(filepath: str) -> str:
+    """Convert absolute path to repo-relative, preferring GITHUB_WORKSPACE"""
+    # Path is imported at module top (line 20)
+    try:
+        abs_path = Path(filepath).resolve()
+
+        # Try GITHUB_WORKSPACE first
+        workspace = os.getenv("GITHUB_WORKSPACE")
+        if workspace:
+            try:
+                return str(abs_path.relative_to(Path(workspace).resolve()))
+            except ValueError:
+                pass
+
+        # Fallback to cwd
+        try:
+            return str(abs_path.relative_to(Path.cwd()))
+        except ValueError:
+            return abs_path.name
+
+    except Exception:
+        return Path(filepath).name