CI: fix NFS EBUSY error in PR test workflow (#13460)

Co-authored-by: Kangyan-Zhou <zky314343421@gmail.com>
Co-authored-by: Mick <mickjagger19@icloud.com>

python/sglang/multimodal_gen/runtime/utils/performance_logger.py

@@ -13,23 +13,66 @@ from dateutil.tz import UTC
 LOG_DIR = os.environ.get("SGLANG_PERF_LOG_DIR")
 if LOG_DIR:
     LOG_DIR = os.path.abspath(LOG_DIR)
-else:
+elif LOG_DIR is None:  # Not set
     project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))
     LOG_DIR = os.path.join(project_root, "logs")
+# if LOG_DIR is "", it will remain "", disabling file logging.
 
 # Configure a specific logger for performance metrics
 perf_logger = logging.getLogger("performance")
 perf_logger.setLevel(logging.INFO)
 perf_logger.propagate = False  # Prevent perf logs from going to the main logger
 
-# Ensure the logs directory exists
-if not os.path.exists(LOG_DIR):
-    os.makedirs(LOG_DIR)
+_perf_logger_initialized = False
 
-# Set up a file handler for the performance logger
-handler = logging.FileHandler(os.path.join(LOG_DIR, "performance.log"))
-handler.setFormatter(logging.Formatter("%(message)s"))
-perf_logger.addHandler(handler)
+
+class OnDemandFileHandler(logging.Handler):
+    """
+    A logging handler that opens the file for each log record, writes, and closes it.
+    This is less performant than FileHandler but avoids long-lived file handles,
+    which can be problematic on certain filesystems like NFS.
+    """
+
+    def __init__(self, filename: str, mode: str = "a", encoding: str | None = None):
+        super().__init__()
+        self.baseFilename = os.path.abspath(filename)
+        self.mode = mode
+        self.encoding = encoding
+        self.terminator = "\n"
+
+    def emit(self, record: logging.LogRecord):
+        """Emit a record."""
+        try:
+            msg = self.format(record)
+            with open(
+                self.baseFilename, self.mode, encoding=self.encoding, errors="replace"
+            ) as f:
+                f.write(msg + self.terminator)
+        except Exception:
+            self.handleError(record)
+
+
+def _initialize_perf_logger():
+    """Initialize the performance logger with a file handler."""
+    global _perf_logger_initialized
+    if _perf_logger_initialized or not LOG_DIR:
+        return
+
+    try:
+        # Ensure the logs directory exists
+        if not os.path.exists(LOG_DIR):
+            os.makedirs(LOG_DIR)
+
+        # Set up a file handler for the performance logger
+        handler = OnDemandFileHandler(os.path.join(LOG_DIR, "performance.log"))
+        handler.setFormatter(logging.Formatter("%(message)s"))
+        perf_logger.addHandler(handler)
+    except (OSError, PermissionError) as e:
+        perf_logger.warning(f"Failed to initialize performance logger: {e}")
+        # Disable file logging if initialization fails
+        globals()["LOG_DIR"] = ""
+    finally:
+        _perf_logger_initialized = True
 
 
 def get_git_commit_hash() -> str:
@@ -69,6 +112,7 @@ class PerformanceLogger:
 
     def log_total_duration(self, tag: str):
         """Logs the total duration of the operation and all recorded steps."""
+        _initialize_perf_logger()
         total_duration = time.monotonic() - self.start_time
         log_entry = {
             "timestamp": datetime.now(UTC).isoformat(),
@@ -82,6 +126,7 @@ class PerformanceLogger:
 
     def log_stage_metric(self, stage_name: str, duration_ms: float):
         """Logs a single pipeline stage timing entry."""
+        _initialize_perf_logger()
         log_entry = {
             "timestamp": datetime.now(UTC).isoformat(),
             "request_id": self.request_id,
@@ -100,6 +145,7 @@ class PerformanceLogger:
             stages: Either a PipelineLoggingInfo instance or any object exposing
                 a mapping of stage metadata via a `stages` attribute/dict.
         """
+        _initialize_perf_logger()
         if stages is None:
             return