From 0c966779022a52224d4d4883d42accd7230ee1f2 Mon Sep 17 00:00:00 2001 From: alisonshao <54658187+alisonshao@users.noreply.github.com> Date: Mon, 17 Nov 2025 21:25:29 -0800 Subject: [PATCH] CI: fix NFS EBUSY error in PR test workflow (#13460) Co-authored-by: Kangyan-Zhou Co-authored-by: Mick --- .github/workflows/pr-test.yml | 99 ++++++++++++++++--- .../runtime/utils/performance_logger.py | 62 ++++++++++-- 2 files changed, 138 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index fe1ba81a2..43e2691cb 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -21,6 +21,9 @@ concurrency: group: pr-test-${{ github.ref }} cancel-in-progress: true +env: + SGLANG_IS_IN_CI: true + jobs: # =============================================== check changes ==================================================== check-changes: @@ -232,6 +235,11 @@ jobs: cd sgl-kernel pytest tests/ + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + sgl-kernel-mla-test: needs: [check-changes, sgl-kernel-build-wheels] if: needs.check-changes.outputs.sgl_kernel == 'true' @@ -263,6 +271,11 @@ jobs: cd test/srt python3 test_mla_deepseek_v3.py + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + sgl-kernel-benchmark-test: needs: [check-changes, sgl-kernel-build-wheels] if: needs.check-changes.outputs.sgl_kernel == 'true' @@ -307,24 +320,35 @@ jobs: echo "All benchmark tests completed!" + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + # =============================================== multimodal_gen ==================================================== - # multimodal-gen-test: - # needs: [check-changes] - # if: needs.check-changes.outputs.multimodal_gen == 'true' - # runs-on: 1-gpu-runner - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 + multimodal-gen-test: + needs: [check-changes] + if: needs.check-changes.outputs.multimodal_gen == 'true' + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 - # - name: Install dependencies - # run: | - # CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion + - name: Install dependencies + run: | + CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion - # - name: Run diffusion server tests - # timeout-minutes: 60 - # run: | - # cd python - # pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py + - name: Run diffusion server tests + timeout-minutes: 60 + run: | + cd python + pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py + + - name: Cleanup logs directory + if: always() + run: | + # Remove logs directory to prevent NFS lock files + rm -rf python/sglang/logs || true # Adding a single CUDA13 smoke test to verify that the kernel builds and runs # TODO: Add back this test when it can pass on CI @@ -390,6 +414,11 @@ jobs: # temporarily put backend-independent cpu tests here python3 run_suite.py --hw cpu --suite default + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + unit-test-backend-1-gpu: needs: [check-changes, stage-a-test-1, sgl-kernel-build-wheels] if: always() && !failure() && !cancelled() && @@ -424,6 +453,11 @@ jobs: cd test/srt python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15 + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + unit-test-backend-2-gpu: needs: [check-changes, unit-test-backend-1-gpu, sgl-kernel-build-wheels] if: always() && !failure() && !cancelled() && @@ -457,6 +491,11 @@ jobs: cd test/srt python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + unit-test-backend-4-gpu: needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels] if: always() && !failure() && !cancelled() && @@ -618,6 +657,11 @@ jobs: python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + performance-test-1-gpu-part-2: needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1] if: always() && !failure() && !cancelled() && @@ -671,6 +715,11 @@ jobs: cd test/srt python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + performance-test-1-gpu-part-3: needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1] if: always() && !failure() && !cancelled() && @@ -718,6 +767,11 @@ jobs: cd test/srt python3 -m unittest test_bench_serving.TestBenchServing.test_embeddings_api_batch_scaling + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + performance-test-2-gpu: needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels] if: always() && !failure() && !cancelled() && @@ -777,6 +831,11 @@ jobs: cd test/srt python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + accuracy-test-1-gpu: needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1] if: always() && !failure() && !cancelled() && @@ -809,6 +868,11 @@ jobs: cd test/srt python3 test_eval_accuracy_large.py + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + accuracy-test-2-gpu: needs: [check-changes, accuracy-test-1-gpu, sgl-kernel-build-wheels] if: always() && !failure() && !cancelled() && @@ -841,6 +905,11 @@ jobs: cd test/srt python3 test_moe_eval_accuracy_large.py + - name: Cleanup logs directory + if: always() + run: | + rm -rf python/sglang/logs || true + unit-test-deepep-4-gpu: needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels] if: always() && !failure() && !cancelled() && diff --git a/python/sglang/multimodal_gen/runtime/utils/performance_logger.py b/python/sglang/multimodal_gen/runtime/utils/performance_logger.py index 28480fe0f..6bf066d57 100644 --- a/python/sglang/multimodal_gen/runtime/utils/performance_logger.py +++ b/python/sglang/multimodal_gen/runtime/utils/performance_logger.py @@ -13,23 +13,66 @@ from dateutil.tz import UTC LOG_DIR = os.environ.get("SGLANG_PERF_LOG_DIR") if LOG_DIR: LOG_DIR = os.path.abspath(LOG_DIR) -else: +elif LOG_DIR is None: # Not set project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../")) LOG_DIR = os.path.join(project_root, "logs") +# if LOG_DIR is "", it will remain "", disabling file logging. # Configure a specific logger for performance metrics perf_logger = logging.getLogger("performance") perf_logger.setLevel(logging.INFO) perf_logger.propagate = False # Prevent perf logs from going to the main logger -# Ensure the logs directory exists -if not os.path.exists(LOG_DIR): - os.makedirs(LOG_DIR) +_perf_logger_initialized = False -# Set up a file handler for the performance logger -handler = logging.FileHandler(os.path.join(LOG_DIR, "performance.log")) -handler.setFormatter(logging.Formatter("%(message)s")) -perf_logger.addHandler(handler) + +class OnDemandFileHandler(logging.Handler): + """ + A logging handler that opens the file for each log record, writes, and closes it. + This is less performant than FileHandler but avoids long-lived file handles, + which can be problematic on certain filesystems like NFS. + """ + + def __init__(self, filename: str, mode: str = "a", encoding: str | None = None): + super().__init__() + self.baseFilename = os.path.abspath(filename) + self.mode = mode + self.encoding = encoding + self.terminator = "\n" + + def emit(self, record: logging.LogRecord): + """Emit a record.""" + try: + msg = self.format(record) + with open( + self.baseFilename, self.mode, encoding=self.encoding, errors="replace" + ) as f: + f.write(msg + self.terminator) + except Exception: + self.handleError(record) + + +def _initialize_perf_logger(): + """Initialize the performance logger with a file handler.""" + global _perf_logger_initialized + if _perf_logger_initialized or not LOG_DIR: + return + + try: + # Ensure the logs directory exists + if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + + # Set up a file handler for the performance logger + handler = OnDemandFileHandler(os.path.join(LOG_DIR, "performance.log")) + handler.setFormatter(logging.Formatter("%(message)s")) + perf_logger.addHandler(handler) + except (OSError, PermissionError) as e: + perf_logger.warning(f"Failed to initialize performance logger: {e}") + # Disable file logging if initialization fails + globals()["LOG_DIR"] = "" + finally: + _perf_logger_initialized = True def get_git_commit_hash() -> str: @@ -69,6 +112,7 @@ class PerformanceLogger: def log_total_duration(self, tag: str): """Logs the total duration of the operation and all recorded steps.""" + _initialize_perf_logger() total_duration = time.monotonic() - self.start_time log_entry = { "timestamp": datetime.now(UTC).isoformat(), @@ -82,6 +126,7 @@ class PerformanceLogger: def log_stage_metric(self, stage_name: str, duration_ms: float): """Logs a single pipeline stage timing entry.""" + _initialize_perf_logger() log_entry = { "timestamp": datetime.now(UTC).isoformat(), "request_id": self.request_id, @@ -100,6 +145,7 @@ class PerformanceLogger: stages: Either a PipelineLoggingInfo instance or any object exposing a mapping of stage metadata via a `stages` attribute/dict. """ + _initialize_perf_logger() if stages is None: return