chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 14 Packages Projects Releases Wiki Activity

CI: fix NFS EBUSY error in PR test workflow (#13460)

Browse Source 
Co-authored-by: Kangyan-Zhou <zky314343421@gmail.com>
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
alisonshao
2025-11-18 13:25:29 +08:00
committed by GitHub
co-authored by Kangyan-Zhou Mick
parent f33860777c
commit 0c96677902
2 changed files with 138 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-test.yml
+84 -15
View File
@@ -21,6 +21,9 @@ concurrency:
group: pr-test-${{ github.ref }}
cancel-in-progress: true
env:
SGLANG_IS_IN_CI: true
jobs:
# =============================================== check changes ====================================================
check-changes:
@@ -232,6 +235,11 @@ jobs:
cd sgl-kernel
pytest tests/
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
sgl-kernel-mla-test:
needs: [check-changes, sgl-kernel-build-wheels]
if: needs.check-changes.outputs.sgl_kernel == 'true'
@@ -263,6 +271,11 @@ jobs:
cd test/srt
python3 test_mla_deepseek_v3.py
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
sgl-kernel-benchmark-test:
needs: [check-changes, sgl-kernel-build-wheels]
if: needs.check-changes.outputs.sgl_kernel == 'true'
@@ -307,24 +320,35 @@ jobs:
echo "All benchmark tests completed!"
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
# =============================================== multimodal_gen ====================================================
# multimodal-gen-test:
# needs: [check-changes]
# if: needs.check-changes.outputs.multimodal_gen == 'true'
# runs-on: 1-gpu-runner
# steps:
# - name: Checkout code
# uses: actions/checkout@v4
multimodal-gen-test:
needs: [check-changes]
if: needs.check-changes.outputs.multimodal_gen == 'true'
runs-on: 1-gpu-runner
steps:
- name: Checkout code
uses: actions/checkout@v4
# - name: Install dependencies
# run: |
# CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion
- name: Install dependencies
run: |
CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh diffusion
# - name: Run diffusion server tests
# timeout-minutes: 60
# run: |
# cd python
# pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
- name: Run diffusion server tests
timeout-minutes: 60
run: |
cd python
pytest -s -v --log-cli-level=INFO sglang/multimodal_gen/test/server/test_server_performance.py
- name: Cleanup logs directory
if: always()
run: |
# Remove logs directory to prevent NFS lock files
rm -rf python/sglang/logs || true
# Adding a single CUDA13 smoke test to verify that the kernel builds and runs
# TODO: Add back this test when it can pass on CI
@@ -390,6 +414,11 @@ jobs:
# temporarily put backend-independent cpu tests here
python3 run_suite.py --hw cpu --suite default
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
unit-test-backend-1-gpu:
needs: [check-changes, stage-a-test-1, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() &&
@@ -424,6 +453,11 @@ jobs:
cd test/srt
python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
unit-test-backend-2-gpu:
needs: [check-changes, unit-test-backend-1-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() &&
@@ -457,6 +491,11 @@ jobs:
cd test/srt
python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
unit-test-backend-4-gpu:
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() &&
@@ -618,6 +657,11 @@ jobs:
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency
python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
performance-test-1-gpu-part-2:
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
if: always() && !failure() && !cancelled() &&
@@ -671,6 +715,11 @@ jobs:
cd test/srt
python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
performance-test-1-gpu-part-3:
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
if: always() && !failure() && !cancelled() &&
@@ -718,6 +767,11 @@ jobs:
cd test/srt
python3 -m unittest test_bench_serving.TestBenchServing.test_embeddings_api_batch_scaling
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
performance-test-2-gpu:
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() &&
@@ -777,6 +831,11 @@ jobs:
cd test/srt
python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
accuracy-test-1-gpu:
needs: [check-changes, sgl-kernel-build-wheels, stage-a-test-1]
if: always() && !failure() && !cancelled() &&
@@ -809,6 +868,11 @@ jobs:
cd test/srt
python3 test_eval_accuracy_large.py
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
accuracy-test-2-gpu:
needs: [check-changes, accuracy-test-1-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() &&
@@ -841,6 +905,11 @@ jobs:
cd test/srt
python3 test_moe_eval_accuracy_large.py
- name: Cleanup logs directory
if: always()
run: |
rm -rf python/sglang/logs || true
unit-test-deepep-4-gpu:
needs: [check-changes, unit-test-backend-2-gpu, sgl-kernel-build-wheels]
if: always() && !failure() && !cancelled() &&
python/sglang/multimodal_gen/runtime/utils/performance_logger.py
+54 -8
View File
@@ -13,23 +13,66 @@ from dateutil.tz import UTC
LOG_DIR = os.environ.get("SGLANG_PERF_LOG_DIR")
if LOG_DIR:
LOG_DIR = os.path.abspath(LOG_DIR)
else:
elif LOG_DIR is None: # Not set
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))
LOG_DIR = os.path.join(project_root, "logs")
# if LOG_DIR is "", it will remain "", disabling file logging.
# Configure a specific logger for performance metrics
perf_logger = logging.getLogger("performance")
perf_logger.setLevel(logging.INFO)
perf_logger.propagate = False # Prevent perf logs from going to the main logger
# Ensure the logs directory exists
if not os.path.exists(LOG_DIR):
os.makedirs(LOG_DIR)
_perf_logger_initialized = False
# Set up a file handler for the performance logger
handler = logging.FileHandler(os.path.join(LOG_DIR, "performance.log"))
handler.setFormatter(logging.Formatter("%(message)s"))
perf_logger.addHandler(handler)
class OnDemandFileHandler(logging.Handler):
"""
A logging handler that opens the file for each log record, writes, and closes it.
This is less performant than FileHandler but avoids long-lived file handles,
which can be problematic on certain filesystems like NFS.
"""
def __init__(self, filename: str, mode: str = "a", encoding: str | None = None):
super().__init__()
self.baseFilename = os.path.abspath(filename)
self.mode = mode
self.encoding = encoding
self.terminator = "\n"
def emit(self, record: logging.LogRecord):
"""Emit a record."""
try:
msg = self.format(record)
with open(
self.baseFilename, self.mode, encoding=self.encoding, errors="replace"
) as f:
f.write(msg + self.terminator)
except Exception:
self.handleError(record)
def _initialize_perf_logger():
"""Initialize the performance logger with a file handler."""
global _perf_logger_initialized
if _perf_logger_initialized or not LOG_DIR:
return
try:
# Ensure the logs directory exists
if not os.path.exists(LOG_DIR):
os.makedirs(LOG_DIR)
# Set up a file handler for the performance logger
handler = OnDemandFileHandler(os.path.join(LOG_DIR, "performance.log"))
handler.setFormatter(logging.Formatter("%(message)s"))
perf_logger.addHandler(handler)
except (OSError, PermissionError) as e:
perf_logger.warning(f"Failed to initialize performance logger: {e}")
# Disable file logging if initialization fails
globals()["LOG_DIR"] = ""
finally:
_perf_logger_initialized = True
def get_git_commit_hash() -> str:
@@ -69,6 +112,7 @@ class PerformanceLogger:
def log_total_duration(self, tag: str):
"""Logs the total duration of the operation and all recorded steps."""
_initialize_perf_logger()
total_duration = time.monotonic() - self.start_time
log_entry = {
"timestamp": datetime.now(UTC).isoformat(),
@@ -82,6 +126,7 @@ class PerformanceLogger:
def log_stage_metric(self, stage_name: str, duration_ms: float):
"""Logs a single pipeline stage timing entry."""
_initialize_perf_logger()
log_entry = {
"timestamp": datetime.now(UTC).isoformat(),
"request_id": self.request_id,
@@ -100,6 +145,7 @@ class PerformanceLogger:
stages: Either a PipelineLoggingInfo instance or any object exposing
a mapping of stage metadata via a `stages` attribute/dict.
"""
_initialize_perf_logger()
if stages is None:
return