diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 4202f6c2e..a1e96316e 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -287,7 +287,7 @@ jobs: - name: Run E2E tests run: | - bash scripts/killall_sglang.sh "nuk_gpus" + bash scripts/killall_sglang.sh all cd sgl-model-gateway ${{ matrix.env_vars }} ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest ${{ matrix.reruns }} ${{ matrix.parallel_opts }} ${{ matrix.test_dirs }} -s -vv -o log_cli=true --log-cli-level=INFO diff --git a/scripts/killall_sglang.sh b/scripts/killall_sglang.sh index e637aaee9..538b55911 100755 --- a/scripts/killall_sglang.sh +++ b/scripts/killall_sglang.sh @@ -1,11 +1,34 @@ #!/bin/bash +# Usage: +# ./killall_sglang.sh - Kill SGLang processes only (NVIDIA mode) +# ./killall_sglang.sh rocm - Kill SGLang processes only (ROCm mode) +# ./killall_sglang.sh all - Kill all GPU processes (NVIDIA mode) +# ./killall_sglang.sh gpus 0,1,2,3 - Kill all processes on specific GPUs + if [ "$1" = "rocm" ]; then echo "Running in ROCm mode" # Clean SGLang processes pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt|sgl_diffusion::' | xargs -r kill -9 +elif [ "$1" = "gpus" ] && [ -n "$2" ]; then + # Kill all processes on specific GPUs only + echo "Killing all processes on GPUs: $2" + + # Show current GPU status + nvidia-smi + + # Build device file list from GPU IDs (e.g., "0,1,2,3" -> "/dev/nvidia0 /dev/nvidia1 ...") + devices=$(echo "$2" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sed 's|^|/dev/nvidia|' | tr '\n' ' ') + echo "Targeting devices: $devices" + + # Kill all processes using specified GPU devices + [ -n "$devices" ] && lsof $devices 2>/dev/null | awk 'NR>1 {print $2}' | sort -u | xargs -r kill -9 2>/dev/null + + # Show GPU status after clean up + nvidia-smi + else # Show current GPU status nvidia-smi @@ -13,8 +36,8 @@ else # Clean SGLang processes pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt|sgl_diffusion::' | xargs -r kill -9 - # Clean all GPU processes if any argument is provided - if [ $# -gt 0 ]; then + # Clean all GPU processes if "all" argument is provided + if [ "$1" = "all" ]; then # Check if sudo is available if command -v sudo >/dev/null 2>&1; then sudo apt-get update