diff --git a/.github/workflows/rerun-ut.yml b/.github/workflows/rerun-ut.yml index 9763a16ff..0e1cf7379 100644 --- a/.github/workflows/rerun-ut.yml +++ b/.github/workflows/rerun-ut.yml @@ -17,10 +17,16 @@ on: required: false type: string default: "" + use_deepep: + description: "Use ci_install_deepep.sh instead of ci_install_dependency.sh" + required: false + type: string + default: "false" env: SGLANG_IS_IN_CI: true SGLANG_CUDA_COREDUMP: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true permissions: actions: write @@ -32,7 +38,8 @@ jobs: timeout-minutes: 120 env: RUNNER_LABELS: ${{ inputs.runner_label }} - IS_BLACKWELL: ${{ inputs.runner_label == '1-gpu-5090' && '1' || '' }} + IS_BLACKWELL: ${{ (inputs.runner_label == '1-gpu-5090' || contains(inputs.runner_label, 'b200')) && '1' || '' }} + SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.runner_label == '8-gpu-h20' && 'mlx5_1,mlx5_2,mlx5_3,mlx5_4' || '' }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -45,7 +52,11 @@ jobs: if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then source /etc/profile.d/sglang-ci.sh fi - bash scripts/ci/cuda/ci_install_dependency.sh + if [[ "${{ inputs.use_deepep }}" == "true" ]]; then + bash scripts/ci/cuda/ci_install_deepep.sh + else + bash scripts/ci/cuda/ci_install_dependency.sh + fi - name: Run test timeout-minutes: 60 diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index eeefee546..b5f9a3847 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -424,6 +424,12 @@ CUDA_SUITE_TO_RUNNER = { "stage-c-test-deepep-8-gpu-h200": "8-gpu-h200", } +DEEPEP_SUITES = { + "stage-c-test-8-gpu-h20", + "stage-c-test-deepep-4-gpu", + "stage-c-test-deepep-8-gpu-h200", +} + def resolve_test_file(file_part): """ @@ -465,7 +471,7 @@ def detect_cuda_suite(file_path_from_test): """ Read a test file and extract the suite from register_cuda_ci(suite="..."). - Returns (suite_name, runner_label, error_message). + Returns (suite_name, runner_label, use_deepep, error_message). """ full_path = f"test/{file_path_from_test}" with open(full_path, "r") as f: @@ -478,6 +484,7 @@ def detect_cuda_suite(file_path_from_test): return ( None, None, + False, ( f"No `register_cuda_ci()` found in `{full_path}`.\n\n" f"This file may not be a registered CUDA CI test." @@ -491,12 +498,14 @@ def detect_cuda_suite(file_path_from_test): return ( suite, None, + False, ( f"Unknown CUDA suite `{suite}` in `{full_path}`.\n\n" f"Known suites: {known}" ), ) - return suite, runner, None + use_deepep = suite in DEEPEP_SUITES + return suite, runner, use_deepep, None def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token): @@ -541,7 +550,7 @@ def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token): return False # Detect suite and runner - suite, runner_label, err = detect_cuda_suite(resolved_path) + suite, runner_label, use_deepep, err = detect_cuda_suite(resolved_path) if err: comment.create_reaction("confused") pr.create_issue_comment(f"❌ {err}") @@ -554,7 +563,7 @@ def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token): print( f"Resolved: file={resolved_path}, selector={test_selector}, " - f"suite={suite}, runner={runner_label}, command='{test_command}'" + f"suite={suite}, runner={runner_label}, deepep={use_deepep}, command='{test_command}'" ) try: @@ -583,12 +592,14 @@ def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token): "test_command": test_command, "runner_label": runner_label, "pr_head_sha": pr_head_sha, + "use_deepep": str(use_deepep).lower(), } else: ref = pr.head.ref inputs = { "test_command": test_command, "runner_label": runner_label, + "use_deepep": str(use_deepep).lower(), } dispatch_time = time.time()