diff --git a/.github/workflows/rerun-ut.yml b/.github/workflows/rerun-ut.yml new file mode 100644 index 000000000..9763a16ff --- /dev/null +++ b/.github/workflows/rerun-ut.yml @@ -0,0 +1,60 @@ +name: Rerun UT +run-name: ${{ inputs.pr_head_sha && format('[rerun-ut] {0}', inputs.pr_head_sha) || '[rerun-ut]' }} + +on: + workflow_dispatch: + inputs: + test_command: + description: "Test command to run (e.g. 'registered/core/test_srt_endpoint.py TestSRTEndpoint.test_simple_decode')" + required: true + type: string + runner_label: + description: "Runner label (e.g. '1-gpu-runner', '1-gpu-5090', '4-gpu-h100')" + required: true + type: string + pr_head_sha: + description: "PR head SHA to checkout (for /rerun-ut on fork PRs)" + required: false + type: string + default: "" + +env: + SGLANG_IS_IN_CI: true + SGLANG_CUDA_COREDUMP: "1" + +permissions: + actions: write + contents: read + +jobs: + rerun-ut-cuda: + runs-on: ${{ inputs.runner_label }} + timeout-minutes: 120 + env: + RUNNER_LABELS: ${{ inputs.runner_label }} + IS_BLACKWELL: ${{ inputs.runner_label == '1-gpu-5090' && '1' || '' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || github.sha }} + + - name: Install dependencies + timeout-minutes: 20 + run: | + if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then + source /etc/profile.d/sglang-ci.sh + fi + bash scripts/ci/cuda/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 60 + run: | + if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then + source /etc/profile.d/sglang-ci.sh + fi + cd test/ + python3 ${{ inputs.test_command }} + + - uses: ./.github/actions/upload-cuda-coredumps + if: always() diff --git a/.github/workflows/slash-command-handler.yml b/.github/workflows/slash-command-handler.yml index 012208f9f..9411e0798 100644 --- a/.github/workflows/slash-command-handler.yml +++ b/.github/workflows/slash-command-handler.yml @@ -19,7 +19,8 @@ jobs: (contains(github.event.comment.body, '/tag-run-ci-label') || contains(github.event.comment.body, '/rerun-failed-ci') || contains(github.event.comment.body, '/tag-and-rerun-ci') || - contains(github.event.comment.body, '/rerun-stage')) + contains(github.event.comment.body, '/rerun-stage') || + contains(github.event.comment.body, '/rerun-ut')) runs-on: ubuntu-latest steps: diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index 19e3babd9..eeefee546 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -1,5 +1,7 @@ +import glob import json import os +import re import sys import time from datetime import datetime, timezone @@ -407,6 +409,243 @@ def handle_rerun_stage( return False +CUDA_SUITE_TO_RUNNER = { + "stage-a-test-1": "1-gpu-runner", + "stage-a-cpu-only": "ubuntu-latest", + "stage-b-test-small-1-gpu": "1-gpu-5090", + "stage-b-test-large-1-gpu": "1-gpu-runner", + "stage-b-test-large-2-gpu": "2-gpu-runner", + "stage-b-test-4-gpu-b200": "4-gpu-b200", + "stage-c-test-4-gpu-h100": "4-gpu-h100", + "stage-c-test-8-gpu-h200": "8-gpu-h200", + "stage-c-test-8-gpu-h20": "8-gpu-h20", + "stage-c-test-4-gpu-b200": "4-gpu-b200", + "stage-c-test-deepep-4-gpu": "4-gpu-h100", + "stage-c-test-deepep-8-gpu-h200": "8-gpu-h200", +} + + +def resolve_test_file(file_part): + """ + Resolve a user-provided file path to a path relative to test/. + + Supports: + - Full path: test/registered/core/test_srt_endpoint.py + - Relative to test/: registered/core/test_srt_endpoint.py + - Bare filename: test_srt_endpoint.py (glob-matched, must be unique) + + Returns (resolved_path, error_message). On success error_message is None. + """ + if file_part.startswith("test/"): + file_part = file_part[len("test/") :] + + if "/" not in file_part: + matches = glob.glob(f"test/registered/**/{file_part}", recursive=True) + if len(matches) == 0: + return ( + None, + f"No test file found matching `{file_part}` under `test/registered/`.", + ) + if len(matches) > 1: + match_list = "\n".join(f"- `{m}`" for m in sorted(matches)) + return None, ( + f"Ambiguous filename `{file_part}` — matched {len(matches)} files:\n\n" + f"{match_list}\n\n" + f"Please provide the full path, e.g. `/rerun-ut {matches[0]}`" + ) + return matches[0][len("test/") :], None + + full_path = f"test/{file_part}" + if not os.path.isfile(full_path): + return None, f"File not found: `{full_path}`" + return file_part, None + + +def detect_cuda_suite(file_path_from_test): + """ + Read a test file and extract the suite from register_cuda_ci(suite="..."). + + Returns (suite_name, runner_label, error_message). + """ + full_path = f"test/{file_path_from_test}" + with open(full_path, "r") as f: + content = f.read() + + match = re.search( + r'register_cuda_ci\([^)]*suite\s*=\s*["\']([^"\']+)["\']', content + ) + if not match: + return ( + None, + None, + ( + f"No `register_cuda_ci()` found in `{full_path}`.\n\n" + f"This file may not be a registered CUDA CI test." + ), + ) + + suite = match.group(1) + runner = CUDA_SUITE_TO_RUNNER.get(suite) + if not runner: + known = ", ".join(f"`{s}`" for s in sorted(CUDA_SUITE_TO_RUNNER)) + return ( + suite, + None, + ( + f"Unknown CUDA suite `{suite}` in `{full_path}`.\n\n" + f"Known suites: {known}" + ), + ) + return suite, runner, None + + +def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token): + """ + Handles the /rerun-ut :: command. + Dispatches a lightweight workflow to run a single test on the correct CUDA runner. + """ + if not ( + user_perms.get("can_rerun_ut", False) + or user_perms.get("can_rerun_stage", False) + ): + print("Permission denied: neither can_rerun_ut nor can_rerun_stage is true.") + return False + + if not test_spec: + comment.create_reaction("confused") + pr.create_issue_comment( + "❌ Please specify a test: `/rerun-ut ::`\n\n" + "Examples:\n" + "- `/rerun-ut test/registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`\n" + "- `/rerun-ut registered/core/test_srt_endpoint.py::TestSRTEndpoint`\n" + "- `/rerun-ut test_srt_endpoint.py`" + ) + return False + + # Parse spec: split on :: to get file path and optional test selector + if "::" in test_spec: + file_part, test_selector = test_spec.split("::", 1) + else: + file_part = test_spec + test_selector = None + + file_part = file_part.strip() + if test_selector: + test_selector = test_selector.strip() + + # Resolve file path + resolved_path, err = resolve_test_file(file_part) + if err: + comment.create_reaction("confused") + pr.create_issue_comment(f"❌ {err}") + return False + + # Detect suite and runner + suite, runner_label, err = detect_cuda_suite(resolved_path) + if err: + comment.create_reaction("confused") + pr.create_issue_comment(f"❌ {err}") + return False + + # Build test_command: file path (+ optional test selector as unittest arg) + test_command = resolved_path + if test_selector: + test_command = f"{resolved_path} {test_selector}" + + print( + f"Resolved: file={resolved_path}, selector={test_selector}, " + f"suite={suite}, runner={runner_label}, command='{test_command}'" + ) + + try: + workflow_name = "Rerun UT" + workflows = gh_repo.get_workflows() + target_workflow = None + for wf in workflows: + if wf.name == workflow_name: + target_workflow = wf + break + + if not target_workflow: + print(f"Error: {workflow_name} workflow not found") + return False + + is_fork = ( + pr.head.repo is None or pr.head.repo.owner.login != gh_repo.owner.login + ) + print(f"PR is from fork: {is_fork}") + + pr_head_sha = None + if is_fork: + ref = "main" + pr_head_sha = pr.head.sha + inputs = { + "test_command": test_command, + "runner_label": runner_label, + "pr_head_sha": pr_head_sha, + } + else: + ref = pr.head.ref + inputs = { + "test_command": test_command, + "runner_label": runner_label, + } + + dispatch_time = time.time() + + dispatch_url = f"https://api.github.com/repos/{gh_repo.full_name}/actions/workflows/{target_workflow.id}/dispatches" + dispatch_resp = requests.post( + dispatch_url, + json={"ref": ref, "inputs": inputs}, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + }, + ) + success = dispatch_resp.status_code in (200, 204) + if not success: + print(f"Dispatch failed: {dispatch_resp.status_code} {dispatch_resp.text}") + + if success: + print(f"Successfully triggered rerun-ut: {test_command}") + comment.create_reaction("+1") + pr.create_issue_comment( + f"✅ Triggered `/rerun-ut` on `{runner_label}` runner:\n" + f"```\ncd test/ && python3 {test_command}\n```" + ) + + run_url = find_workflow_run_url( + gh_repo, + target_workflow.id, + ref, + "rerun-ut", + token, + dispatch_time, + pr_head_sha=pr_head_sha, + max_wait=30, + ) + if run_url: + pr.create_issue_comment(f"🔗 [View workflow run]({run_url})") + else: + pr.create_issue_comment( + f"⚠️ Could not retrieve workflow run URL. " + f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress." + ) + return True + else: + print("Failed to trigger workflow_dispatch") + return False + + except Exception as e: + print(f"Error triggering rerun-ut: {e}") + comment.create_reaction("confused") + pr.create_issue_comment( + f"❌ Failed to trigger rerun-ut: {str(e)}\n\n" + f"Please check the logs or contact maintainers." + ) + return False + + def main(): # 1. Load Environment Variables token = get_env_var("GITHUB_TOKEN") @@ -427,7 +666,7 @@ def main(): pr = repo.get_pull(pr_number) comment = repo.get_issue(pr_number).get_comment(comment_id) - # PR authors can always rerun failed CI on their own PRs, + # PR authors can always rerun failed CI and rerun individual UTs on their own PRs, # even if they are not listed in CI_PERMISSIONS.json. # Note: /tag-run-ci-label and /rerun-stage still require CI_PERMISSIONS.json. if pr.user.login == user_login: @@ -442,6 +681,7 @@ def main(): f"User {user_login} is the PR author and has existing CI permissions." ) user_perms["can_rerun_failed_ci"] = True + user_perms["can_rerun_ut"] = True if not user_perms: print(f"User {user_login} does not have any configured permissions. Exiting.") @@ -486,6 +726,11 @@ def main(): stage_name = parts[1].strip() if len(parts) > 1 else None handle_rerun_stage(repo, pr, comment, user_perms, stage_name, token) + elif first_line.startswith("/rerun-ut"): + parts = first_line.split(maxsplit=1) + test_spec = parts[1].strip() if len(parts) > 1 else None + handle_rerun_ut(repo, pr, comment, user_perms, test_spec, token) + else: print(f"Unknown or ignored command: {first_line}")