[CI] support /rerun-ut command in slash handler (#19800)
This commit is contained in:
Liangsheng Yin
60
.github/workflows/rerun-ut.yml
vendored
Normal file
60
.github/workflows/rerun-ut.yml
vendored
Normal file
@@ -0,0 +1,60 @@
|
||||
name: Rerun UT
|
||||
run-name: ${{ inputs.pr_head_sha && format('[rerun-ut] {0}', inputs.pr_head_sha) || '[rerun-ut]' }}
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_command:
|
||||
description: "Test command to run (e.g. 'registered/core/test_srt_endpoint.py TestSRTEndpoint.test_simple_decode')"
|
||||
required: true
|
||||
type: string
|
||||
runner_label:
|
||||
description: "Runner label (e.g. '1-gpu-runner', '1-gpu-5090', '4-gpu-h100')"
|
||||
required: true
|
||||
type: string
|
||||
pr_head_sha:
|
||||
description: "PR head SHA to checkout (for /rerun-ut on fork PRs)"
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
rerun-ut-cuda:
|
||||
runs-on: ${{ inputs.runner_label }}
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
RUNNER_LABELS: ${{ inputs.runner_label }}
|
||||
IS_BLACKWELL: ${{ inputs.runner_label == '1-gpu-5090' && '1' || '' }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.pr_head_sha || github.sha }}
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then
|
||||
source /etc/profile.d/sglang-ci.sh
|
||||
fi
|
||||
bash scripts/ci/cuda/ci_install_dependency.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then
|
||||
source /etc/profile.d/sglang-ci.sh
|
||||
fi
|
||||
cd test/
|
||||
python3 ${{ inputs.test_command }}
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: always()
|
||||
3
.github/workflows/slash-command-handler.yml
vendored
3
.github/workflows/slash-command-handler.yml
vendored
@@ -19,7 +19,8 @@ jobs:
|
||||
(contains(github.event.comment.body, '/tag-run-ci-label') ||
|
||||
contains(github.event.comment.body, '/rerun-failed-ci') ||
|
||||
contains(github.event.comment.body, '/tag-and-rerun-ci') ||
|
||||
contains(github.event.comment.body, '/rerun-stage'))
|
||||
contains(github.event.comment.body, '/rerun-stage') ||
|
||||
contains(github.event.comment.body, '/rerun-ut'))
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
@@ -407,6 +409,243 @@ def handle_rerun_stage(
|
||||
return False
|
||||
|
||||
|
||||
CUDA_SUITE_TO_RUNNER = {
|
||||
"stage-a-test-1": "1-gpu-runner",
|
||||
"stage-a-cpu-only": "ubuntu-latest",
|
||||
"stage-b-test-small-1-gpu": "1-gpu-5090",
|
||||
"stage-b-test-large-1-gpu": "1-gpu-runner",
|
||||
"stage-b-test-large-2-gpu": "2-gpu-runner",
|
||||
"stage-b-test-4-gpu-b200": "4-gpu-b200",
|
||||
"stage-c-test-4-gpu-h100": "4-gpu-h100",
|
||||
"stage-c-test-8-gpu-h200": "8-gpu-h200",
|
||||
"stage-c-test-8-gpu-h20": "8-gpu-h20",
|
||||
"stage-c-test-4-gpu-b200": "4-gpu-b200",
|
||||
"stage-c-test-deepep-4-gpu": "4-gpu-h100",
|
||||
"stage-c-test-deepep-8-gpu-h200": "8-gpu-h200",
|
||||
}
|
||||
|
||||
|
||||
def resolve_test_file(file_part):
|
||||
"""
|
||||
Resolve a user-provided file path to a path relative to test/.
|
||||
|
||||
Supports:
|
||||
- Full path: test/registered/core/test_srt_endpoint.py
|
||||
- Relative to test/: registered/core/test_srt_endpoint.py
|
||||
- Bare filename: test_srt_endpoint.py (glob-matched, must be unique)
|
||||
|
||||
Returns (resolved_path, error_message). On success error_message is None.
|
||||
"""
|
||||
if file_part.startswith("test/"):
|
||||
file_part = file_part[len("test/") :]
|
||||
|
||||
if "/" not in file_part:
|
||||
matches = glob.glob(f"test/registered/**/{file_part}", recursive=True)
|
||||
if len(matches) == 0:
|
||||
return (
|
||||
None,
|
||||
f"No test file found matching `{file_part}` under `test/registered/`.",
|
||||
)
|
||||
if len(matches) > 1:
|
||||
match_list = "\n".join(f"- `{m}`" for m in sorted(matches))
|
||||
return None, (
|
||||
f"Ambiguous filename `{file_part}` — matched {len(matches)} files:\n\n"
|
||||
f"{match_list}\n\n"
|
||||
f"Please provide the full path, e.g. `/rerun-ut {matches[0]}`"
|
||||
)
|
||||
return matches[0][len("test/") :], None
|
||||
|
||||
full_path = f"test/{file_part}"
|
||||
if not os.path.isfile(full_path):
|
||||
return None, f"File not found: `{full_path}`"
|
||||
return file_part, None
|
||||
|
||||
|
||||
def detect_cuda_suite(file_path_from_test):
|
||||
"""
|
||||
Read a test file and extract the suite from register_cuda_ci(suite="...").
|
||||
|
||||
Returns (suite_name, runner_label, error_message).
|
||||
"""
|
||||
full_path = f"test/{file_path_from_test}"
|
||||
with open(full_path, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
match = re.search(
|
||||
r'register_cuda_ci\([^)]*suite\s*=\s*["\']([^"\']+)["\']', content
|
||||
)
|
||||
if not match:
|
||||
return (
|
||||
None,
|
||||
None,
|
||||
(
|
||||
f"No `register_cuda_ci()` found in `{full_path}`.\n\n"
|
||||
f"This file may not be a registered CUDA CI test."
|
||||
),
|
||||
)
|
||||
|
||||
suite = match.group(1)
|
||||
runner = CUDA_SUITE_TO_RUNNER.get(suite)
|
||||
if not runner:
|
||||
known = ", ".join(f"`{s}`" for s in sorted(CUDA_SUITE_TO_RUNNER))
|
||||
return (
|
||||
suite,
|
||||
None,
|
||||
(
|
||||
f"Unknown CUDA suite `{suite}` in `{full_path}`.\n\n"
|
||||
f"Known suites: {known}"
|
||||
),
|
||||
)
|
||||
return suite, runner, None
|
||||
|
||||
|
||||
def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token):
|
||||
"""
|
||||
Handles the /rerun-ut <file>::<TestClass.test_method> command.
|
||||
Dispatches a lightweight workflow to run a single test on the correct CUDA runner.
|
||||
"""
|
||||
if not (
|
||||
user_perms.get("can_rerun_ut", False)
|
||||
or user_perms.get("can_rerun_stage", False)
|
||||
):
|
||||
print("Permission denied: neither can_rerun_ut nor can_rerun_stage is true.")
|
||||
return False
|
||||
|
||||
if not test_spec:
|
||||
comment.create_reaction("confused")
|
||||
pr.create_issue_comment(
|
||||
"❌ Please specify a test: `/rerun-ut <file>::<TestClass.test_method>`\n\n"
|
||||
"Examples:\n"
|
||||
"- `/rerun-ut test/registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`\n"
|
||||
"- `/rerun-ut registered/core/test_srt_endpoint.py::TestSRTEndpoint`\n"
|
||||
"- `/rerun-ut test_srt_endpoint.py`"
|
||||
)
|
||||
return False
|
||||
|
||||
# Parse spec: split on :: to get file path and optional test selector
|
||||
if "::" in test_spec:
|
||||
file_part, test_selector = test_spec.split("::", 1)
|
||||
else:
|
||||
file_part = test_spec
|
||||
test_selector = None
|
||||
|
||||
file_part = file_part.strip()
|
||||
if test_selector:
|
||||
test_selector = test_selector.strip()
|
||||
|
||||
# Resolve file path
|
||||
resolved_path, err = resolve_test_file(file_part)
|
||||
if err:
|
||||
comment.create_reaction("confused")
|
||||
pr.create_issue_comment(f"❌ {err}")
|
||||
return False
|
||||
|
||||
# Detect suite and runner
|
||||
suite, runner_label, err = detect_cuda_suite(resolved_path)
|
||||
if err:
|
||||
comment.create_reaction("confused")
|
||||
pr.create_issue_comment(f"❌ {err}")
|
||||
return False
|
||||
|
||||
# Build test_command: file path (+ optional test selector as unittest arg)
|
||||
test_command = resolved_path
|
||||
if test_selector:
|
||||
test_command = f"{resolved_path} {test_selector}"
|
||||
|
||||
print(
|
||||
f"Resolved: file={resolved_path}, selector={test_selector}, "
|
||||
f"suite={suite}, runner={runner_label}, command='{test_command}'"
|
||||
)
|
||||
|
||||
try:
|
||||
workflow_name = "Rerun UT"
|
||||
workflows = gh_repo.get_workflows()
|
||||
target_workflow = None
|
||||
for wf in workflows:
|
||||
if wf.name == workflow_name:
|
||||
target_workflow = wf
|
||||
break
|
||||
|
||||
if not target_workflow:
|
||||
print(f"Error: {workflow_name} workflow not found")
|
||||
return False
|
||||
|
||||
is_fork = (
|
||||
pr.head.repo is None or pr.head.repo.owner.login != gh_repo.owner.login
|
||||
)
|
||||
print(f"PR is from fork: {is_fork}")
|
||||
|
||||
pr_head_sha = None
|
||||
if is_fork:
|
||||
ref = "main"
|
||||
pr_head_sha = pr.head.sha
|
||||
inputs = {
|
||||
"test_command": test_command,
|
||||
"runner_label": runner_label,
|
||||
"pr_head_sha": pr_head_sha,
|
||||
}
|
||||
else:
|
||||
ref = pr.head.ref
|
||||
inputs = {
|
||||
"test_command": test_command,
|
||||
"runner_label": runner_label,
|
||||
}
|
||||
|
||||
dispatch_time = time.time()
|
||||
|
||||
dispatch_url = f"https://api.github.com/repos/{gh_repo.full_name}/actions/workflows/{target_workflow.id}/dispatches"
|
||||
dispatch_resp = requests.post(
|
||||
dispatch_url,
|
||||
json={"ref": ref, "inputs": inputs},
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.github+json",
|
||||
},
|
||||
)
|
||||
success = dispatch_resp.status_code in (200, 204)
|
||||
if not success:
|
||||
print(f"Dispatch failed: {dispatch_resp.status_code} {dispatch_resp.text}")
|
||||
|
||||
if success:
|
||||
print(f"Successfully triggered rerun-ut: {test_command}")
|
||||
comment.create_reaction("+1")
|
||||
pr.create_issue_comment(
|
||||
f"✅ Triggered `/rerun-ut` on `{runner_label}` runner:\n"
|
||||
f"```\ncd test/ && python3 {test_command}\n```"
|
||||
)
|
||||
|
||||
run_url = find_workflow_run_url(
|
||||
gh_repo,
|
||||
target_workflow.id,
|
||||
ref,
|
||||
"rerun-ut",
|
||||
token,
|
||||
dispatch_time,
|
||||
pr_head_sha=pr_head_sha,
|
||||
max_wait=30,
|
||||
)
|
||||
if run_url:
|
||||
pr.create_issue_comment(f"🔗 [View workflow run]({run_url})")
|
||||
else:
|
||||
pr.create_issue_comment(
|
||||
f"⚠️ Could not retrieve workflow run URL. "
|
||||
f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
|
||||
)
|
||||
return True
|
||||
else:
|
||||
print("Failed to trigger workflow_dispatch")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error triggering rerun-ut: {e}")
|
||||
comment.create_reaction("confused")
|
||||
pr.create_issue_comment(
|
||||
f"❌ Failed to trigger rerun-ut: {str(e)}\n\n"
|
||||
f"Please check the logs or contact maintainers."
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
# 1. Load Environment Variables
|
||||
token = get_env_var("GITHUB_TOKEN")
|
||||
@@ -427,7 +666,7 @@ def main():
|
||||
pr = repo.get_pull(pr_number)
|
||||
comment = repo.get_issue(pr_number).get_comment(comment_id)
|
||||
|
||||
# PR authors can always rerun failed CI on their own PRs,
|
||||
# PR authors can always rerun failed CI and rerun individual UTs on their own PRs,
|
||||
# even if they are not listed in CI_PERMISSIONS.json.
|
||||
# Note: /tag-run-ci-label and /rerun-stage still require CI_PERMISSIONS.json.
|
||||
if pr.user.login == user_login:
|
||||
@@ -442,6 +681,7 @@ def main():
|
||||
f"User {user_login} is the PR author and has existing CI permissions."
|
||||
)
|
||||
user_perms["can_rerun_failed_ci"] = True
|
||||
user_perms["can_rerun_ut"] = True
|
||||
|
||||
if not user_perms:
|
||||
print(f"User {user_login} does not have any configured permissions. Exiting.")
|
||||
@@ -486,6 +726,11 @@ def main():
|
||||
stage_name = parts[1].strip() if len(parts) > 1 else None
|
||||
handle_rerun_stage(repo, pr, comment, user_perms, stage_name, token)
|
||||
|
||||
elif first_line.startswith("/rerun-ut"):
|
||||
parts = first_line.split(maxsplit=1)
|
||||
test_spec = parts[1].strip() if len(parts) > 1 else None
|
||||
handle_rerun_ut(repo, pr, comment, user_perms, test_spec, token)
|
||||
|
||||
else:
|
||||
print(f"Unknown or ignored command: {first_line}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user