From e884b176325037a65680d21f19f5cb0e026c808a Mon Sep 17 00:00:00 2001 From: Kangyan-Zhou Date: Sat, 31 Jan 2026 20:37:55 -0800 Subject: [PATCH] Fix rerun stage command with merged commit history (#17960) --- .github/workflows/pr-test.yml | 121 ++++++++++++++++++++++++++++++---- 1 file changed, 110 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index ca8c6ea9f..4b3c835d9 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -72,10 +72,16 @@ jobs: check-changes: runs-on: ubuntu-latest outputs: - main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} - sgl_kernel: ${{ steps.filter.outputs.sgl_kernel }} # sgl-kernel tests only run when kernels are rebuilt - jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} - multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} + # Use API-based detection for target_stage mode (filter-api), otherwise use dorny/paths-filter (filter) + main_package: ${{ steps.filter-api.outputs.main_package || steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} + # sgl_kernel is forced to false when target_stage is set, since sgl-kernel-build-wheels won't run + # This prevents CUSTOM_BUILD_SGL_KERNEL=true when the wheel artifacts aren't available + # Note: If PR has kernel changes AND target_stage is set, the validate-target-stage step will fail + sgl_kernel: ${{ !inputs.target_stage && (steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel) }} + # Raw sgl_kernel value before target_stage override (used for validation) + sgl_kernel_raw: ${{ steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel }} + jit_kernel: ${{ steps.filter-api.outputs.jit_kernel || steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} + multimodal_gen: ${{ steps.filter-api.outputs.multimodal_gen || steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} max_parallel: ${{ steps.set-parallel.outputs.max_parallel }} b200_runner: ${{ steps.set-runner.outputs.b200_runner }} enable_retry: ${{ steps.set-retry.outputs.enable_retry }} @@ -102,7 +108,9 @@ jobs: - name: Detect file changes id: filter uses: dorny/paths-filter@v3 - if: steps.run-mode.outputs.run_all_tests != 'true' + # Only use paths-filter for pull_request events (where it works correctly) + # For workflow_dispatch with target_stage, we use GitHub API in the next step + if: steps.run-mode.outputs.run_all_tests != 'true' && !inputs.target_stage with: filters: | main_package: @@ -124,6 +132,70 @@ jobs: - "python/pyproject.toml" - ".github/workflows/pr-test.yml" + # For /rerun-stage (workflow_dispatch with target_stage), dorny/paths-filter doesn't work + # correctly because it falls back to "last commit" detection which breaks for merge commits. + # Instead, we use the GitHub API to compare the PR commit against main. + - name: Detect file changes via API (for target_stage) + id: filter-api + if: inputs.target_stage && inputs.pr_head_sha + env: + GH_TOKEN: ${{ github.token }} + run: | + echo "Detecting file changes via GitHub API for target_stage mode..." + echo "PR head SHA: ${{ inputs.pr_head_sha }}" + + # Get the list of changed files by comparing PR commit against main + # This correctly handles merge commits by looking at the actual PR diff + CHANGED_FILES=$(gh api "repos/${{ github.repository }}/compare/main...${{ inputs.pr_head_sha }}" \ + --jq '[.files[].filename] | .[]' 2>/dev/null || echo "") + + if [ -z "$CHANGED_FILES" ]; then + echo "Warning: Could not fetch changed files from API, assuming no changes" + echo "sgl_kernel=false" >> $GITHUB_OUTPUT + echo "main_package=false" >> $GITHUB_OUTPUT + echo "jit_kernel=false" >> $GITHUB_OUTPUT + echo "multimodal_gen=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Changed files:" + echo "$CHANGED_FILES" | head -20 + echo "..." + + # Check for sgl-kernel changes + if echo "$CHANGED_FILES" | grep -q "^sgl-kernel/"; then + echo "sgl_kernel=true" >> $GITHUB_OUTPUT + echo "Detected sgl-kernel changes" + else + echo "sgl_kernel=false" >> $GITHUB_OUTPUT + fi + + # Check for main_package changes (excluding multimodal_gen) + # Note: Need to filter out multimodal_gen before checking, not pipe grep -q output + MAIN_PKG_FILES=$(echo "$CHANGED_FILES" | grep -E "^(python/sglang/|python/pyproject\.toml|scripts/ci/cuda/|scripts/ci/utils/|test/|\.github/workflows/pr-test\.yml)" | grep -v "^python/sglang/multimodal_gen/" || true) + if [ -n "$MAIN_PKG_FILES" ]; then + echo "main_package=true" >> $GITHUB_OUTPUT + echo "Detected main_package changes" + else + echo "main_package=false" >> $GITHUB_OUTPUT + fi + + # Check for jit_kernel changes + if echo "$CHANGED_FILES" | grep -qE "^(python/sglang/jit_kernel/|python/pyproject\.toml|\.github/workflows/pr-test\.yml)"; then + echo "jit_kernel=true" >> $GITHUB_OUTPUT + echo "Detected jit_kernel changes" + else + echo "jit_kernel=false" >> $GITHUB_OUTPUT + fi + + # Check for multimodal_gen changes + if echo "$CHANGED_FILES" | grep -qE "^(python/sglang/multimodal_gen/|python/sglang/cli/|python/pyproject\.toml|\.github/workflows/pr-test\.yml)"; then + echo "multimodal_gen=true" >> $GITHUB_OUTPUT + echo "Detected multimodal_gen changes" + else + echo "multimodal_gen=false" >> $GITHUB_OUTPUT + fi + - name: Set max-parallel based on run type id: set-parallel run: | @@ -142,8 +214,12 @@ jobs: - name: Set B200 runner tag id: set-runner run: | - sgl_kernel="${{ steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }}" - if [[ "$sgl_kernel" == "true" ]]; then + # Use kernel-build runner only when sgl_kernel changes are detected AND we're not in target_stage mode + # (target_stage skips wheel builds, so we can't use custom kernels) + # Use API-based detection (filter-api) for target_stage mode, otherwise use dorny/paths-filter (filter) + sgl_kernel="${{ steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }}" + target_stage="${{ inputs.target_stage }}" + if [[ "$sgl_kernel" == "true" && -z "$target_stage" ]]; then echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT else echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT @@ -166,6 +242,26 @@ jobs: echo "Filtered run, continue-on-error disabled" fi + - name: Validate target_stage with kernel changes + # Use API-based detection (filter-api) for target_stage mode, otherwise use dorny/paths-filter (filter) + if: inputs.target_stage && (steps.filter-api.outputs.sgl_kernel == 'true' || steps.filter.outputs.sgl_kernel == 'true') + run: | + echo "::error::Cannot use /rerun-stage when PR has sgl-kernel changes." + echo "::error::The sgl-kernel-build-wheels job is skipped in target_stage mode, but this PR modifies sgl-kernel/ files." + echo "::error::Please use /tag-and-rerun-ci to run the full workflow including kernel builds." + echo "" + echo "ERROR: Cannot use /rerun-stage when PR has sgl-kernel changes." + echo "" + echo "This PR modifies files in sgl-kernel/, which requires building custom kernel wheels." + echo "The /rerun-stage command skips the wheel build job, so the test would run against" + echo "the wrong (PyPI) version of sgl-kernel instead of your changes." + echo "" + echo "To properly test your kernel changes, use one of these commands instead:" + echo " /tag-and-rerun-ci - Re-run the full workflow including kernel builds" + echo " /rerun-ci - Re-run the full workflow" + echo "" + exit 1 + - name: Show filter results in summary (table) run: | { @@ -173,10 +269,13 @@ jobs: echo "" echo "| Component | Changed |" echo "|-------------------|---------|" - echo "| main_package | ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} |" - echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel || steps.run-mode.outputs.run_all_tests }} |" - echo "| jit_kernel | ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} |" - echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} |" + echo "| main_package | ${{ steps.filter-api.outputs.main_package || steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} |" + echo "| sgl_kernel (raw) | ${{ steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel }} |" + echo "| sgl_kernel (used) | ${{ !inputs.target_stage && (steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel) }} |" + echo "| jit_kernel | ${{ steps.filter-api.outputs.jit_kernel || steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} |" + echo "| multimodal_gen | ${{ steps.filter-api.outputs.multimodal_gen || steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} |" + echo "| target_stage | ${{ inputs.target_stage || '(none)' }} |" + echo "| detection_method | ${{ inputs.target_stage && 'GitHub API' || 'dorny/paths-filter' }} |" echo "| max_parallel | ${{ steps.set-parallel.outputs.max_parallel }} |" echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |" echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"