diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index fd0bc66e7..e9b56ac6a 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -27,6 +27,7 @@ concurrency: env: SGLANG_IS_IN_CI: true + SGLANG_CI_ENABLE_RETRY: true jobs: # =============================================== check changes ==================================================== @@ -550,10 +551,10 @@ jobs: CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh - name: Run test - timeout-minutes: 30 + timeout-minutes: 40 run: | cd test/srt - python3 run_suite.py --suite quantization_test + python3 run_suite.py --suite quantization_test --enable-retry unit-test-backend-1-gpu: needs: [check-changes, call-gate, stage-a-test-1] @@ -592,10 +593,10 @@ jobs: CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh - name: Run test - timeout-minutes: 30 + timeout-minutes: 40 run: | cd test/srt - python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15 + python3 run_suite.py --suite per-commit-1-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 15 --enable-retry unit-test-backend-2-gpu: needs: [check-changes, call-gate, unit-test-backend-1-gpu] @@ -633,10 +634,10 @@ jobs: CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh - name: Run test - timeout-minutes: 30 + timeout-minutes: 40 run: | cd test/srt - python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --enable-retry unit-test-backend-4-gpu: needs: [check-changes, call-gate, unit-test-backend-2-gpu] @@ -674,10 +675,10 @@ jobs: CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh - name: Run test - timeout-minutes: 20 + timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 + python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --enable-retry unit-test-backend-8-gpu-h200: needs: [check-changes, call-gate, unit-test-backend-2-gpu] @@ -721,10 +722,10 @@ jobs: python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code - name: Run test - timeout-minutes: 20 + timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 + python3 run_suite.py --suite per-commit-8-gpu-h200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --enable-retry unit-test-backend-8-gpu-h20: needs: [check-changes, call-gate, unit-test-backend-2-gpu] @@ -763,10 +764,10 @@ jobs: CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh - name: Run test - timeout-minutes: 20 + timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + python3 run_suite.py --suite per-commit-8-gpu-h20 --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --enable-retry performance-test-1-gpu-part-1: needs: [check-changes, call-gate, stage-a-test-1] @@ -1055,10 +1056,12 @@ jobs: pip install -e . - name: Evaluate accuracy - timeout-minutes: 20 + timeout-minutes: 25 run: | cd test/srt python3 test_eval_accuracy_large.py + env: + SGLANG_CI_ENABLE_RETRY: ${{ env.SGLANG_CI_ENABLE_RETRY }} accuracy-test-2-gpu: needs: [check-changes, call-gate, accuracy-test-1-gpu] @@ -1095,10 +1098,12 @@ jobs: pip install -e . - name: Evaluate accuracy (TP=2) - timeout-minutes: 20 + timeout-minutes: 25 run: | cd test/srt python3 test_moe_eval_accuracy_large.py + env: + SGLANG_CI_ENABLE_RETRY: ${{ env.SGLANG_CI_ENABLE_RETRY }} unit-test-deepep-4-gpu: needs: [check-changes, call-gate, unit-test-backend-2-gpu] @@ -1132,10 +1137,10 @@ jobs: CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh - name: Run test - timeout-minutes: 20 + timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite per-commit-4-gpu-deepep + python3 run_suite.py --suite per-commit-4-gpu-deepep --enable-retry unit-test-deepep-8-gpu: needs: [check-changes, call-gate, unit-test-backend-2-gpu] @@ -1169,10 +1174,10 @@ jobs: CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_deepep.sh - name: Run test - timeout-minutes: 20 + timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite per-commit-8-gpu-h200-deepep + python3 run_suite.py --suite per-commit-8-gpu-h200-deepep --enable-retry unit-test-backend-4-gpu-b200: needs: [check-changes, call-gate, unit-test-backend-2-gpu] @@ -1211,10 +1216,10 @@ jobs: CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh - name: Run test - timeout-minutes: 30 + timeout-minutes: 40 run: | cd test/srt - IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 + IS_BLACKWELL=1 python3 run_suite.py --suite per-commit-4-gpu-b200 --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 --enable-retry # TODO: Add gb200 tests back after the ci runner is fixed # unit-test-backend-4-gpu-gb200: @@ -1251,10 +1256,10 @@ jobs: # CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} IS_BLACKWELL=1 GRACE_BLACKWELL=1 bash scripts/ci/ci_install_deepep.sh # - name: Run test - # timeout-minutes: 45 + # timeout-minutes: 55 # run: | # cd test/srt - # python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 + # python3 run_suite.py --suite per-commit-4-gpu-gb200 --auto-partition-id 0 --auto-partition-size 1 --timeout-per-file 3600 --enable-retry pr-test-finish: needs: diff --git a/python/sglang/test/ci/ci_utils.py b/python/sglang/test/ci/ci_utils.py index 447a14a54..7de298645 100644 --- a/python/sglang/test/ci/ci_utils.py +++ b/python/sglang/test/ci/ci_utils.py @@ -1,4 +1,6 @@ +import logging import os +import re import subprocess import threading import time @@ -7,6 +9,8 @@ from typing import Callable, List, Optional from sglang.srt.utils.common import kill_process_tree +logger = logging.getLogger(__name__) + @dataclass class TestFile: @@ -14,6 +18,61 @@ class TestFile: estimated_time: float = 60 +# Patterns that indicate retriable accuracy/performance failures +RETRIABLE_PATTERNS = [ + r"AssertionError:.*not greater than", + r"AssertionError:.*not less than", + r"AssertionError:.*not equal to", + r"AssertionError:.*!=.*expected", + r"accuracy", + r"score", + r"latency", + r"throughput", +] + +# Patterns that indicate non-retriable failures (real code errors) +NON_RETRIABLE_PATTERNS = [ + r"SyntaxError", + r"ImportError", + r"ModuleNotFoundError", + r"NameError", + r"TypeError", + r"AttributeError", + r"RuntimeError", + r"CUDA out of memory", + r"OOM", + r"Segmentation fault", + r"core dumped", + r"ConnectionRefusedError", + r"FileNotFoundError", +] + + +def is_retriable_failure(output: str) -> tuple[bool, str]: + """ + Determine if a test failure is retriable based on output patterns. + + Returns: + tuple: (is_retriable, reason) + """ + # Check for non-retriable patterns first + for pattern in NON_RETRIABLE_PATTERNS: + if re.search(pattern, output, re.IGNORECASE): + return False, f"non-retriable error: {pattern}" + + # Check for retriable patterns + for pattern in RETRIABLE_PATTERNS: + if re.search(pattern, output, re.IGNORECASE): + return True, f"retriable pattern: {pattern}" + + # If we have an AssertionError but didn't match non-retriable, assume retriable + if re.search(r"AssertionError", output): + return True, "AssertionError (assuming retriable)" + + # Default: not retriable + return False, "unknown failure type" + + def run_with_timeout( func: Callable, args: tuple = (), @@ -38,8 +97,21 @@ def run_with_timeout( return ret_value[0] +def write_github_step_summary(content: str): + """Write content to GitHub Step Summary if available.""" + summary_file = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_file: + with open(summary_file, "a") as f: + f.write(content) + + def run_unittest_files( - files: List[TestFile], timeout_per_file: float, continue_on_error: bool = False + files: List[TestFile], + timeout_per_file: float, + continue_on_error: bool = False, + enable_retry: bool = False, + max_attempts: int = 2, + retry_wait_seconds: int = 60, ): """ Run a list of test files. @@ -49,86 +121,168 @@ def run_unittest_files( timeout_per_file: Timeout in seconds for each test file continue_on_error: If True, continue running remaining tests even if one fails. If False, stop at first failure (default behavior for PR tests). + enable_retry: If True, retry failed tests that appear to be accuracy/performance + assertion failures (not code errors). + max_attempts: Maximum number of attempts per file including initial run (default: 2). + retry_wait_seconds: Seconds to wait between retries (default: 60). """ tic = time.perf_counter() success = True passed_tests = [] failed_tests = [] + retried_tests = [] # Track which tests were retried for i, file in enumerate(files): filename, estimated_time = file.name, file.estimated_time process = None + output_lines = [] - def run_one_file(filename): - nonlocal process + def run_one_file(filename, capture_output=False): + nonlocal process, output_lines - filename = os.path.join(os.getcwd(), filename) - print( - f".\n.\nBegin ({i}/{len(files) - 1}):\npython3 {filename}\n.\n.\n", - flush=True, + full_path = os.path.join(os.getcwd(), filename) + logger.info( + f".\n.\nBegin ({i}/{len(files) - 1}):\npython3 {full_path}\n.\n.\n" ) - tic = time.perf_counter() + file_tic = time.perf_counter() - process = subprocess.Popen( - ["python3", filename], stdout=None, stderr=None, env=os.environ - ) - process.wait() - elapsed = time.perf_counter() - tic + if capture_output: + # Capture output for retry decision + process = subprocess.Popen( + ["python3", full_path], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=os.environ, + text=True, + ) + output_lines = [] + for line in process.stdout: + logger.info(line.rstrip()) + output_lines.append(line) + process.wait() + else: + process = subprocess.Popen( + ["python3", full_path], stdout=None, stderr=None, env=os.environ + ) + process.wait() - print( - f".\n.\nEnd ({i}/{len(files) - 1}):\n{filename=}, {elapsed=:.0f}, {estimated_time=}\n.\n.\n", - flush=True, + elapsed = time.perf_counter() - file_tic + + logger.info( + f".\n.\nEnd ({i}/{len(files) - 1}):\n{filename=}, {elapsed=:.0f}, {estimated_time=}\n.\n.\n" ) return process.returncode - try: - ret_code = run_with_timeout( - run_one_file, args=(filename,), timeout=timeout_per_file - ) - if ret_code != 0: - print( - f"\n✗ FAILED: {filename} returned exit code {ret_code}\n", - flush=True, + # Retry loop for each file + attempt = 1 + file_passed = False + was_retried = False + + while attempt <= (max_attempts if enable_retry else 1): + if attempt > 1: + logger.info( + f"\n[CI Retry] Attempt {attempt}/{max_attempts} for {filename}\n" ) - success = False - failed_tests.append((filename, f"exit code {ret_code}")) - if not continue_on_error: - # Stop at first failure for PR tests + was_retried = True + + try: + ret_code = run_with_timeout( + run_one_file, + args=(filename,), + kwargs={"capture_output": enable_retry}, + timeout=timeout_per_file, + ) + + if ret_code == 0: + file_passed = True + if was_retried: + logger.info( + f"\n✓ PASSED on retry (attempt {attempt}): {filename}\n" + ) + retried_tests.append((filename, attempt, "passed")) + passed_tests.append(filename) break - # Otherwise continue to next test for nightly tests - else: - passed_tests.append(filename) - except TimeoutError: - kill_process_tree(process.pid) - time.sleep(5) - print( - f"\n✗ TIMEOUT: {filename} after {timeout_per_file} seconds\n", - flush=True, - ) - success = False - failed_tests.append((filename, f"timeout after {timeout_per_file}s")) - if not continue_on_error: - # Stop at first timeout for PR tests + else: + # Check if we should retry + if enable_retry and attempt < max_attempts: + output = "".join(output_lines) + is_retriable, reason = is_retriable_failure(output) + + if is_retriable: + logger.info( + f"\n[CI Retry] {filename} failed with {reason}" + ) + logger.info( + f"[CI Retry] Waiting {retry_wait_seconds}s before retry...\n" + ) + time.sleep(retry_wait_seconds) + attempt += 1 + continue + else: + logger.info( + f"\n[CI Retry] {filename} failed with {reason} - not retrying\n" + ) + + # No retry or not retriable + logger.info( + f"\n✗ FAILED: {filename} returned exit code {ret_code}\n" + ) + if was_retried: + retried_tests.append((filename, attempt, "failed")) + failed_tests.append((filename, f"exit code {ret_code}")) + break + + except TimeoutError: + kill_process_tree(process.pid) + time.sleep(5) + logger.info( + f"\n✗ TIMEOUT: {filename} after {timeout_per_file} seconds\n" + ) + if was_retried: + retried_tests.append((filename, attempt, "timeout")) + failed_tests.append((filename, f"timeout after {timeout_per_file}s")) break - # Otherwise continue to next test for nightly tests + + if not file_passed: + success = False + if not continue_on_error: + break + + elapsed_total = time.perf_counter() - tic if success: - print(f"Success. Time elapsed: {time.perf_counter() - tic:.2f}s", flush=True) + logger.info(f"Success. Time elapsed: {elapsed_total:.2f}s") else: - print(f"Fail. Time elapsed: {time.perf_counter() - tic:.2f}s", flush=True) + logger.info(f"Fail. Time elapsed: {elapsed_total:.2f}s") # Print summary - print(f"\n{'='*60}", flush=True) - print(f"Test Summary: {len(passed_tests)}/{len(files)} passed", flush=True) - print(f"{'='*60}", flush=True) + logger.info(f"\n{'='*60}") + logger.info(f"Test Summary: {len(passed_tests)}/{len(files)} passed") + if enable_retry and retried_tests: + logger.info(f"Retries: {len(retried_tests)} test(s) were retried") + logger.info(f"{'='*60}") if passed_tests: - print("✓ PASSED:", flush=True) + logger.info("✓ PASSED:") for test in passed_tests: - print(f" {test}", flush=True) + logger.info(f" {test}") if failed_tests: - print("\n✗ FAILED:", flush=True) + logger.info("\n✗ FAILED:") for test, reason in failed_tests: - print(f" {test} ({reason})", flush=True) - print(f"{'='*60}\n", flush=True) + logger.info(f" {test} ({reason})") + if retried_tests: + logger.info("\n↻ RETRIED:") + for test, attempts, result in retried_tests: + logger.info(f" {test} ({attempts} attempts, {result})") + logger.info(f"{'='*60}\n") + + # Write GitHub Step Summary + if retried_tests: + summary = "\n### CI Retry Summary\n\n" + summary += "| Test File | Attempts | Result |\n" + summary += "|-----------|----------|--------|\n" + for test, attempts, result in retried_tests: + summary += f"| `{test}` | {attempts} | {result} |\n" + summary += "\n" + write_github_step_summary(summary) return 0 if success else -1 diff --git a/test/run_suite.py b/test/run_suite.py index 49ef52ee9..855f212a0 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -136,6 +136,9 @@ def run_a_suite(args): test_files, timeout_per_file=args.timeout_per_file, continue_on_error=args.continue_on_error, + enable_retry=args.enable_retry, + max_attempts=args.max_attempts, + retry_wait_seconds=args.retry_wait_seconds, ) @@ -178,6 +181,24 @@ def main(): type=int, help="Use auto load balancing. The number of parts.", ) + parser.add_argument( + "--enable-retry", + action="store_true", + default=False, + help="Enable smart retry for accuracy/performance assertion failures (not code errors)", + ) + parser.add_argument( + "--max-attempts", + type=int, + default=2, + help="Maximum number of attempts per file including initial run (default: 2)", + ) + parser.add_argument( + "--retry-wait-seconds", + type=int, + default=60, + help="Seconds to wait between retries (default: 60)", + ) args = parser.parse_args() # Validate auto-partition arguments diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 52389e8c2..21d24b9ca 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -511,6 +511,24 @@ def main(): default=False, help="Continue running remaining tests even if one fails (useful for nightly tests)", ) + arg_parser.add_argument( + "--enable-retry", + action="store_true", + default=False, + help="Enable smart retry for accuracy/performance assertion failures (not code errors)", + ) + arg_parser.add_argument( + "--max-attempts", + type=int, + default=2, + help="Maximum number of attempts per file including initial run (default: 2)", + ) + arg_parser.add_argument( + "--retry-wait-seconds", + type=int, + default=60, + help="Seconds to wait between retries (default: 60)", + ) args = arg_parser.parse_args() print(f"{args=}") @@ -526,7 +544,14 @@ def main(): print("The running tests are ", [f.name for f in files]) - exit_code = run_unittest_files(files, args.timeout_per_file, args.continue_on_error) + exit_code = run_unittest_files( + files, + args.timeout_per_file, + args.continue_on_error, + enable_retry=args.enable_retry, + max_attempts=args.max_attempts, + retry_wait_seconds=args.retry_wait_seconds, + ) exit(exit_code)