diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index d1e9520c0..576e609c1 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -125,7 +125,7 @@ jobs: timeout-minutes: 60 run: | cd test/srt - python3 run_suite.py --suite nightly-1-gpu + python3 run_suite.py --suite nightly-1-gpu --continue-on-error nightly-test-4-gpu: if: github.repository == 'sgl-project/sglang' @@ -143,7 +143,7 @@ jobs: timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite nightly-4-gpu + python3 run_suite.py --suite nightly-4-gpu --continue-on-error nightly-test-8-gpu-h200: if: github.repository == 'sgl-project/sglang' @@ -161,7 +161,7 @@ jobs: timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite nightly-8-gpu-h200 + python3 run_suite.py --suite nightly-8-gpu-h200 --continue-on-error nightly-test-8-gpu-h20: if: github.repository == 'sgl-project/sglang' @@ -181,7 +181,7 @@ jobs: timeout-minutes: 30 run: | cd test/srt - python3 run_suite.py --suite nightly-8-gpu-h20 + python3 run_suite.py --suite nightly-8-gpu-h20 --continue-on-error nightly-test-4-gpu-b200: if: github.repository == 'sgl-project/sglang' @@ -199,4 +199,4 @@ jobs: timeout-minutes: 60 run: | cd test/srt - python3 run_suite.py --suite nightly-4-gpu-b200 + python3 run_suite.py --suite nightly-4-gpu-b200 --continue-on-error diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index bdc50b8f3..42af14537 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -735,9 +735,22 @@ class TestFile: estimated_time: float = 60 -def run_unittest_files(files: List[TestFile], timeout_per_file: float): +def run_unittest_files( + files: List[TestFile], timeout_per_file: float, continue_on_error: bool = False +): + """ + Run a list of test files. + + Args: + files: List of TestFile objects to run + timeout_per_file: Timeout in seconds for each test file + continue_on_error: If True, continue running remaining tests even if one fails. + If False, stop at first failure (default behavior for PR tests). + """ tic = time.perf_counter() success = True + passed_tests = [] + failed_tests = [] for i, file in enumerate(files): filename, estimated_time = file.name, file.estimated_time @@ -769,24 +782,52 @@ def run_unittest_files(files: List[TestFile], timeout_per_file: float): ret_code = run_with_timeout( run_one_file, args=(filename,), timeout=timeout_per_file ) - assert ( - ret_code == 0 - ), f"expected return code 0, but {filename} returned {ret_code}" + if ret_code != 0: + print( + f"\nāœ— FAILED: {filename} returned exit code {ret_code}\n", + flush=True, + ) + success = False + failed_tests.append((filename, f"exit code {ret_code}")) + if not continue_on_error: + # Stop at first failure for PR tests + break + # Otherwise continue to next test for nightly tests + else: + passed_tests.append(filename) except TimeoutError: kill_process_tree(process.pid) time.sleep(5) print( - f"\nTimeout after {timeout_per_file} seconds when running {filename}\n", + f"\nāœ— TIMEOUT: {filename} after {timeout_per_file} seconds\n", flush=True, ) success = False - break + failed_tests.append((filename, f"timeout after {timeout_per_file}s")) + if not continue_on_error: + # Stop at first timeout for PR tests + break + # Otherwise continue to next test for nightly tests if success: print(f"Success. Time elapsed: {time.perf_counter() - tic:.2f}s", flush=True) else: print(f"Fail. Time elapsed: {time.perf_counter() - tic:.2f}s", flush=True) + # Print summary + print(f"\n{'='*60}", flush=True) + print(f"Test Summary: {len(passed_tests)}/{len(files)} passed", flush=True) + print(f"{'='*60}", flush=True) + if passed_tests: + print("āœ“ PASSED:", flush=True) + for test in passed_tests: + print(f" {test}", flush=True) + if failed_tests: + print("\nāœ— FAILED:", flush=True) + for test, reason in failed_tests: + print(f" {test} ({reason})", flush=True) + print(f"{'='*60}\n", flush=True) + return 0 if success else -1 diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 19b6b0586..f1972a4cc 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -645,6 +645,12 @@ if __name__ == "__main__": type=int, help="Use auto load balancing. The number of parts.", ) + arg_parser.add_argument( + "--continue-on-error", + action="store_true", + default=False, + help="Continue running remaining tests even if one fails (useful for nightly tests)", + ) args = arg_parser.parse_args() print(f"{args=}") @@ -662,5 +668,5 @@ if __name__ == "__main__": print("The running tests are ", [f.name for f in files]) - exit_code = run_unittest_files(files, args.timeout_per_file) + exit_code = run_unittest_files(files, args.timeout_per_file, args.continue_on_error) exit(exit_code)