diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 894eaa061..8ff7f6eee 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -674,6 +674,10 @@ jobs: runs-on: 2-gpu-runner env: RUNNER_LABELS: 2-gpu-runner + strategy: + fail-fast: false + matrix: + partition: [0, 1] steps: - name: Checkout code uses: actions/checkout@v4 @@ -700,7 +704,7 @@ jobs: if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then CONTINUE_ON_ERROR_FLAG="--continue-on-error" fi - python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu $CONTINUE_ON_ERROR_FLAG + python3 run_suite.py --hw cuda --suite stage-b-test-large-2-gpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size 2 $CONTINUE_ON_ERROR_FLAG stage-c-test-large-4-gpu: needs: [check-changes, call-gate, stage-b-test-small-1-gpu, stage-b-test-large-1-gpu, stage-b-test-large-2-gpu, sgl-kernel-build-wheels] @@ -1027,57 +1031,6 @@ jobs: IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 $CONTINUE_ON_ERROR_FLAG - unit-test-backend-2-gpu: - needs: [check-changes, call-gate, unit-test-backend-1-gpu] - if: | - always() && - ( - (inputs.target_stage == 'unit-test-backend-2-gpu') || - ( - !inputs.target_stage && - (github.event_name == 'schedule' || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 2-gpu-runner - env: - RUNNER_LABELS: 2-gpu-runner - strategy: - fail-fast: false - matrix: - part: [0, 1] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda12.9 - - - name: Install dependencies - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - run: | - cd test/srt - RETRY_FLAG="" - if [[ "${{ needs.check-changes.outputs.enable_retry }}" == "true" ]]; then - RETRY_FLAG="--enable-retry" - fi - CONTINUE_ON_ERROR_FLAG="" - if [[ "${{ needs.check-changes.outputs.continue_on_error }}" == "true" ]]; then - CONTINUE_ON_ERROR_FLAG="--continue-on-error" - fi - python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 $RETRY_FLAG $CONTINUE_ON_ERROR_FLAG - unit-test-backend-4-gpu: needs: [check-changes, call-gate, unit-test-backend-1-gpu, stage-b-test-4-gpu-b200] if: | @@ -1802,7 +1755,6 @@ jobs: stage-c-test-large-4-gpu, quantization-test, unit-test-backend-1-gpu, - unit-test-backend-2-gpu, stage-b-test-4-gpu-b200, unit-test-backend-4-gpu, unit-test-backend-8-gpu-h20, diff --git a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py index 9210f38d8..17391c577 100644 --- a/python/sglang/multimodal_gen/runtime/loader/weight_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/weight_utils.py @@ -219,6 +219,26 @@ def safetensors_weights_iterator( yield name, param +def _load_pt_file(bin_file: str, device: str) -> dict: + """Load a PyTorch checkpoint file, handling legacy tar format. + + PyTorch 2.6 changed the default of weights_only from False to True. + Legacy tar format files cannot be loaded with weights_only=True. + This function tries weights_only=True first, then falls back to False + for legacy tar format files from trusted sources (HuggingFace Hub). + """ + try: + return torch.load(bin_file, map_location=device, weights_only=True) + except RuntimeError as e: + if "legacy .tar format" in str(e): + logger.warning( + "Loading %s with weights_only=False (legacy tar format)", + os.path.basename(bin_file), + ) + return torch.load(bin_file, map_location=device, weights_only=False) + raise + + def pt_weights_iterator( hf_weights_files: list[str], to_cpu: bool = True, @@ -234,7 +254,7 @@ def pt_weights_iterator( disable=not enable_tqdm, bar_format=_BAR_FORMAT, ): - state = torch.load(bin_file, map_location=device, weights_only=True) + state = _load_pt_file(bin_file, device) yield from state.items() del state diff --git a/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py b/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py index 67740409f..c678c0998 100644 --- a/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py +++ b/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py @@ -65,7 +65,10 @@ _active_symmetric_memory_context = None def is_symmetric_memory_enabled(): - return get_global_server_args().enable_symm_mem + try: + return get_global_server_args().enable_symm_mem + except ValueError: + return False def set_graph_pool_id(graph_pool_id): diff --git a/python/sglang/srt/model_loader/weight_utils.py b/python/sglang/srt/model_loader/weight_utils.py index 59a9da951..f483774f2 100644 --- a/python/sglang/srt/model_loader/weight_utils.py +++ b/python/sglang/srt/model_loader/weight_utils.py @@ -822,6 +822,26 @@ def multi_thread_safetensors_weights_iterator( yield name, param +def _load_pt_file(bin_file: str) -> dict: + """Load a PyTorch checkpoint file, handling legacy tar format. + + PyTorch 2.6 changed the default of weights_only from False to True. + Legacy tar format files cannot be loaded with weights_only=True. + This function tries weights_only=True first, then falls back to False + for legacy tar format files from trusted sources (HuggingFace Hub). + """ + try: + return torch.load(bin_file, map_location="cpu", weights_only=True) + except RuntimeError as e: + if "legacy .tar format" in str(e): + logger.warning( + "Loading %s with weights_only=False (legacy tar format)", + os.path.basename(bin_file), + ) + return torch.load(bin_file, map_location="cpu", weights_only=False) + raise + + def pt_weights_iterator( hf_weights_files: List[str], ) -> Generator[Tuple[str, torch.Tensor], None, None]: @@ -835,7 +855,7 @@ def pt_weights_iterator( disable=not enable_tqdm, bar_format=_BAR_FORMAT, ): - state = torch.load(bin_file, map_location="cpu", weights_only=True) + state = _load_pt_file(bin_file) yield from state.items() del state @@ -849,12 +869,9 @@ def multi_thread_pt_weights_iterator( not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 ) - def _load_file(bin_file: str): - return torch.load(bin_file, map_location="cpu", weights_only=True) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [ - executor.submit(_load_file, bin_file) for bin_file in hf_weights_files + executor.submit(_load_pt_file, bin_file) for bin_file in hf_weights_files ] if enable_tqdm: diff --git a/scripts/ci/slash_command_handler.py b/scripts/ci/slash_command_handler.py index 14ab0c245..d1cc5f9c2 100644 --- a/scripts/ci/slash_command_handler.py +++ b/scripts/ci/slash_command_handler.py @@ -155,7 +155,6 @@ def handle_rerun_stage( "multimodal-gen-test-2-gpu", "quantization-test", "unit-test-backend-1-gpu", - "unit-test-backend-2-gpu", "stage-b-test-4-gpu-b200", "unit-test-backend-4-gpu", "unit-test-backend-8-gpu-h200", diff --git a/test/srt/test_disaggregation_basic.py b/test/registered/disaggregation/test_disaggregation_basic.py similarity index 99% rename from test/srt/test_disaggregation_basic.py rename to test/registered/disaggregation/test_disaggregation_basic.py index c012c051c..bc2e77f1d 100644 --- a/test/srt/test_disaggregation_basic.py +++ b/test/registered/disaggregation/test_disaggregation_basic.py @@ -7,6 +7,7 @@ import openai import requests from transformers import AutoTokenizer +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, @@ -19,6 +20,8 @@ from sglang.test.test_utils import ( popen_launch_pd_server, ) +register_cuda_ci(est_time=400, suite="stage-b-test-large-2-gpu") + class TestDisaggregationAccuracy(PDDisaggregationServerBase): @classmethod diff --git a/test/srt/test_data_parallelism.py b/test/registered/distributed/test_data_parallelism.py similarity index 91% rename from test/srt/test_data_parallelism.py rename to test/registered/distributed/test_data_parallelism.py index e54e3a5cf..25eba4a36 100644 --- a/test/srt/test_data_parallelism.py +++ b/test/registered/distributed/test_data_parallelism.py @@ -5,6 +5,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, @@ -14,6 +15,9 @@ from sglang.test.test_utils import ( popen_launch_server, ) +register_cuda_ci(est_time=73, suite="stage-b-test-large-2-gpu") +register_amd_ci(est_time=73, suite="stage-b-test-large-2-gpu-amd") + class TestDataParallelism(CustomTestCase): @classmethod diff --git a/test/srt/test_dp_attention.py b/test/registered/distributed/test_dp_attention.py similarity index 97% rename from test/srt/test_dp_attention.py rename to test/registered/distributed/test_dp_attention.py index 80bf43af4..2a3d6b608 100644 --- a/test/srt/test_dp_attention.py +++ b/test/registered/distributed/test_dp_attention.py @@ -5,6 +5,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test from sglang.test.run_eval import run_eval @@ -19,6 +20,8 @@ from sglang.test.test_utils import ( popen_launch_server, ) +register_cuda_ci(est_time=350, suite="stage-b-test-large-2-gpu") + class TestDPAttentionDP2TP2(CustomTestCase): @classmethod diff --git a/test/srt/test_load_weights_from_remote_instance.py b/test/registered/distributed/test_load_weights_from_remote_instance.py similarity index 98% rename from test/srt/test_load_weights_from_remote_instance.py rename to test/registered/distributed/test_load_weights_from_remote_instance.py index db094da0c..128b9a86f 100644 --- a/test/srt/test_load_weights_from_remote_instance.py +++ b/test/registered/distributed/test_load_weights_from_remote_instance.py @@ -24,6 +24,7 @@ import torch import torch.multiprocessing as mp import sglang as sgl +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( DEFAULT_PORT_FOR_SRT_TEST_RUNNER, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, @@ -37,6 +38,9 @@ from sglang.utils import terminate_process mp.set_start_method("spawn", force=True) +register_cuda_ci(est_time=72, suite="stage-b-test-large-2-gpu") +register_amd_ci(est_time=72, suite="stage-b-test-large-2-gpu-amd") + def verify_params_close(params1, params2, error_msg): """Verify if two parameter arrays are close enough.""" diff --git a/test/srt/hicache/test_hicache_storage_3fs_backend.py b/test/registered/hicache/test_hicache_storage_3fs_backend.py similarity index 93% rename from test/srt/hicache/test_hicache_storage_3fs_backend.py rename to test/registered/hicache/test_hicache_storage_3fs_backend.py index c91772881..ec28b53c3 100644 --- a/test/srt/hicache/test_hicache_storage_3fs_backend.py +++ b/test/registered/hicache/test_hicache_storage_3fs_backend.py @@ -1,7 +1,7 @@ """ Benchmark tests for HiCache Storage with 3FS backend. Usage: - python3 -m pytest test/srt/hicache/test_hicache_storage_3fs_backend.py -v + python3 -m pytest test/registered/hicache/test_hicache_storage_3fs_backend.py -v """ import json @@ -10,8 +10,11 @@ import unittest from test_hicache_storage_file_backend import HiCacheStorageBaseMixin +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase +register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu") + class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin): """Base mixin class with common setup and utilities""" diff --git a/test/srt/hicache/test_hicache_storage_file_backend.py b/test/registered/hicache/test_hicache_storage_file_backend.py similarity index 98% rename from test/srt/hicache/test_hicache_storage_file_backend.py rename to test/registered/hicache/test_hicache_storage_file_backend.py index 382db07b3..153536fc3 100644 --- a/test/srt/hicache/test_hicache_storage_file_backend.py +++ b/test/registered/hicache/test_hicache_storage_file_backend.py @@ -1,7 +1,7 @@ """ E2E tests for HiCache Storage functionality. Usage: - python3 -m pytest test/srt/hicache/test_hicache_storage_e2e.py -v + python3 -m pytest test/registered/hicache/test_hicache_storage_file_backend.py -v """ import json @@ -18,6 +18,7 @@ import requests from sglang.bench_serving import get_tokenizer from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, @@ -29,6 +30,8 @@ from sglang.test.test_utils import ( popen_launch_server, ) +register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu") + class HiCacheStorageBaseMixin: """Base mixin class with common setup and utilities""" diff --git a/test/srt/hicache/test_hicache_storage_mooncake_backend.py b/test/registered/hicache/test_hicache_storage_mooncake_backend.py similarity index 98% rename from test/srt/hicache/test_hicache_storage_mooncake_backend.py rename to test/registered/hicache/test_hicache_storage_mooncake_backend.py index 7e0c83134..947bce776 100644 --- a/test/srt/hicache/test_hicache_storage_mooncake_backend.py +++ b/test/registered/hicache/test_hicache_storage_mooncake_backend.py @@ -1,7 +1,7 @@ """ Benchmark tests for HiCache Storage with Mooncake backend. Usage: - python3.10 -m pytest test/srt/hicache/test_hicache_storage_mooncake_backend.py -v + python3.10 -m pytest test/registered/hicache/test_hicache_storage_mooncake_backend.py -v """ import os @@ -12,6 +12,7 @@ import unittest import requests from test_hicache_storage_file_backend import HiCacheStorageBaseMixin +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, CustomTestCase, @@ -19,6 +20,8 @@ from sglang.test.test_utils import ( is_in_ci, ) +register_cuda_ci(est_time=300, suite="stage-b-test-large-2-gpu") + class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin): """Base mixin class with common setup and utilities""" diff --git a/test/srt/models/test_kimi_linear_models.py b/test/registered/models/test_kimi_linear_models.py similarity index 91% rename from test/srt/models/test_kimi_linear_models.py rename to test/registered/models/test_kimi_linear_models.py index 25e280f1f..88ef6d7ad 100644 --- a/test/srt/models/test_kimi_linear_models.py +++ b/test/registered/models/test_kimi_linear_models.py @@ -2,6 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -10,6 +11,8 @@ from sglang.test.test_utils import ( popen_launch_server, ) +register_cuda_ci(est_time=90, suite="stage-b-test-large-2-gpu") + class TestKimiLinear(CustomTestCase): @classmethod diff --git a/test/srt/models/test_nvidia_nemotron_nano_v2.py b/test/registered/models/test_nvidia_nemotron_nano_v2.py similarity index 96% rename from test/srt/models/test_nvidia_nemotron_nano_v2.py rename to test/registered/models/test_nvidia_nemotron_nano_v2.py index 656eef838..21a5a9aa8 100644 --- a/test/srt/models/test_nvidia_nemotron_nano_v2.py +++ b/test/registered/models/test_nvidia_nemotron_nano_v2.py @@ -1,9 +1,12 @@ import unittest from sglang.srt.utils import is_blackwell +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase +register_cuda_ci(est_time=132, suite="stage-b-test-large-2-gpu") + class TestNvidiaNemotronNanoV2BF16(GSM8KMixin, DefaultServerBase): model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2" diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index d07dcec37..7b73c4098 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -16,17 +16,6 @@ suites = { TestFile("test_video_utils.py", 5), TestFile("test_modelopt_export.py", 9), ], - "per-commit-2-gpu": [ - TestFile("hicache/test_hicache_storage_3fs_backend.py", 200), - TestFile("hicache/test_hicache_storage_file_backend.py", 200), - TestFile("hicache/test_hicache_storage_mooncake_backend.py", 300), - TestFile("models/test_kimi_linear_models.py", 90), - TestFile("models/test_nvidia_nemotron_nano_v2.py", 132), - TestFile("test_data_parallelism.py", 73), - TestFile("test_disaggregation_basic.py", 400), - TestFile("test_dp_attention.py", 350), - TestFile("test_load_weights_from_remote_instance.py", 72), - ], "per-commit-4-gpu": [ TestFile("models/test_qwen3_next_models.py", 650), TestFile("test_gpt_oss_4gpu.py", 300), @@ -116,11 +105,6 @@ suite_amd = { # TestFile("test_vision_chunked_prefill.py", 175), # Disabled temporarily and track in #7701 # TestFile("test_wave_attention_backend.py", 150), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127 ], - "per-commit-amd-mi35x": [], - "per-commit-2-gpu-amd": [ - TestFile("test_data_parallelism.py", 73), - TestFile("test_load_weights_from_remote_instance.py", 72), - ], "per-commit-4-gpu-amd": [ TestFile("test_pp_single_node.py", 150), ],