diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index dea2dfdab..568d179f0 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -191,7 +191,7 @@ jobs: timeout-minutes: 120 run: | cd test - python3 nightly/test_text_models_gsm8k_eval.py + python3 run_suite.py --hw cuda --suite nightly-eval-text-2-gpu --nightly --continue-on-error --timeout-per-file 4500 # Text model performance tests nightly-test-text-perf-2-gpu-runner: @@ -216,7 +216,7 @@ jobs: run: | cd test rm -rf performance_profiles_text_models/ - python3 nightly/test_text_models_perf.py + python3 run_suite.py --hw cuda --suite nightly-perf-text-2-gpu --nightly --continue-on-error - name: Publish traces to storage repo env: @@ -244,7 +244,7 @@ jobs: timeout-minutes: 240 run: | cd test - python3 nightly/test_vlms_mmmu_eval.py + python3 run_suite.py --hw cuda --suite nightly-eval-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 9000 # VLM performance tests nightly-test-vlm-perf-2-gpu-runner: @@ -269,7 +269,7 @@ jobs: run: | cd test rm -rf performance_profiles_vlms/ - python3 nightly/test_vlms_perf.py + python3 run_suite.py --hw cuda --suite nightly-perf-vlm-2-gpu --nightly --continue-on-error - name: Publish traces to storage repo env: diff --git a/test/nightly/accuracy_test_runner.py b/python/sglang/test/accuracy_test_runner.py similarity index 100% rename from test/nightly/accuracy_test_runner.py rename to python/sglang/test/accuracy_test_runner.py diff --git a/python/sglang/test/ascend/__init__.py b/python/sglang/test/ascend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/nightly/ascend/llm_models/gsm8k_ascend_mixin.py b/python/sglang/test/ascend/gsm8k_ascend_mixin.py similarity index 100% rename from test/nightly/ascend/llm_models/gsm8k_ascend_mixin.py rename to python/sglang/test/ascend/gsm8k_ascend_mixin.py diff --git a/test/nightly/ascend/vlm_models/test_vlm_utils.py b/python/sglang/test/ascend/vlm_utils.py similarity index 98% rename from test/nightly/ascend/vlm_models/test_vlm_utils.py rename to python/sglang/test/ascend/vlm_utils.py index 6c5eae64b..324030a34 100644 --- a/test/nightly/ascend/vlm_models/test_vlm_utils.py +++ b/python/sglang/test/ascend/vlm_utils.py @@ -87,7 +87,7 @@ class TestVLMModels(CustomTestCase): "--limit", limit, "--config", - "/__w/sglang/sglang/test/nightly/ascend/vlm_models/mmmu-val.yaml", + "/__w/sglang/sglang/test/registered/ascend/vlm_models/mmmu-val.yaml", ] subprocess.run( diff --git a/test/nightly/nightly_utils.py b/python/sglang/test/nightly_utils.py similarity index 100% rename from test/nightly/nightly_utils.py rename to python/sglang/test/nightly_utils.py diff --git a/test/nightly/performance_test_runner.py b/python/sglang/test/performance_test_runner.py similarity index 99% rename from test/nightly/performance_test_runner.py rename to python/sglang/test/performance_test_runner.py index 48fb452cb..0e76d539d 100644 --- a/test/nightly/performance_test_runner.py +++ b/python/sglang/test/performance_test_runner.py @@ -1,9 +1,8 @@ from dataclasses import dataclass, field from typing import List, Optional, Tuple -from nightly_utils import NightlyBenchmarkRunner - from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings diff --git a/test/nightly/run_combined_tests.py b/python/sglang/test/run_combined_tests.py similarity index 97% rename from test/nightly/run_combined_tests.py rename to python/sglang/test/run_combined_tests.py index 4e576753f..b80a77d44 100644 --- a/test/nightly/run_combined_tests.py +++ b/python/sglang/test/run_combined_tests.py @@ -1,18 +1,17 @@ from typing import List, Optional -from accuracy_test_runner import ( +from sglang.test.accuracy_test_runner import ( AccuracyTestParams, AccuracyTestResult, run_accuracy_test, write_accuracy_github_summary, ) -from nightly_utils import NightlyBenchmarkRunner -from performance_test_runner import ( +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.performance_test_runner import ( PerformanceTestParams, PerformanceTestResult, run_performance_test, ) - from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings, is_in_ci diff --git a/test/registered/8-gpu-models/test_deepseek_v31.py b/test/registered/8-gpu-models/test_deepseek_v31.py index a9816b11f..3dce28c2e 100644 --- a/test/registered/8-gpu-models/test_deepseek_v31.py +++ b/test/registered/8-gpu-models/test_deepseek_v31.py @@ -1,15 +1,9 @@ -import sys import unittest -from pathlib import Path - -# Add nightly directory to path for run_combined_tests import -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) - -from accuracy_test_runner import AccuracyTestParams -from performance_test_runner import PerformanceTestParams -from run_combined_tests import run_combined_tests +from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system # Runs on both H200 and B200 via nightly-8-gpu-common suite diff --git a/test/registered/8-gpu-models/test_deepseek_v32.py b/test/registered/8-gpu-models/test_deepseek_v32.py index e818939db..7fbef77bb 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32.py +++ b/test/registered/8-gpu-models/test_deepseek_v32.py @@ -1,15 +1,9 @@ -import sys import unittest -from pathlib import Path - -# Add nightly directory to path for run_combined_tests import -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) - -from accuracy_test_runner import AccuracyTestParams -from performance_test_runner import PerformanceTestParams -from run_combined_tests import run_combined_tests +from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system register_cuda_ci(est_time=8000, suite="nightly-8-gpu-common", nightly=True) diff --git a/test/registered/8-gpu-models/test_glm_46.py b/test/registered/8-gpu-models/test_glm_46.py index dbd04d17e..2b5481107 100644 --- a/test/registered/8-gpu-models/test_glm_46.py +++ b/test/registered/8-gpu-models/test_glm_46.py @@ -1,15 +1,9 @@ -import sys import unittest -from pathlib import Path - -# Add nightly directory to path for run_combined_tests import -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) - -from accuracy_test_runner import AccuracyTestParams -from performance_test_runner import PerformanceTestParams -from run_combined_tests import run_combined_tests +from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite diff --git a/test/registered/8-gpu-models/test_kimi_k2.py b/test/registered/8-gpu-models/test_kimi_k2.py index 5d3d9b05a..5f818a731 100644 --- a/test/registered/8-gpu-models/test_kimi_k2.py +++ b/test/registered/8-gpu-models/test_kimi_k2.py @@ -1,15 +1,9 @@ -import sys import unittest -from pathlib import Path - -# Add nightly directory to path for run_combined_tests import -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) - -from accuracy_test_runner import AccuracyTestParams -from performance_test_runner import PerformanceTestParams -from run_combined_tests import run_combined_tests +from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite diff --git a/test/registered/8-gpu-models/test_minimax_m2.py b/test/registered/8-gpu-models/test_minimax_m2.py index b83ceff4d..397990952 100644 --- a/test/registered/8-gpu-models/test_minimax_m2.py +++ b/test/registered/8-gpu-models/test_minimax_m2.py @@ -1,15 +1,9 @@ -import sys import unittest -from pathlib import Path - -# Add nightly directory to path for run_combined_tests import -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) - -from accuracy_test_runner import AccuracyTestParams -from performance_test_runner import PerformanceTestParams -from run_combined_tests import run_combined_tests +from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200 via nightly-8-gpu-common suite diff --git a/test/registered/8-gpu-models/test_mistral_large3.py b/test/registered/8-gpu-models/test_mistral_large3.py index aef06ba06..45c2c3c85 100644 --- a/test/registered/8-gpu-models/test_mistral_large3.py +++ b/test/registered/8-gpu-models/test_mistral_large3.py @@ -1,16 +1,10 @@ import os -import sys import unittest -from pathlib import Path - -# Add nightly directory to path for run_combined_tests import -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) - -from accuracy_test_runner import AccuracyTestParams -from performance_test_runner import PerformanceTestParams -from run_combined_tests import run_combined_tests +from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system # Runs on both H200 and B200 via nightly-8-gpu-common suite diff --git a/test/registered/8-gpu-models/test_qwen3_235b.py b/test/registered/8-gpu-models/test_qwen3_235b.py index 17f5ec965..4888494bc 100644 --- a/test/registered/8-gpu-models/test_qwen3_235b.py +++ b/test/registered/8-gpu-models/test_qwen3_235b.py @@ -1,15 +1,9 @@ -import sys import unittest -from pathlib import Path - -# Add nightly directory to path for run_combined_tests import -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly")) - -from accuracy_test_runner import AccuracyTestParams -from performance_test_runner import PerformanceTestParams -from run_combined_tests import run_combined_tests +from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.performance_test_runner import PerformanceTestParams +from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system # Runs on both H200 and B200 via nightly-8-gpu-common suite diff --git a/test/nightly/ascend/embedding_models/test_ascend_embedding_models.py b/test/registered/ascend/embedding_models/test_ascend_embedding_models.py similarity index 100% rename from test/nightly/ascend/embedding_models/test_ascend_embedding_models.py rename to test/registered/ascend/embedding_models/test_ascend_embedding_models.py diff --git a/test/nightly/ascend/llm_models/test_ascend_afm_4_5b.py b/test/registered/ascend/llm_models/test_ascend_afm_4_5b.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_afm_4_5b.py rename to test/registered/ascend/llm_models/test_ascend_afm_4_5b.py index 78dddbdd2..7d1437f32 100644 --- a/test/nightly/ascend/llm_models/test_ascend_afm_4_5b.py +++ b/test/registered/ascend/llm_models/test_ascend_afm_4_5b.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_baichuan2_13b_chat.py b/test/registered/ascend/llm_models/test_ascend_baichuan2_13b_chat.py similarity index 87% rename from test/nightly/ascend/llm_models/test_ascend_baichuan2_13b_chat.py rename to test/registered/ascend/llm_models/test_ascend_baichuan2_13b_chat.py index c02c08378..31aceaac2 100644 --- a/test/nightly/ascend/llm_models/test_ascend_baichuan2_13b_chat.py +++ b/test/registered/ascend/llm_models/test_ascend_baichuan2_13b_chat.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_c4ai_command_r_v01.py b/test/registered/ascend/llm_models/test_ascend_c4ai_command_r_v01.py similarity index 100% rename from test/nightly/ascend/llm_models/test_ascend_c4ai_command_r_v01.py rename to test/registered/ascend/llm_models/test_ascend_c4ai_command_r_v01.py diff --git a/test/nightly/ascend/llm_models/test_ascend_charglm2_6b.py b/test/registered/ascend/llm_models/test_ascend_charglm2_6b.py similarity index 90% rename from test/nightly/ascend/llm_models/test_ascend_charglm2_6b.py rename to test/registered/ascend/llm_models/test_ascend_charglm2_6b.py index 2a7760d6c..b598280a7 100644 --- a/test/nightly/ascend/llm_models/test_ascend_charglm2_6b.py +++ b/test/registered/ascend/llm_models/test_ascend_charglm2_6b.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_exaone_3.py b/test/registered/ascend/llm_models/test_ascend_exaone_3.py similarity index 90% rename from test/nightly/ascend/llm_models/test_ascend_exaone_3.py rename to test/registered/ascend/llm_models/test_ascend_exaone_3.py index 133ad709f..04541316f 100644 --- a/test/nightly/ascend/llm_models/test_ascend_exaone_3.py +++ b/test/registered/ascend/llm_models/test_ascend_exaone_3.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_gemma_3_1b_it.py b/test/registered/ascend/llm_models/test_ascend_gemma_3_1b_it.py similarity index 87% rename from test/nightly/ascend/llm_models/test_ascend_gemma_3_1b_it.py rename to test/registered/ascend/llm_models/test_ascend_gemma_3_1b_it.py index f4789090e..e018a890a 100644 --- a/test/nightly/ascend/llm_models/test_ascend_gemma_3_1b_it.py +++ b/test/registered/ascend/llm_models/test_ascend_gemma_3_1b_it.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_glm4_9b_chat.py b/test/registered/ascend/llm_models/test_ascend_glm4_9b_chat.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_glm4_9b_chat.py rename to test/registered/ascend/llm_models/test_ascend_glm4_9b_chat.py index d3f4c6f70..6e96fed7c 100644 --- a/test/nightly/ascend/llm_models/test_ascend_glm4_9b_chat.py +++ b/test/registered/ascend/llm_models/test_ascend_glm4_9b_chat.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_granite_3_0_3b_a800m.py b/test/registered/ascend/llm_models/test_ascend_granite_3_0_3b_a800m.py similarity index 86% rename from test/nightly/ascend/llm_models/test_ascend_granite_3_0_3b_a800m.py rename to test/registered/ascend/llm_models/test_ascend_granite_3_0_3b_a800m.py index 759691fc7..4ea622e35 100644 --- a/test/nightly/ascend/llm_models/test_ascend_granite_3_0_3b_a800m.py +++ b/test/registered/ascend/llm_models/test_ascend_granite_3_0_3b_a800m.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_granite_3_1_8b.py b/test/registered/ascend/llm_models/test_ascend_granite_3_1_8b.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_granite_3_1_8b.py rename to test/registered/ascend/llm_models/test_ascend_granite_3_1_8b.py index 3e487b83c..f5a5f0a84 100644 --- a/test/nightly/ascend/llm_models/test_ascend_granite_3_1_8b.py +++ b/test/registered/ascend/llm_models/test_ascend_granite_3_1_8b.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_internlm2_7b.py b/test/registered/ascend/llm_models/test_ascend_internlm2_7b.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_internlm2_7b.py rename to test/registered/ascend/llm_models/test_ascend_internlm2_7b.py index be1dca99f..16c2fdf60 100644 --- a/test/nightly/ascend/llm_models/test_ascend_internlm2_7b.py +++ b/test/registered/ascend/llm_models/test_ascend_internlm2_7b.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_ling_lite.py b/test/registered/ascend/llm_models/test_ascend_ling_lite.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_ling_lite.py rename to test/registered/ascend/llm_models/test_ascend_ling_lite.py index c2af52fc9..8264805d3 100644 --- a/test/nightly/ascend/llm_models/test_ascend_ling_lite.py +++ b/test/registered/ascend/llm_models/test_ascend_ling_lite.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_llama_2_7b.py b/test/registered/ascend/llm_models/test_ascend_llama_2_7b.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_llama_2_7b.py rename to test/registered/ascend/llm_models/test_ascend_llama_2_7b.py index 97fefada6..33a1369cd 100644 --- a/test/nightly/ascend/llm_models/test_ascend_llama_2_7b.py +++ b/test/registered/ascend/llm_models/test_ascend_llama_2_7b.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_mimo_7b_rl.py b/test/registered/ascend/llm_models/test_ascend_mimo_7b_rl.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_mimo_7b_rl.py rename to test/registered/ascend/llm_models/test_ascend_mimo_7b_rl.py index c025523cc..ea70abf1a 100644 --- a/test/nightly/ascend/llm_models/test_ascend_mimo_7b_rl.py +++ b/test/registered/ascend/llm_models/test_ascend_mimo_7b_rl.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_mistral_7b.py b/test/registered/ascend/llm_models/test_ascend_mistral_7b.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_mistral_7b.py rename to test/registered/ascend/llm_models/test_ascend_mistral_7b.py index 072f86474..51c194273 100644 --- a/test/nightly/ascend/llm_models/test_ascend_mistral_7b.py +++ b/test/registered/ascend/llm_models/test_ascend_mistral_7b.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_persimmon_8b_chat.py b/test/registered/ascend/llm_models/test_ascend_persimmon_8b_chat.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_persimmon_8b_chat.py rename to test/registered/ascend/llm_models/test_ascend_persimmon_8b_chat.py index 174f14dfa..695386009 100644 --- a/test/nightly/ascend/llm_models/test_ascend_persimmon_8b_chat.py +++ b/test/registered/ascend/llm_models/test_ascend_persimmon_8b_chat.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_phi_4_multimodal.py b/test/registered/ascend/llm_models/test_ascend_phi_4_multimodal.py similarity index 85% rename from test/nightly/ascend/llm_models/test_ascend_phi_4_multimodal.py rename to test/registered/ascend/llm_models/test_ascend_phi_4_multimodal.py index d4ebeca48..728b71a47 100644 --- a/test/nightly/ascend/llm_models/test_ascend_phi_4_multimodal.py +++ b/test/registered/ascend/llm_models/test_ascend_phi_4_multimodal.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/test_ascend_smollm_1_7b.py b/test/registered/ascend/llm_models/test_ascend_smollm_1_7b.py similarity index 90% rename from test/nightly/ascend/llm_models/test_ascend_smollm_1_7b.py rename to test/registered/ascend/llm_models/test_ascend_smollm_1_7b.py index 652934b4a..0ccbd1966 100644 --- a/test/nightly/ascend/llm_models/test_ascend_smollm_1_7b.py +++ b/test/registered/ascend/llm_models/test_ascend_smollm_1_7b.py @@ -1,7 +1,6 @@ import unittest -from gsm8k_ascend_mixin import GSM8KAscendMixin - +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin from sglang.test.ci.ci_register import register_npu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/nightly/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja b/test/registered/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja similarity index 100% rename from test/nightly/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja rename to test/registered/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja diff --git a/test/nightly/ascend/rerank_models/test_ascend_cross_encoder_models.py b/test/registered/ascend/rerank_models/test_ascend_cross_encoder_models.py similarity index 100% rename from test/nightly/ascend/rerank_models/test_ascend_cross_encoder_models.py rename to test/registered/ascend/rerank_models/test_ascend_cross_encoder_models.py diff --git a/test/nightly/ascend/vlm_models/mmmu-val.yaml b/test/registered/ascend/vlm_models/mmmu-val.yaml similarity index 100% rename from test/nightly/ascend/vlm_models/mmmu-val.yaml rename to test/registered/ascend/vlm_models/mmmu-val.yaml diff --git a/test/nightly/ascend/vlm_models/test_ascend_gemma_3_4b_it.py b/test/registered/ascend/vlm_models/test_ascend_gemma_3_4b_it.py similarity index 87% rename from test/nightly/ascend/vlm_models/test_ascend_gemma_3_4b_it.py rename to test/registered/ascend/vlm_models/test_ascend_gemma_3_4b_it.py index ee68f4b4d..8079157e0 100644 --- a/test/nightly/ascend/vlm_models/test_ascend_gemma_3_4b_it.py +++ b/test/registered/ascend/vlm_models/test_ascend_gemma_3_4b_it.py @@ -1,7 +1,6 @@ import unittest -from test_vlm_utils import TestVLMModels - +from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/nightly/ascend/vlm_models/test_ascend_janus_pro_1b.py b/test/registered/ascend/vlm_models/test_ascend_janus_pro_1b.py similarity index 88% rename from test/nightly/ascend/vlm_models/test_ascend_janus_pro_1b.py rename to test/registered/ascend/vlm_models/test_ascend_janus_pro_1b.py index da6c4d13d..f447c4b3a 100644 --- a/test/nightly/ascend/vlm_models/test_ascend_janus_pro_1b.py +++ b/test/registered/ascend/vlm_models/test_ascend_janus_pro_1b.py @@ -1,7 +1,6 @@ import unittest -from test_vlm_utils import TestVLMModels - +from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/nightly/ascend/vlm_models/test_ascend_janus_pro_7b.py b/test/registered/ascend/vlm_models/test_ascend_janus_pro_7b.py similarity index 87% rename from test/nightly/ascend/vlm_models/test_ascend_janus_pro_7b.py rename to test/registered/ascend/vlm_models/test_ascend_janus_pro_7b.py index 7e85b4f30..4c5249158 100644 --- a/test/nightly/ascend/vlm_models/test_ascend_janus_pro_7b.py +++ b/test/registered/ascend/vlm_models/test_ascend_janus_pro_7b.py @@ -1,7 +1,6 @@ import unittest -from test_vlm_utils import TestVLMModels - +from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/nightly/ascend/vlm_models/test_ascend_mimo_vl_7b_rl.py b/test/registered/ascend/vlm_models/test_ascend_mimo_vl_7b_rl.py similarity index 88% rename from test/nightly/ascend/vlm_models/test_ascend_mimo_vl_7b_rl.py rename to test/registered/ascend/vlm_models/test_ascend_mimo_vl_7b_rl.py index fccc5bb41..fc4f8164a 100644 --- a/test/nightly/ascend/vlm_models/test_ascend_mimo_vl_7b_rl.py +++ b/test/registered/ascend/vlm_models/test_ascend_mimo_vl_7b_rl.py @@ -1,7 +1,6 @@ import unittest -from test_vlm_utils import TestVLMModels - +from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/nightly/ascend/vlm_models/test_ascend_minicpm_o_2_6.py b/test/registered/ascend/vlm_models/test_ascend_minicpm_o_2_6.py similarity index 87% rename from test/nightly/ascend/vlm_models/test_ascend_minicpm_o_2_6.py rename to test/registered/ascend/vlm_models/test_ascend_minicpm_o_2_6.py index 048669280..96bda8e9f 100644 --- a/test/nightly/ascend/vlm_models/test_ascend_minicpm_o_2_6.py +++ b/test/registered/ascend/vlm_models/test_ascend_minicpm_o_2_6.py @@ -1,7 +1,6 @@ import unittest -from test_vlm_utils import TestVLMModels - +from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/nightly/ascend/vlm_models/test_ascend_minicpm_v_2_6.py b/test/registered/ascend/vlm_models/test_ascend_minicpm_v_2_6.py similarity index 87% rename from test/nightly/ascend/vlm_models/test_ascend_minicpm_v_2_6.py rename to test/registered/ascend/vlm_models/test_ascend_minicpm_v_2_6.py index 3ed6fb15f..e0beb81f1 100644 --- a/test/nightly/ascend/vlm_models/test_ascend_minicpm_v_2_6.py +++ b/test/registered/ascend/vlm_models/test_ascend_minicpm_v_2_6.py @@ -1,7 +1,6 @@ import unittest -from test_vlm_utils import TestVLMModels - +from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/nightly/ascend/vlm_models/test_ascend_phi4_multimodal_instruct.py b/test/registered/ascend/vlm_models/test_ascend_phi4_multimodal_instruct.py similarity index 88% rename from test/nightly/ascend/vlm_models/test_ascend_phi4_multimodal_instruct.py rename to test/registered/ascend/vlm_models/test_ascend_phi4_multimodal_instruct.py index 191b541a6..0ecb49304 100644 --- a/test/nightly/ascend/vlm_models/test_ascend_phi4_multimodal_instruct.py +++ b/test/registered/ascend/vlm_models/test_ascend_phi4_multimodal_instruct.py @@ -1,7 +1,6 @@ import unittest -from test_vlm_utils import TestVLMModels - +from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/nightly/ascend/vlm_models/test_ascend_qwen2_5_vl_3b_instruct.py b/test/registered/ascend/vlm_models/test_ascend_qwen2_5_vl_3b_instruct.py similarity index 88% rename from test/nightly/ascend/vlm_models/test_ascend_qwen2_5_vl_3b_instruct.py rename to test/registered/ascend/vlm_models/test_ascend_qwen2_5_vl_3b_instruct.py index dda18933b..933f4ba28 100644 --- a/test/nightly/ascend/vlm_models/test_ascend_qwen2_5_vl_3b_instruct.py +++ b/test/registered/ascend/vlm_models/test_ascend_qwen2_5_vl_3b_instruct.py @@ -1,7 +1,6 @@ import unittest -from test_vlm_utils import TestVLMModels - +from sglang.test.ascend.vlm_utils import TestVLMModels from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=400, suite="nightly-4-npu-a3", nightly=True) diff --git a/test/nightly/test_deepseek_r1_fp8_trtllm_backend.py b/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py similarity index 100% rename from test/nightly/test_deepseek_r1_fp8_trtllm_backend.py rename to test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py diff --git a/test/registered/backends/test_deepseek_v3_fp4_cutlass_moe.py b/test/registered/backends/test_deepseek_v3_fp4_cutlass_moe.py new file mode 100644 index 000000000..c3a509efa --- /dev/null +++ b/test/registered/backends/test_deepseek_v3_fp4_cutlass_moe.py @@ -0,0 +1,75 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_cuda_ci(est_time=900, suite="nightly-4-gpu-b200", nightly=True) + +FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" +SERVER_LAUNCH_TIMEOUT = 1000 + + +class TestDeepseekV3FP4CutlassMoE(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "4", + "--ep", + "4", + "--attention-backend", + "trtllm_mla", + "--moe-runner-backend", + "flashinfer_cutlass", + "--quantization", + "modelopt_fp4", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1319, + parallel=1319, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/nightly/test_flashinfer_trtllm_gen_attn_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py similarity index 100% rename from test/nightly/test_flashinfer_trtllm_gen_attn_backend.py rename to test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py diff --git a/test/nightly/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py similarity index 100% rename from test/nightly/test_flashinfer_trtllm_gen_moe_backend.py rename to test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py diff --git a/test/nightly/test_qwen3_fp4_trtllm_gen_moe.py b/test/registered/backends/test_qwen3_fp4_trtllm_gen_moe.py similarity index 83% rename from test/nightly/test_qwen3_fp4_trtllm_gen_moe.py rename to test/registered/backends/test_qwen3_fp4_trtllm_gen_moe.py index f6b562431..f215af49b 100644 --- a/test/nightly/test_qwen3_fp4_trtllm_gen_moe.py +++ b/test/registered/backends/test_qwen3_fp4_trtllm_gen_moe.py @@ -1,7 +1,8 @@ import unittest from types import SimpleNamespace -from sglang.srt.utils import kill_process_tree +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -10,7 +11,13 @@ from sglang.test.test_utils import ( popen_launch_server, ) +# modelopt_fp4 requires SM 100+ (Blackwell) +register_cuda_ci(est_time=300, suite="nightly-1-gpu", nightly=True) + +@unittest.skipIf( + get_device_sm() < 100, "Test requires CUDA SM 100 or higher (Blackwell)" +) class TestFlashinferTrtllmGenMoeBackend(CustomTestCase): @classmethod def setUpClass(cls): diff --git a/test/nightly/test_batch_invariant_ops.py b/test/registered/core/test_batch_invariant_ops.py similarity index 100% rename from test/nightly/test_batch_invariant_ops.py rename to test/registered/core/test_batch_invariant_ops.py diff --git a/test/nightly/test_cpp_radix_cache.py b/test/registered/core/test_cpp_radix_cache.py similarity index 100% rename from test/nightly/test_cpp_radix_cache.py rename to test/registered/core/test_cpp_radix_cache.py diff --git a/test/nightly/test_deepseek_v3_deterministic.py b/test/registered/core/test_deepseek_v3_deterministic.py similarity index 100% rename from test/nightly/test_deepseek_v3_deterministic.py rename to test/registered/core/test_deepseek_v3_deterministic.py diff --git a/test/nightly/test_qwen3_next_deterministic.py b/test/registered/core/test_qwen3_next_deterministic.py similarity index 100% rename from test/nightly/test_qwen3_next_deterministic.py rename to test/registered/core/test_qwen3_next_deterministic.py diff --git a/test/nightly/test_text_models_gsm8k_eval.py b/test/registered/eval/test_text_models_gsm8k_eval.py similarity index 97% rename from test/nightly/test_text_models_gsm8k_eval.py rename to test/registered/eval/test_text_models_gsm8k_eval.py index c0bf11cea..5fc1eedf8 100644 --- a/test/nightly/test_text_models_gsm8k_eval.py +++ b/test/registered/eval/test_text_models_gsm8k_eval.py @@ -4,6 +4,7 @@ import warnings from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, @@ -19,6 +20,8 @@ from sglang.test.test_utils import ( write_results_to_json, ) +register_cuda_ci(est_time=3600, suite="nightly-eval-text-2-gpu", nightly=True) + MODEL_SCORE_THRESHOLDS = { "meta-llama/Llama-3.1-8B-Instruct": 0.82, "mistralai/Mistral-7B-Instruct-v0.3": 0.58, diff --git a/test/nightly/test_vlms_mmmu_eval.py b/test/registered/eval/test_vlms_mmmu_eval.py similarity index 97% rename from test/nightly/test_vlms_mmmu_eval.py rename to test/registered/eval/test_vlms_mmmu_eval.py index 7f9b5aae4..a8a4a43d3 100644 --- a/test/nightly/test_vlms_mmmu_eval.py +++ b/test/registered/eval/test_vlms_mmmu_eval.py @@ -4,6 +4,7 @@ import warnings from types import SimpleNamespace from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -15,6 +16,8 @@ from sglang.test.test_utils import ( write_results_to_json, ) +register_cuda_ci(est_time=7200, suite="nightly-eval-vlm-2-gpu", nightly=True) + MODEL_THRESHOLDS = { # Conservative thresholds on 100 MMMU samples, especially for latency thresholds ModelLaunchSettings("deepseek-ai/deepseek-vl2-small"): ModelEvalMetrics( diff --git a/test/nightly/test_fp4_moe.py b/test/registered/kernels/test_fp4_moe.py similarity index 100% rename from test/nightly/test_fp4_moe.py rename to test/registered/kernels/test_fp4_moe.py diff --git a/test/nightly/test_nsa_indexer.py b/test/registered/kernels/test_nsa_indexer.py similarity index 100% rename from test/nightly/test_nsa_indexer.py rename to test/registered/kernels/test_nsa_indexer.py diff --git a/test/nightly/test_gpt_oss_4gpu_perf.py b/test/registered/perf/test_gpt_oss_4gpu_perf.py similarity index 92% rename from test/nightly/test_gpt_oss_4gpu_perf.py rename to test/registered/perf/test_gpt_oss_4gpu_perf.py index 34238c6eb..3cf201f9b 100644 --- a/test/nightly/test_gpt_oss_4gpu_perf.py +++ b/test/registered/perf/test_gpt_oss_4gpu_perf.py @@ -1,8 +1,7 @@ import unittest -from nightly_utils import NightlyBenchmarkRunner - from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.nightly_utils import NightlyBenchmarkRunner from sglang.test.test_utils import DEFAULT_URL_FOR_TEST register_cuda_ci(est_time=600, suite="nightly-4-gpu-b200", nightly=True) @@ -38,7 +37,7 @@ class TestNightlyGptOss4GpuPerformance(unittest.TestCase): for model_path, other_args in self.models: with self.subTest(model=model_path): - results, success = self.runner.run_benchmark_for_model( + results, success, _ = self.runner.run_benchmark_for_model( model_path=model_path, batch_sizes=self.batch_sizes, input_lens=self.input_lens, diff --git a/test/nightly/test_text_models_perf.py b/test/registered/perf/test_text_models_perf.py similarity index 89% rename from test/nightly/test_text_models_perf.py rename to test/registered/perf/test_text_models_perf.py index 1e2cf70ff..83689eef9 100644 --- a/test/nightly/test_text_models_perf.py +++ b/test/registered/perf/test_text_models_perf.py @@ -1,7 +1,7 @@ import unittest -from nightly_utils import NightlyBenchmarkRunner - +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.nightly_utils import NightlyBenchmarkRunner from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, ModelLaunchSettings, @@ -9,6 +9,8 @@ from sglang.test.test_utils import ( parse_models, ) +register_cuda_ci(est_time=3600, suite="nightly-perf-text-2-gpu", nightly=True) + PROFILE_DIR = "performance_profiles_text_models" @@ -37,7 +39,7 @@ class TestNightlyTextModelsPerformance(unittest.TestCase): for model_setup in self.models: with self.subTest(model=model_setup.model_path): - results, success = self.runner.run_benchmark_for_model( + results, success, _ = self.runner.run_benchmark_for_model( model_path=model_setup.model_path, batch_sizes=self.batch_sizes, input_lens=self.input_lens, diff --git a/test/nightly/test_vlms_perf.py b/test/registered/perf/test_vlms_perf.py similarity index 91% rename from test/nightly/test_vlms_perf.py rename to test/registered/perf/test_vlms_perf.py index b837c0262..d9c7db1ee 100644 --- a/test/nightly/test_vlms_perf.py +++ b/test/registered/perf/test_vlms_perf.py @@ -2,8 +2,8 @@ import os import unittest import warnings -from nightly_utils import NightlyBenchmarkRunner - +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.nightly_utils import NightlyBenchmarkRunner from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, ModelLaunchSettings, @@ -11,6 +11,8 @@ from sglang.test.test_utils import ( parse_models, ) +register_cuda_ci(est_time=7200, suite="nightly-perf-vlm-2-gpu", nightly=True) + PROFILE_DIR = "performance_profiles_vlms" MODEL_DEFAULTS = [ @@ -64,7 +66,7 @@ class TestNightlyVLMModelsPerformance(unittest.TestCase): "--dataset-name=mmmu", ] - results, success = self.runner.run_benchmark_for_model( + results, success, _ = self.runner.run_benchmark_for_model( model_path=model_setup.model_path, batch_sizes=self.batch_sizes, input_lens=self.input_lens, diff --git a/test/registered/models/deepseek/test_deepseek_v3_fp4_mtp_stage_b.py b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_stage_b.py similarity index 100% rename from test/registered/models/deepseek/test_deepseek_v3_fp4_mtp_stage_b.py rename to test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_stage_b.py diff --git a/test/nightly/test_eagle_infer_beta_dp_attention_large.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py similarity index 94% rename from test/nightly/test_eagle_infer_beta_dp_attention_large.py rename to test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py index 6bb02e38f..1d88db0e6 100644 --- a/test/nightly/test_eagle_infer_beta_dp_attention_large.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py @@ -5,6 +5,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST, @@ -16,6 +17,9 @@ from sglang.test.test_utils import ( write_github_step_summary, ) +# 16 GPU test (4 TP x 4 DP), runs on 2x 8-GPU B200 nodes +register_cuda_ci(est_time=600, suite="nightly-8-gpu-b200", nightly=True) + def test_gsm8k(base_url: str): requests.get(base_url + "/flush_cache") diff --git a/test/nightly/test_encoder_dp.py b/test/registered/vlm/test_encoder_dp.py similarity index 100% rename from test/nightly/test_encoder_dp.py rename to test/registered/vlm/test_encoder_dp.py diff --git a/test/run_suite.py b/test/run_suite.py index 87ed5aed9..636ca7777 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -43,6 +43,11 @@ NIGHTLY_SUITES = { "nightly-8-gpu-h200-basic", # Basic tests for large models on H200 "nightly-8-gpu-b200-basic", # Basic tests for large models on B200 "nightly-8-gpu-common", # Common tests that run on both H200 and B200 + # Eval and perf suites (2-gpu) + "nightly-eval-text-2-gpu", + "nightly-eval-vlm-2-gpu", + "nightly-perf-text-2-gpu", + "nightly-perf-vlm-2-gpu", ], HWBackend.AMD: ["nightly-amd", "nightly-amd-8-gpu"], HWBackend.CPU: [], @@ -149,14 +154,10 @@ def run_a_suite(args): auto_partition_id = args.auto_partition_id auto_partition_size = args.auto_partition_size - # Temporary: search broadly for nightly tests during migration to registered/ - if nightly: - files = glob.glob("**/*.py", recursive=True) - sanity_check = False # Allow files without registration during migration - else: - files = glob.glob("registered/**/*.py", recursive=True) - # Strict: all registered files must have proper registration - sanity_check = True + # All tests (per-commit and nightly) are now in registered/ + files = glob.glob("registered/**/*.py", recursive=True) + # Strict: all registered files must have proper registration + sanity_check = True all_tests = collect_tests(files, sanity_check=sanity_check) ci_tests, skipped_tests = filter_tests(all_tests, hw, suite, nightly)