diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml index 6e8e4824a..6a321925e 100644 --- a/.github/workflows/pr-test-rust.yml +++ b/.github/workflows/pr-test-rust.yml @@ -13,7 +13,7 @@ on: workflow_dispatch: concurrency: - group: pr-test-rust-${{ github.ref }} + group: router-tests-${{ github.ref }} cancel-in-progress: true env: @@ -70,7 +70,7 @@ jobs: python -c "import sglang_router; print('Python package: OK')" python -c "from sglang_router.sglang_router_rs import Router; print('Rust extension: OK')" python -m sglang_router.launch_router --help > /dev/null && echo "Entry point: OK" - unit-test-rust: + router-unit-tests: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-ci') runs-on: ubuntu-latest steps: @@ -133,7 +133,7 @@ jobs: if: always() run: sccache --show-stats - pytest-rust: + router-http-tests: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-ci') runs-on: 4-gpu-a10 timeout-minutes: 32 @@ -157,6 +157,16 @@ jobs: cache-all-crates: true cache-on-failure: true + - name: Cache router build + uses: actions/cache@v4 + with: + path: | + sgl-router/target + sgl-router/.venv + key: ${{ runner.os }}-router-${{ hashFiles('sgl-router/Cargo.lock', 'sgl-router/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-router- + - name: Install SGLang dependencies run: | sudo --preserve-env=PATH bash scripts/ci/ci_install_dependency.sh @@ -165,18 +175,17 @@ jobs: run: | source "$HOME/.cargo/env" export RUSTC_WRAPPER=sccache - pip install -e "sgl-router" cd sgl-router - pip install maturin - maturin build --release --out dist --features vendored-openssl - pip install --force-reinstall dist/*.whl + python3 -m venv .venv + python3 -m pip install --upgrade pip maturin + maturin develop --release --profile ci --features vendored-openssl - name: Run Python unit tests run: | cd sgl-router source "$HOME/.cargo/env" - pip install pytest pytest-cov pytest-xdist + python3 -m pip install pytest pytest-cov pytest-xdist pytest -q py_test/unit --cov=sglang_router --cov-report=term-missing --cov-fail-under=80 - name: Run Python integration tests @@ -184,16 +193,17 @@ jobs: cd sgl-router source "$HOME/.cargo/env" # Integration tests use FastAPI/uvicorn for mock workers - pip install fastapi uvicorn orjson - pytest -q -m integration + python3 -m pip install fastapi uvicorn orjson + pytest -q py_test/integration_mock - name: Run Python E2E tests run: | bash scripts/killall_sglang.sh "nuk_gpus" cd sgl-router + source "$HOME/.cargo/env" python3 -m pip --no-cache-dir install --upgrade --ignore-installed blinker - python3 -m pip --no-cache-dir install --upgrade --break-system-packages genai-bench==0.0.2 - pytest -m e2e -s -vv -o log_cli=true --log-cli-level=INFO + python3 -m pip --no-cache-dir install --upgrade genai-bench==0.0.2 + pytest py_test/e2e_http -s -vv -o log_cli=true --log-cli-level=INFO - name: Upload benchmark results if: success() @@ -202,7 +212,7 @@ jobs: name: genai-bench-results-all-policies path: sgl-router/benchmark_**/ - pytest-rust-2: + router-grpc-response-api-tests: if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-ci') runs-on: 4-gpu-a10 timeout-minutes: 32 @@ -226,6 +236,16 @@ jobs: cache-all-crates: true cache-on-failure: true + - name: Cache router build + uses: actions/cache@v4 + with: + path: | + sgl-router/target + sgl-router/.venv + key: ${{ runner.os }}-router-${{ hashFiles('sgl-router/Cargo.lock', 'sgl-router/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-router- + - name: Install SGLang dependencies run: | sudo --preserve-env=PATH bash scripts/ci/ci_install_dependency.sh @@ -264,20 +284,22 @@ jobs: source "$HOME/.cargo/env" export RUSTC_WRAPPER=sccache cd sgl-router - pip install maturin - maturin build --release --out dist --features vendored-openssl - pip install --force-reinstall dist/*.whl + python3 -m venv .venv + python3 -m pip install --upgrade pip maturin + maturin develop --release --profile ci --features vendored-openssl - name: Run Python E2E response API tests run: | bash scripts/killall_sglang.sh "nuk_gpus" cd sgl-router + source "$HOME/.cargo/env" SHOW_ROUTER_LOGS=1 pytest py_test/e2e_response_api -s -vv -o log_cli=true --log-cli-level=INFO - name: Run Python E2E gRPC tests run: | bash scripts/killall_sglang.sh "nuk_gpus" cd sgl-router + source "$HOME/.cargo/env" SHOW_ROUTER_LOGS=1 ROUTER_LOCAL_MODEL_PATH="/home/ubuntu/models" pytest py_test/e2e_grpc -s -vv -o log_cli=true --log-cli-level=INFO - name: Cleanup Oracle Database @@ -288,14 +310,14 @@ jobs: finish: - needs: [maturin-build-test, unit-test-rust, pytest-rust, pytest-rust-2] + needs: [maturin-build-test, router-unit-tests, router-http-tests, router-grpc-response-api-tests] runs-on: ubuntu-latest steps: - name: Finish run: echo "This is an empty step to ensure that all jobs are completed." summarize-benchmarks: - needs: pytest-rust + needs: router-http-tests runs-on: ubuntu-latest if: success() diff --git a/sgl-router/.coveragerc b/sgl-router/.coveragerc index 5bab1e8d2..f908584fe 100644 --- a/sgl-router/.coveragerc +++ b/sgl-router/.coveragerc @@ -1,9 +1,9 @@ [run] source = py_src/sglang_router omit = - py_src/sglang_router/mini_lb.py + */sglang_router/mini_lb.py [report] fail_under = 80 omit = - py_src/sglang_router/mini_lb.py + */sglang_router/mini_lb.py diff --git a/sgl-router/Cargo.toml b/sgl-router/Cargo.toml index 25482ae27..a8a1b4bde 100644 --- a/sgl-router/Cargo.toml +++ b/sgl-router/Cargo.toml @@ -129,6 +129,13 @@ lto = "fat" # Full LTO for smaller binaries codegen-units = 1 # Better optimization, slower compile strip = true # Strip debug symbols +[profile.ci] +inherits = "release" +opt-level = 2 # Lighter optimization (still fast runtime, much faster compile) +lto = "thin" # Thin LTO - good balance +codegen-units = 16 # More parallelization for faster builds +strip = true + [profile.dev] opt-level = 0 debug = 1 diff --git a/sgl-router/README.md b/sgl-router/README.md index 8bac66e7c..b491f0de7 100644 --- a/sgl-router/README.md +++ b/sgl-router/README.md @@ -43,7 +43,6 @@ High-performance model routing control and data plane for large-scale LLM deploy - Additional guides, API references, and deployment patterns are continuously updated alongside SGLang releases. ## Installation - ### Prerequisites - **Rust and Cargo** ```bash diff --git a/sgl-router/py_test/e2e/conftest.py b/sgl-router/py_test/e2e_http/conftest.py similarity index 100% rename from sgl-router/py_test/e2e/conftest.py rename to sgl-router/py_test/e2e_http/conftest.py diff --git a/sgl-router/py_test/e2e/test_e2e_embeddings.py b/sgl-router/py_test/e2e_http/test_e2e_embeddings.py similarity index 100% rename from sgl-router/py_test/e2e/test_e2e_embeddings.py rename to sgl-router/py_test/e2e_http/test_e2e_embeddings.py diff --git a/sgl-router/py_test/e2e/test_pd_router.py b/sgl-router/py_test/e2e_http/test_pd_router.py similarity index 100% rename from sgl-router/py_test/e2e/test_pd_router.py rename to sgl-router/py_test/e2e_http/test_pd_router.py diff --git a/sgl-router/py_test/e2e/test_regular_router.py b/sgl-router/py_test/e2e_http/test_regular_router.py similarity index 100% rename from sgl-router/py_test/e2e/test_regular_router.py rename to sgl-router/py_test/e2e_http/test_regular_router.py diff --git a/sgl-router/py_test/integration/__init__.py b/sgl-router/py_test/integration_mock/__init__.py similarity index 100% rename from sgl-router/py_test/integration/__init__.py rename to sgl-router/py_test/integration_mock/__init__.py diff --git a/sgl-router/py_test/integration/conftest.py b/sgl-router/py_test/integration_mock/conftest.py similarity index 100% rename from sgl-router/py_test/integration/conftest.py rename to sgl-router/py_test/integration_mock/conftest.py diff --git a/sgl-router/py_test/integration/load_balancing/__init__.py b/sgl-router/py_test/integration_mock/load_balancing/__init__.py similarity index 100% rename from sgl-router/py_test/integration/load_balancing/__init__.py rename to sgl-router/py_test/integration_mock/load_balancing/__init__.py diff --git a/sgl-router/py_test/integration/load_balancing/test_cache_aware.py b/sgl-router/py_test/integration_mock/load_balancing/test_cache_aware.py similarity index 100% rename from sgl-router/py_test/integration/load_balancing/test_cache_aware.py rename to sgl-router/py_test/integration_mock/load_balancing/test_cache_aware.py diff --git a/sgl-router/py_test/integration/load_balancing/test_power_of_two.py b/sgl-router/py_test/integration_mock/load_balancing/test_power_of_two.py similarity index 100% rename from sgl-router/py_test/integration/load_balancing/test_power_of_two.py rename to sgl-router/py_test/integration_mock/load_balancing/test_power_of_two.py diff --git a/sgl-router/py_test/integration/load_balancing/test_random.py b/sgl-router/py_test/integration_mock/load_balancing/test_random.py similarity index 100% rename from sgl-router/py_test/integration/load_balancing/test_random.py rename to sgl-router/py_test/integration_mock/load_balancing/test_random.py diff --git a/sgl-router/py_test/integration/load_balancing/test_round_robin.py b/sgl-router/py_test/integration_mock/load_balancing/test_round_robin.py similarity index 100% rename from sgl-router/py_test/integration/load_balancing/test_round_robin.py rename to sgl-router/py_test/integration_mock/load_balancing/test_round_robin.py diff --git a/sgl-router/py_test/integration/test_api_auth.py b/sgl-router/py_test/integration_mock/test_api_auth.py similarity index 100% rename from sgl-router/py_test/integration/test_api_auth.py rename to sgl-router/py_test/integration_mock/test_api_auth.py diff --git a/sgl-router/py_test/integration/test_circuit_breaker.py b/sgl-router/py_test/integration_mock/test_circuit_breaker.py similarity index 100% rename from sgl-router/py_test/integration/test_circuit_breaker.py rename to sgl-router/py_test/integration_mock/test_circuit_breaker.py diff --git a/sgl-router/py_test/integration/test_fault_tolerance.py b/sgl-router/py_test/integration_mock/test_fault_tolerance.py similarity index 100% rename from sgl-router/py_test/integration/test_fault_tolerance.py rename to sgl-router/py_test/integration_mock/test_fault_tolerance.py diff --git a/sgl-router/py_test/integration/test_mtls.py b/sgl-router/py_test/integration_mock/test_mtls.py similarity index 100% rename from sgl-router/py_test/integration/test_mtls.py rename to sgl-router/py_test/integration_mock/test_mtls.py diff --git a/sgl-router/py_test/integration/test_payload_size.py b/sgl-router/py_test/integration_mock/test_payload_size.py similarity index 100% rename from sgl-router/py_test/integration/test_payload_size.py rename to sgl-router/py_test/integration_mock/test_payload_size.py diff --git a/sgl-router/py_test/integration/test_pd_routing.py b/sgl-router/py_test/integration_mock/test_pd_routing.py similarity index 100% rename from sgl-router/py_test/integration/test_pd_routing.py rename to sgl-router/py_test/integration_mock/test_pd_routing.py diff --git a/sgl-router/py_test/integration/test_rate_limiting.py b/sgl-router/py_test/integration_mock/test_rate_limiting.py similarity index 100% rename from sgl-router/py_test/integration/test_rate_limiting.py rename to sgl-router/py_test/integration_mock/test_rate_limiting.py diff --git a/sgl-router/py_test/integration/test_retries.py b/sgl-router/py_test/integration_mock/test_retries.py similarity index 100% rename from sgl-router/py_test/integration/test_retries.py rename to sgl-router/py_test/integration_mock/test_retries.py diff --git a/sgl-router/py_test/integration/test_service_discovery_shim.py b/sgl-router/py_test/integration_mock/test_service_discovery_shim.py similarity index 100% rename from sgl-router/py_test/integration/test_service_discovery_shim.py rename to sgl-router/py_test/integration_mock/test_service_discovery_shim.py diff --git a/sgl-router/py_test/integration/test_worker_management.py b/sgl-router/py_test/integration_mock/test_worker_management.py similarity index 100% rename from sgl-router/py_test/integration/test_worker_management.py rename to sgl-router/py_test/integration_mock/test_worker_management.py