From 84a9d0eab2a65d3d89bc5877842844fe21196b6b Mon Sep 17 00:00:00 2001 From: Simo Lin Date: Tue, 28 Oct 2025 12:18:51 -0700 Subject: [PATCH] [router] support arm, windows, mac, linux, reduce wheel size and number (#12285) --- .github/workflows/nightly-release-router.yml | 117 +++++++++++++------ .github/workflows/release-pypi-router.yml | 111 +++++++++++++----- sgl-router/Cargo.toml | 9 +- 3 files changed, 168 insertions(+), 69 deletions(-) diff --git a/.github/workflows/nightly-release-router.yml b/.github/workflows/nightly-release-router.yml index 00fd02cc8..7b16c2127 100644 --- a/.github/workflows/nightly-release-router.yml +++ b/.github/workflows/nightly-release-router.yml @@ -10,14 +10,40 @@ on: jobs: build: - name: Build on ${{ matrix.os }} (${{ matrix.target }}) + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }}) runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false matrix: + os: [ubuntu, macos, windows] + target: [x86_64, aarch64] + manylinux: [auto] include: - os: ubuntu + platform: linux + - os: windows + ls: dir target: x86_64 + python-architecture: x64 + interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 + - os: macos + target: aarch64 + interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 + - os: ubuntu + platform: linux + target: aarch64 + # musllinux + - os: ubuntu + platform: linux + target: x86_64 + manylinux: musllinux_1_1 + - os: ubuntu + platform: linux + target: aarch64 + manylinux: musllinux_1_1 + exclude: + - os: windows + target: aarch64 steps: - uses: actions/checkout@v4 @@ -29,56 +55,79 @@ jobs: mv sglang-repo/sgl-router/* . rm -rf sglang-repo ls -alt + shell: bash - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.13" + architecture: ${{ matrix.python-architecture || 'x64' }} - name: Modify version for nightly release run: | # Get current version from pyproject.toml CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])") - # Create nightly version with date: e.g., 0.1.9.dev20250112 + # Create nightly version with date: e.g., 0.2.1.dev20250128 NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)" echo "Nightly version: $NIGHTLY_VERSION" # Update pyproject.toml with nightly version (temporary, not committed) - sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml + sed -i.bak "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml # Verify the change cat pyproject.toml | grep "^version" + shell: bash - - name: Install build dependencies - run: | - python -m pip install -U pip - python -m pip install build twine auditwheel tomli + - name: Install twine and tomli + run: pip install -U twine tomli - - name: Build package - uses: pypa/cibuildwheel@v2.21.3 - env: - CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64" - CIBW_BEFORE_ALL: | - yum update -y && yum install -y openssl-devel wget unzip && \ - # Install latest protoc (v32.0) that supports proto3 - cd /tmp && \ - wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ - unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ - rm protoc-32.0-linux-x86_64.zip && \ - # Install Rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH" + - name: Install protoc (macOS) + if: matrix.os == 'macos' + run: brew install protobuf + + - name: Install protoc (Windows) + if: matrix.os == 'windows' + run: choco install protoc -y + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux || 'auto' }} + args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }} + rust-toolchain: stable + docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ + before-script-linux: | + # Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC) + if command -v yum &> /dev/null; then + yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make + # Install cross-compilation toolchain for aarch64 if needed + if [ "${{ matrix.target }}" = "aarch64" ]; then + yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true + fi + elif command -v apt-get &> /dev/null; then + apt-get update && apt-get install -y wget unzip gcc g++ perl make + # Install cross-compilation toolchain for aarch64 if needed + if [ "${{ matrix.target }}" = "aarch64" ]; then + apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true + fi + fi + (cd /tmp && \ + wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ + unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ + rm protoc-32.0-linux-x86_64.zip) + protoc --version - name: List built packages - run: ls -lh wheelhouse/ + run: ${{ matrix.ls || 'ls -lh' }} dist/ - name: Check packages - run: twine check --strict wheelhouse/* + run: twine check --strict dist/* - uses: actions/upload-artifact@v4 with: - name: packages-${{ matrix.os }}-${{ matrix.target }} - path: wheelhouse/ + name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }} + path: dist/ build-sdist: name: Build SDist @@ -98,28 +147,28 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.13" - name: Modify version for nightly release run: | # Get current version from pyproject.toml CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])") - # Create nightly version with date: e.g., 0.1.9.dev20250112 + # Create nightly version with date: e.g., 0.2.1.dev20250128 NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)" echo "Nightly version: $NIGHTLY_VERSION" # Update pyproject.toml with nightly version (temporary, not committed) - sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${CURRENT_VERSION}\"/g" pyproject.toml - sed -i "0,/version = \"${CURRENT_VERSION}\"/s//version = \"${NIGHTLY_VERSION}\"/" pyproject.toml + sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml # Verify the change cat pyproject.toml | grep "^version" - name: Build SDist - run: | - pip install build - python -m pip install -U packaging tomli - python -m build --sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + rust-toolchain: stable - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/release-pypi-router.yml b/.github/workflows/release-pypi-router.yml index 6447d09f5..71122499e 100644 --- a/.github/workflows/release-pypi-router.yml +++ b/.github/workflows/release-pypi-router.yml @@ -1,6 +1,4 @@ -# Reference: https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/.github/workflows/build_wheels.yml#L1 - -name: Release SGLang Router to PyPI +name: Release SGLang Model Gateway to PyPI on: push: @@ -12,14 +10,40 @@ on: jobs: build: - name: Build on ${{ matrix.os }} (${{ matrix.target }}) + name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }}) runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false matrix: + os: [ubuntu, macos, windows] + target: [x86_64, aarch64] + manylinux: [auto] include: - os: ubuntu + platform: linux + - os: windows + ls: dir target: x86_64 + python-architecture: x64 + interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 + - os: macos + target: aarch64 + interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 + - os: ubuntu + platform: linux + target: aarch64 + # musllinux + - os: ubuntu + platform: linux + target: x86_64 + manylinux: musllinux_1_1 + - os: ubuntu + platform: linux + target: aarch64 + manylinux: musllinux_1_1 + exclude: + - os: windows + target: aarch64 steps: - uses: actions/checkout@v4 @@ -31,42 +55,64 @@ jobs: mv sglang-repo/sgl-router/* . rm -rf sglang-repo ls -alt + shell: bash - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.13" + architecture: ${{ matrix.python-architecture || 'x64' }} - - name: Install build dependencies - run: | - python -m pip install -U pip - python -m pip install build twine auditwheel + - name: Install twine + run: pip install -U twine - - name: Build package - uses: pypa/cibuildwheel@v2.21.3 - env: - CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64" - CIBW_BEFORE_ALL: | - yum update -y && yum install -y openssl-devel wget unzip && \ - # Install latest protoc (v32.0) that supports proto3 - cd /tmp && \ - wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ - unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ - rm protoc-32.0-linux-x86_64.zip && \ - # Install Rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH" + - name: Install protoc (macOS) + if: matrix.os == 'macos' + run: brew install protobuf + + - name: Install protoc (Windows) + if: matrix.os == 'windows' + run: choco install protoc -y + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux || 'auto' }} + args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }} + rust-toolchain: stable + docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ + before-script-linux: | + # Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC) + if command -v yum &> /dev/null; then + yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make + # Install cross-compilation toolchain for aarch64 if needed + if [ "${{ matrix.target }}" = "aarch64" ]; then + yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true + fi + elif command -v apt-get &> /dev/null; then + apt-get update && apt-get install -y wget unzip gcc g++ perl make + # Install cross-compilation toolchain for aarch64 if needed + if [ "${{ matrix.target }}" = "aarch64" ]; then + apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true + fi + fi + (cd /tmp && \ + wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ + unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ + rm protoc-32.0-linux-x86_64.zip) + protoc --version - name: List built packages - run: ls -lh wheelhouse/ + run: ${{ matrix.ls || 'ls -lh' }} dist/ - name: Check packages - run: twine check --strict wheelhouse/* + run: twine check --strict dist/* - uses: actions/upload-artifact@v4 with: - name: packages-${{ matrix.os }}-${{ matrix.target }} - path: wheelhouse/ + name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }} + path: dist/ build-sdist: name: Build SDist @@ -86,13 +132,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.13" - name: Build SDist - run: | - pip install build - python -m pip install -U packaging - python -m build --sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + rust-toolchain: stable - uses: actions/upload-artifact@v4 with: diff --git a/sgl-router/Cargo.toml b/sgl-router/Cargo.toml index b7b965f42..c863f4a9c 100644 --- a/sgl-router/Cargo.toml +++ b/sgl-router/Cargo.toml @@ -36,7 +36,7 @@ rand = "0.9.2" reqwest = { version = "0.12.8", features = ["stream", "blocking", "json", "rustls-tls"], default-features = false } futures-util = "0.3" futures = "0.3" -pyo3 = { version = "0.27.1", features = ["extension-module"] } +pyo3 = { version = "0.27.1", features = ["extension-module", "abi3-py38"] } dashmap = "6.1.0" blake3 = "1.5" http = "1.1.0" @@ -66,6 +66,7 @@ tiktoken-rs = { version = "0.7.0" } minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins"] } minijinja-contrib = { version = "2.0", features = ["pycompat"] } rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +openssl = { version = "0.10.73", features = ["vendored"] } hf-hub = { version = "0.4.3", features = ["tokio"] } rmcp = { version = "0.8.3", features = ["client", "server", "transport-child-process", @@ -119,8 +120,10 @@ harness = false path = "benches/tool_parser_benchmark.rs" [profile.release] -lto = "thin" -codegen-units = 1 +opt-level = "z" # Optimize for size +lto = "fat" # Full LTO for smaller binaries +codegen-units = 1 # Better optimization, slower compile +strip = true # Strip debug symbols [profile.dev] opt-level = 0