feature: adding nightly wheel workflow and indexer (#14924)
This commit is contained in:
Douglas Yang
160
.github/workflows/release-pypi-nightly.yml
vendored
Normal file
160
.github/workflows/release-pypi-nightly.yml
vendored
Normal file
@@ -0,0 +1,160 @@
|
||||
name: Release PyPI Nightly Wheels
|
||||
|
||||
on:
|
||||
# Triggered by nightly Docker workflow to use same commit
|
||||
repository_dispatch:
|
||||
types: [nightly-release]
|
||||
# Manual trigger for testing
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
commit_sha:
|
||||
description: 'Specific commit SHA to build (leave empty for latest)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: release-pypi-nightly-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-nightly-wheel:
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
nightly_version: ${{ steps.gen_version.outputs.nightly_version }}
|
||||
commit_hash: ${{ steps.gen_version.outputs.commit_hash }}
|
||||
build_date: ${{ steps.gen_version.outputs.build_date }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Use commit from: 1) Docker workflow, 2) manual input, 3) latest main
|
||||
ref: ${{ github.event.client_payload.commit_sha || inputs.commit_sha || github.sha }}
|
||||
fetch-depth: 0 # Need full history for version generation
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Generate nightly version
|
||||
id: gen_version
|
||||
run: |
|
||||
# Get base version from version.py
|
||||
BASE_VERSION=$(cat python/sglang/version.py | cut -d'"' -f2)
|
||||
|
||||
# Get commit info
|
||||
COMMIT_HASH=$(git rev-parse --short HEAD)
|
||||
COMMIT_COUNT=$(git rev-list --count HEAD)
|
||||
|
||||
# Get current date in YYYY-MM-DD format
|
||||
BUILD_DATE=$(date -u +%Y-%m-%d)
|
||||
|
||||
# Generate nightly version following PEP 440
|
||||
# Format: {base_version}.dev{commit_count}+g{commit_hash}
|
||||
NIGHTLY_VERSION="${BASE_VERSION}.dev${COMMIT_COUNT}+g${COMMIT_HASH}"
|
||||
|
||||
echo "Base version: ${BASE_VERSION}"
|
||||
echo "Nightly version: ${NIGHTLY_VERSION}"
|
||||
echo "Commit: ${COMMIT_HASH}"
|
||||
echo "Build date: ${BUILD_DATE}"
|
||||
|
||||
echo "nightly_version=${NIGHTLY_VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT
|
||||
echo "base_version=${BASE_VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "build_date=${BUILD_DATE}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Update pyproject.toml with nightly version
|
||||
run: |
|
||||
cd python
|
||||
BASE_VERSION=$(cat sglang/version.py | cut -d'"' -f2)
|
||||
NIGHTLY_VERSION="${{ steps.gen_version.outputs.nightly_version }}"
|
||||
|
||||
# Update version (temporary, not committed)
|
||||
sed -i "s/version = \"${BASE_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
|
||||
|
||||
# Verify update
|
||||
echo "Updated version in pyproject.toml:"
|
||||
grep "^version" pyproject.toml
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
cd python
|
||||
pip install build wheel setuptools
|
||||
|
||||
- name: Build wheel
|
||||
run: |
|
||||
cd python
|
||||
cp ../README.md ../LICENSE .
|
||||
python3 -m build --wheel
|
||||
|
||||
# List built wheels
|
||||
echo "Built wheel:"
|
||||
ls -lh dist/
|
||||
|
||||
- name: Upload wheel artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: nightly-wheel
|
||||
path: python/dist/*.whl
|
||||
retention-days: 7
|
||||
|
||||
release-nightly:
|
||||
needs: build-nightly-wheel
|
||||
runs-on: ubuntu-latest
|
||||
environment: 'prod'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Download wheel artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: nightly-wheel
|
||||
path: dist/
|
||||
|
||||
- name: List downloaded wheels
|
||||
run: |
|
||||
echo "Downloaded wheel:"
|
||||
ls -lh dist/
|
||||
|
||||
- name: Create GitHub Release for nightly wheel
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: nightly-${{ needs.build-nightly-wheel.outputs.build_date }}-${{ needs.build-nightly-wheel.outputs.commit_hash }}
|
||||
name: Nightly Build ${{ needs.build-nightly-wheel.outputs.build_date }} (${{ needs.build-nightly-wheel.outputs.commit_hash }})
|
||||
repository: sgl-project/whl
|
||||
token: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
|
||||
prerelease: true
|
||||
body: |
|
||||
Nightly build from commit ${{ github.sha }}
|
||||
Build date: ${{ needs.build-nightly-wheel.outputs.build_date }}
|
||||
Version: ${{ needs.build-nightly-wheel.outputs.nightly_version }}
|
||||
files: |
|
||||
dist/*.whl
|
||||
|
||||
- name: Clone wheel index repository
|
||||
run: |
|
||||
git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl
|
||||
cd sgl-whl
|
||||
git config --local user.name "sglang-bot"
|
||||
git config --local user.email "sglangbot@gmail.com"
|
||||
env:
|
||||
WHL_TOKEN: ${{ secrets.GH_PAT_FOR_WHL_RELEASE }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Update wheel index
|
||||
run: |
|
||||
python3 scripts/update_nightly_whl_index.py \
|
||||
--commit-hash ${{ needs.build-nightly-wheel.outputs.commit_hash }} \
|
||||
--nightly-version ${{ needs.build-nightly-wheel.outputs.nightly_version }} \
|
||||
--build-date ${{ needs.build-nightly-wheel.outputs.build_date }}
|
||||
|
||||
- name: Push wheel index
|
||||
run: |
|
||||
cd sgl-whl
|
||||
git add -A
|
||||
git diff --staged --quiet || git commit -m "Update nightly wheel index for commit ${{ needs.build-nightly-wheel.outputs.commit_hash }}"
|
||||
git push
|
||||
170
scripts/update_nightly_whl_index.py
Executable file
170
scripts/update_nightly_whl_index.py
Executable file
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Update the wheel index for nightly SGLang releases.
|
||||
|
||||
This script generates a single PyPI-compatible index.html file at nightly/index.html
|
||||
containing all historical nightly builds, ordered by commit count (newest first).
|
||||
|
||||
Reference: https://github.com/flashinfer-ai/flashinfer/blob/v0.2.0/scripts/update_whl_index.py
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import pathlib
|
||||
import re
|
||||
|
||||
|
||||
def compute_sha256(file_path: pathlib.Path) -> str:
|
||||
"""Compute SHA256 hash of a file."""
|
||||
sha256_hash = hashlib.sha256()
|
||||
with open(file_path, "rb") as f:
|
||||
for byte_block in iter(lambda: f.read(4096), b""):
|
||||
sha256_hash.update(byte_block)
|
||||
return sha256_hash.hexdigest()
|
||||
|
||||
|
||||
def update_wheel_index(commit_hash: str, nightly_version: str, build_date: str = None):
|
||||
"""Update the wheel index for nightly releases.
|
||||
|
||||
Creates a single index at nightly/index.html containing all historical nightlies.
|
||||
|
||||
Args:
|
||||
commit_hash: Short git commit hash (e.g., 'c5f1e86')
|
||||
nightly_version: Full nightly version string (e.g., '0.5.6.post1.dev7716+gc5f1e86')
|
||||
build_date: Build date in YYYY-MM-DD format (e.g., '2025-12-13')
|
||||
"""
|
||||
dist_dir = pathlib.Path("dist")
|
||||
whl_repo_dir = pathlib.Path("sgl-whl")
|
||||
|
||||
if not dist_dir.exists():
|
||||
print(f"Warning: {dist_dir} does not exist, skipping index update")
|
||||
return
|
||||
|
||||
# Base URL for wheels stored in GitHub Releases
|
||||
base_url = "https://github.com/sgl-project/whl/releases/download"
|
||||
# Use date-based tag if build_date is provided, otherwise fall back to commit-only
|
||||
if build_date:
|
||||
release_tag = f"nightly-{build_date}-{commit_hash}"
|
||||
else:
|
||||
release_tag = f"nightly-{commit_hash}"
|
||||
|
||||
# Create nightly directory structure following PEP 503
|
||||
# /nightly/index.html -> links to sglang/
|
||||
# /nightly/sglang/index.html -> contains wheel links
|
||||
nightly_dir = whl_repo_dir / "nightly"
|
||||
nightly_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
sglang_dir = nightly_dir / "sglang"
|
||||
sglang_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
root_index = nightly_dir / "index.html"
|
||||
package_index = sglang_dir / "index.html"
|
||||
|
||||
print(f"\nUpdating nightly wheel index")
|
||||
print(f" Root index: {root_index}")
|
||||
print(f" Package index: {package_index}")
|
||||
|
||||
# Read existing package index if it exists
|
||||
existing_links = []
|
||||
if package_index.exists():
|
||||
with open(package_index, "r") as f:
|
||||
content = f.read()
|
||||
# Extract existing links (skip header and HTML boilerplate)
|
||||
existing_links = [
|
||||
line for line in content.split("\n") if line.startswith("<a href=")
|
||||
]
|
||||
|
||||
# Generate new links for current wheels
|
||||
new_links = []
|
||||
for wheel_path in sorted(dist_dir.glob("*.whl")):
|
||||
try:
|
||||
filename = wheel_path.name
|
||||
sha256 = compute_sha256(wheel_path)
|
||||
|
||||
# URL format: {base_url}/{release_tag}/{filename}#sha256={hash}
|
||||
wheel_url = f"{base_url}/{release_tag}/{filename}#sha256={sha256}"
|
||||
link = f'<a href="{wheel_url}">{filename}</a><br>'
|
||||
|
||||
new_links.append(link)
|
||||
print(f" Added: {filename}")
|
||||
except Exception as e:
|
||||
print(f" Error processing {wheel_path.name}: {e}")
|
||||
continue
|
||||
|
||||
if not new_links:
|
||||
print(" No new wheels to add")
|
||||
return
|
||||
|
||||
# Combine existing and new links (new links first for latest)
|
||||
all_links = new_links + existing_links
|
||||
|
||||
# Remove duplicates while preserving order (newer first)
|
||||
seen = set()
|
||||
unique_links = []
|
||||
for link in all_links:
|
||||
# Extract filename from link to check for duplicates
|
||||
filename_match = re.search(r">([^<]+\.whl)</a>", link)
|
||||
if filename_match:
|
||||
filename = filename_match.group(1)
|
||||
if filename not in seen:
|
||||
seen.add(filename)
|
||||
unique_links.append(link)
|
||||
|
||||
# Write root index (links to sglang package directory)
|
||||
with open(root_index, "w") as f:
|
||||
f.write("<!DOCTYPE html>\n")
|
||||
f.write('<a href="sglang/">sglang</a>\n')
|
||||
|
||||
print(f" Written root index: {root_index}")
|
||||
|
||||
# Write package index in minimal format (matching production sgl-kernel index)
|
||||
with open(package_index, "w") as f:
|
||||
f.write("<!DOCTYPE html>\n")
|
||||
f.write("<h1>SGLang Nightly Wheels</h1>\n")
|
||||
# Write links only
|
||||
f.write("\n".join(unique_links))
|
||||
f.write("\n")
|
||||
|
||||
print(f" Written {len(unique_links)} total wheels to {package_index}")
|
||||
print(f"\nDone! Users can install with:")
|
||||
print(
|
||||
f" pip install sglang --pre --extra-index-url https://sgl-project.github.io/whl/nightly/"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Update wheel index for nightly SGLang releases"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--commit-hash",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Short git commit hash (e.g., 'c5f1e86')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--nightly-version",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Full nightly version string (e.g., '0.5.6.post1.dev7716+gc5f1e86')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--build-date",
|
||||
type=str,
|
||||
required=False,
|
||||
help="Build date in YYYY-MM-DD format (e.g., '2025-12-13')",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Updating nightly wheel index")
|
||||
print(f" Commit: {args.commit_hash}")
|
||||
print(f" Version: {args.nightly_version}")
|
||||
if args.build_date:
|
||||
print(f" Build date: {args.build_date}")
|
||||
|
||||
update_wheel_index(args.commit_hash, args.nightly_version, args.build_date)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user