diff --git a/.github/workflows/release-docker-cu13.yml b/.github/workflows/release-docker-cu13.yml index 29d2c2ade..aa2348333 100644 --- a/.github/workflows/release-docker-cu13.yml +++ b/.github/workflows/release-docker-cu13.yml @@ -63,6 +63,7 @@ jobs: --build-arg CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \ --build-arg GRACE_BLACKWELL=${{ matrix.grace_blackwell }} \ --build-arg USE_LATEST_SGLANG=1 \ + --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \ -t lmsysorg/sglang:${{ matrix.tag }} \ --no-cache \ . diff --git a/python/pyproject.toml b/python/pyproject.toml index fdf597405..f7800a932 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -66,6 +66,7 @@ dependencies = [ "timm==1.0.16", "torch_memory_saver==0.0.9", "torch==2.9.1", + "torchao==0.9.0", "torchaudio==2.9.1", "torchcodec==0.8.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torchcodec does not exist in those systems. If not provided, transformer will use torchvision instead by default. "torchvision", diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index 0957c74f5..74e2b0c58 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -57,6 +57,7 @@ dependencies = [ "soundfile==0.13.1", "tiktoken", "timm==1.0.16", + "torchao==0.9.0", "tqdm", "transformers==4.57.1", "uvicorn", diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index f4a236d34..7efce6dfa 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -57,6 +57,7 @@ runtime_common = [ "soundfile==0.13.1", "tiktoken", "timm==1.0.16", + "torchao==0.9.0", "tqdm", "transformers==4.57.1", "uvicorn", diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index 2b3486e5d..f88480acf 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -61,6 +61,7 @@ dependencies = [ "soundfile==0.13.1", "tiktoken", "timm==1.0.16", + "torchao==0.9.0", "tqdm", "transformers==4.57.1", "uvicorn",