diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 8b64d9122..585ecb84e 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -128,6 +128,22 @@ echo "Installing python extras: [${EXTRAS}]" $PIP_CMD install -e "python[${EXTRAS}]" --extra-index-url https://download.pytorch.org/whl/${CU_VERSION} $PIP_INSTALL_SUFFIX +# Fix CUDA version mismatch between torch and torchaudio. +# PyPI's torch 2.9.1 bundles cu128 but torchaudio from pytorch.org/cu129 uses cu129. +# This mismatch causes torchaudio's C extension to fail loading, producing: +# "partially initialized module 'torchaudio' has no attribute 'lib'" +# We cannot replace torch with cu129 (breaks sgl_kernel ABI), so instead we reinstall +# torchaudio/torchvision from an index matching torch's CUDA version. +TORCH_CUDA_VER=$(python3 -c "import torch; v=torch.version.cuda; parts=v.split('.'); print(f'cu{parts[0]}{parts[1]}')") +echo "Detected torch CUDA version: ${TORCH_CUDA_VER}" +if [ "${TORCH_CUDA_VER}" != "${CU_VERSION}" ]; then + # Pin versions to match what was installed by pyproject.toml (strip +cuXYZ suffix) + TORCHAUDIO_VER=$(pip show torchaudio 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//') + TORCHVISION_VER=$(pip show torchvision 2>/dev/null | grep "^Version:" | awk '{print $2}' | sed 's/+.*//') + echo "Reinstalling torchaudio==${TORCHAUDIO_VER} torchvision==${TORCHVISION_VER} from ${TORCH_CUDA_VER} index to match torch..." + $PIP_CMD install "torchaudio==${TORCHAUDIO_VER}" "torchvision==${TORCHVISION_VER}" --index-url "https://download.pytorch.org/whl/${TORCH_CUDA_VER}" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX +fi + # Install router for pd-disagg test $PIP_CMD install sglang-router $PIP_INSTALL_SUFFIX