From e5ac6229e18662dff4bef85582d8c71dcd885fff Mon Sep 17 00:00:00 2001 From: Kangyan-Zhou Date: Sat, 31 Jan 2026 23:30:51 -0800 Subject: [PATCH] Fix installation script for H200 runners (#18050) --- scripts/ci/cuda/ci_install_deepep.sh | 42 ++++++++++++++++++++---- scripts/ci/cuda/ci_install_dependency.sh | 27 +++++++++++++-- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/scripts/ci/cuda/ci_install_deepep.sh b/scripts/ci/cuda/ci_install_deepep.sh index 9e54df24c..bb4185799 100755 --- a/scripts/ci/cuda/ci_install_deepep.sh +++ b/scripts/ci/cuda/ci_install_deepep.sh @@ -21,17 +21,40 @@ if python3 -c "import deep_ep" >/dev/null 2>&1; then fi # Install system dependencies -apt install -y curl wget git sudo rdma-core infiniband-diags openssh-server perftest libibumad3 libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils libnl-3-200 libnl-route-3-200 librdmacm1 build-essential cmake +# Use fallback logic in case apt fails due to unrelated broken packages on the runner +DEEPEP_SYSTEM_DEPS="curl wget git sudo rdma-core infiniband-diags openssh-server perftest libibumad3 libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils libnl-3-200 libnl-route-3-200 librdmacm1 build-essential cmake" +apt-get install -y --no-install-recommends $DEEPEP_SYSTEM_DEPS || { + echo "Warning: apt-get install failed, checking if required packages are available..." + for pkg in $DEEPEP_SYSTEM_DEPS; do + if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then + echo "ERROR: Required package $pkg is not installed and apt-get failed" + exit 1 + fi + done + echo "All required packages are already installed, continuing..." +} # Install GDRCopy rm -rf /opt/gdrcopy && mkdir -p /opt/gdrcopy cd /opt/gdrcopy git clone https://github.com/NVIDIA/gdrcopy.git . git checkout v2.5.1 -apt update -apt install -y nvidia-dkms-580 -apt install -y build-essential devscripts debhelper fakeroot pkg-config dkms -apt install -y check libsubunit0 libsubunit-dev python3-venv +apt-get update || true # May fail due to unrelated broken packages +GDRCOPY_DEPS_1="nvidia-dkms-580" +GDRCOPY_DEPS_2="build-essential devscripts debhelper fakeroot pkg-config dkms" +GDRCOPY_DEPS_3="check libsubunit0 libsubunit-dev python3-venv" +for deps_group in "$GDRCOPY_DEPS_1" "$GDRCOPY_DEPS_2" "$GDRCOPY_DEPS_3"; do + apt-get install -y --no-install-recommends $deps_group || { + echo "Warning: apt-get install failed for '$deps_group', checking if packages are available..." + for pkg in $deps_group; do + if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then + echo "ERROR: Required package $pkg is not installed and apt-get failed" + exit 1 + fi + done + echo "All required packages from '$deps_group' are already installed, continuing..." + } +done cd packages CUDA=/usr/local/cuda ./build-deb-packages.sh dpkg -i gdrdrv-dkms_*.deb @@ -44,7 +67,14 @@ LIB_PATH="/usr/lib/$ARCH-linux-gnu" if [ ! -e "$LIB_PATH/libmlx5.so" ]; then ln -s $LIB_PATH/libmlx5.so.1 $LIB_PATH/libmlx5.so fi -apt-get update && apt-get install -y libfabric-dev +apt-get update || true +apt-get install -y --no-install-recommends libfabric-dev || { + if ! dpkg -l libfabric-dev 2>/dev/null | grep -q "^ii"; then + echo "ERROR: Required package libfabric-dev is not installed and apt-get failed" + exit 1 + fi + echo "libfabric-dev is already installed, continuing..." +} # Install DeepEP DEEPEP_DIR=/root/.cache/deepep diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index a6b1483db..8281292fb 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -27,7 +27,19 @@ echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" python3 -c 'import os, shutil, tempfile, getpass; cache_dir = os.environ.get("TORCHINDUCTOR_CACHE_DIR") or os.path.join(tempfile.gettempdir(), "torchinductor_" + getpass.getuser()); shutil.rmtree(cache_dir, ignore_errors=True)' # Install apt packages -apt install -y git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils +# Use --no-install-recommends and ignore errors from unrelated broken packages on the runner +# The NVIDIA driver packages may have broken dependencies that are unrelated to these packages +apt-get install -y --no-install-recommends git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils || { + echo "Warning: apt-get install failed, checking if required packages are available..." + # Verify the packages we need are actually installed + for pkg in git libnuma-dev libssl-dev pkg-config libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils; do + if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then + echo "ERROR: Required package $pkg is not installed and apt-get failed" + exit 1 + fi + done + echo "All required packages are already installed, continuing..." +} # Check if protoc of correct architecture is already installed if command -v protoc >/dev/null 2>&1; then @@ -47,8 +59,17 @@ if [ "${INSTALL_PROTOC:-0}" = "1" ]; then echo "Installing protoc..." if command -v apt-get &> /dev/null; then # Ubuntu/Debian - apt-get update - apt-get install -y wget unzip gcc g++ perl make + apt-get update || true # May fail due to unrelated broken packages + apt-get install -y --no-install-recommends wget unzip gcc g++ perl make || { + echo "Warning: apt-get install failed, checking if required packages are available..." + for pkg in wget unzip gcc g++ perl make; do + if ! dpkg -l "$pkg" 2>/dev/null | grep -q "^ii"; then + echo "ERROR: Required package $pkg is not installed and apt-get failed" + exit 1 + fi + done + echo "All required packages are already installed, continuing..." + } elif command -v yum &> /dev/null; then # RHEL/CentOS yum update -y