set CUDA_DEVICE_MAX_CONNECTIONS=1 (#1113)

2024-08-16 03:47:39 +10:00
parent 93d4e354d8
commit 0cb099e20a
1 changed files with 1 additions and 0 deletions
@@ -360,6 +360,7 @@ def _set_envs_and_config(server_args: ServerArgs):
    os.environ["NCCL_CUMEM_ENABLE"] = "0"
    os.environ["NCCL_NVLS_ENABLE"] = "0"
    os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
+    os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"

    # Set ulimit
    set_ulimit()