diff --git a/docs/advanced_features/vlm_query.ipynb b/docs/advanced_features/vlm_query.ipynb index f5497f5b7..dd35f5eae 100644 --- a/docs/advanced_features/vlm_query.ipynb +++ b/docs/advanced_features/vlm_query.ipynb @@ -64,6 +64,8 @@ "\n", "nest_asyncio.apply()\n", "\n", + "import sglang.test.doc_patch # noqa: F401\n", + "\n", "model_path = \"Qwen/Qwen2.5-VL-3B-Instruct\"\n", "chat_template = \"qwen2-vl\"\n", "example_image_url = \"https://raw.githubusercontent.com/sgl-project/sglang/main/examples/assets/example_image.png\"" diff --git a/docs/basic_usage/offline_engine_api.ipynb b/docs/basic_usage/offline_engine_api.ipynb index 9c03e90a7..fe8a9e304 100644 --- a/docs/basic_usage/offline_engine_api.ipynb +++ b/docs/basic_usage/offline_engine_api.ipynb @@ -66,7 +66,7 @@ "import asyncio\n", "\n", "import sglang as sgl\n", - "import sglang.test.doc_patch\n", + "import sglang.test.doc_patch # noqa: F401\n", "from sglang.utils import async_stream_and_merge, stream_and_merge\n", "\n", "llm = sgl.Engine(model_path=\"qwen/qwen2.5-0.5b-instruct\")" diff --git a/python/sglang/test/doc_patch.py b/python/sglang/test/doc_patch.py index 0febb295c..503ce86e7 100644 --- a/python/sglang/test/doc_patch.py +++ b/python/sglang/test/doc_patch.py @@ -3,6 +3,7 @@ Do some monkey patch to make the documentation compilation faster and more relia - Avoid port conflicts - Reduce the server launch time +- Limit GPU memory usage to allow multiple servers on the same machine """ import weakref @@ -26,6 +27,9 @@ def patched_post_init(self): self.max_running_requests = DEFAULT_MAX_RUNNING_REQUESTS if self.max_total_tokens is None: self.max_total_tokens = DEFAULT_MAX_TOTAL_TOKENS + # Disable CUDA graphs to avoid memory spikes during capture. + # Notebooks only run a few sample requests, so perf is not critical. + self.disable_cuda_graph = True self.cuda_graph_max_bs = 4 @@ -47,6 +51,7 @@ def launch_server_cmd(command: str, host: str = "0.0.0.0", port: int = None): extra_flags = ( f"--max-running-requests {DEFAULT_MAX_RUNNING_REQUESTS} " f"--max-total-tokens {DEFAULT_MAX_TOTAL_TOKENS} " + f"--disable-cuda-graph " f"--cuda-graph-max-bs 4" )