diff --git a/python/sglang/srt/mem_cache/cp_shared_l2_pool.py b/python/sglang/srt/mem_cache/cp_shared_l2_pool.py index 5bc11f40c..5fa9cd0ea 100644 --- a/python/sglang/srt/mem_cache/cp_shared_l2_pool.py +++ b/python/sglang/srt/mem_cache/cp_shared_l2_pool.py @@ -1439,6 +1439,34 @@ def _extract_smaps_page_sizes(smaps_text: str) -> list[int]: return page_sizes +def cp_shared_l2_hugetlbfs_free_bytes(hugetlbfs_dir: str | os.PathLike[str]) -> tuple[int, int, int]: + """Free capacity of a hugetlbfs mount via statvfs. Returns (free_bytes, free_pages, page_bytes). On a + hugetlbfs mount f_frsize is the hugepage size and f_bavail is the free hugepage count, so this reflects the + reserved-but-unused hugepages backing the mount (NOT regular RAM, which psutil reports).""" + st = os.statvfs(os.fspath(hugetlbfs_dir)) + page_bytes = int(st.f_frsize or st.f_bsize) + free_pages = int(st.f_bavail) + return free_pages * page_bytes, free_pages, page_bytes + + +def check_cp_shared_l2_hugetlbfs_capacity( + hugetlbfs_dir: str | os.PathLike[str], requested_bytes: int +) -> None: + """Fail LOUD (clean message) if the hugetlbfs mount lacks enough free hugepages for requested_bytes -- + instead of the cryptic mmap/ftruncate ENOMEM you get otherwise. The shared-L2 slab is backed by hugepages, + so this is the CORRECT capacity check (the regular psutil.available check is wrong for this pool: hugepages + are carved out of regular RAM, so reserving them drops psutil.available).""" + free_bytes, free_pages, page_bytes = cp_shared_l2_hugetlbfs_free_bytes(hugetlbfs_dir) + if requested_bytes > free_bytes: + need_pages = -(-int(requested_bytes) // page_bytes) + _failfast( + f"insufficient hugepages in {os.fspath(hugetlbfs_dir)!r}: need {requested_bytes / 1e9:.2f} GB " + f"({need_pages} x {page_bytes // (1024 * 1024)}M pages) but only {free_bytes / 1e9:.2f} GB free " + f"({free_pages} pages). Reserve more hugepages (echo N | sudo tee " + f".../hugepages-2048kB/nr_hugepages) or reduce --hicache-size." + ) + + def create_cp_shared_host_slab( *, directory: str | os.PathLike[str], @@ -1456,6 +1484,10 @@ def create_cp_shared_host_slab( if not name or Path(name).name != name: _failfast("host slab name must be a single non-empty file name") mapped_nbytes = round_up_cp_shared_l2_host_slab_bytes(nbytes) + # Fail loud here (clean message) if the reserved hugepages can't hold this slab, rather than at the mmap + # below with a cryptic ENOMEM. Per-slab + sequential, so each call sees the free remaining after prior slabs. + if validate_production: + check_cp_shared_l2_hugetlbfs_capacity(hugetlbfs_dir, mapped_nbytes) path = str(Path(hugetlbfs_dir) / name) flags = os.O_RDWR | os.O_CREAT | os.O_EXCL diff --git a/python/sglang/srt/mem_cache/memory_pool_host.py b/python/sglang/srt/mem_cache/memory_pool_host.py index 6ebc3d878..ba4362a9d 100644 --- a/python/sglang/srt/mem_cache/memory_pool_host.py +++ b/python/sglang/srt/mem_cache/memory_pool_host.py @@ -1052,22 +1052,34 @@ class HostKVCache(abc.ABC): self.size > device_pool.size ), "The host memory should be larger than the device memory with the current protocol" - # Verify there is enough available host memory. - host_mem = psutil.virtual_memory() + # Verify there is enough available host memory -- ONLY for the default (regular-malloc) allocator. + # A custom host_tensor_allocator (e.g. the CP shared-L2 slab over hugetlbfs) allocates from HUGEPAGES, + # which are carved OUT of regular RAM -- so psutil.virtual_memory().available drops by the hugepage + # reservation and this check would spuriously fail (it inspects regular memory, not the hugepages the + # slab actually uses). The custom allocator owns its own capacity check: a hugetlbfs mmap fail-louds if + # the reserved hugepages are insufficient (the correct check for that pool). So skip the psutil check + # when a custom allocator is supplied. requested_bytes = self.size * self.size_per_token - # preserve at least 10GB for other usage - ten_gb = 10 * (1024**3) - available_bytes = host_mem.available - ten_gb - if requested_bytes > available_bytes: - raise ValueError( - f"Not enough host memory available. Requesting " - f"{requested_bytes / 1e9:.2f} GB but only have " - f"{available_bytes / 1e9:.2f} GB free. Please reduce the " - f"size of the hierarchical cache." + if host_tensor_allocator is None: + host_mem = psutil.virtual_memory() + # preserve at least 10GB for other usage + ten_gb = 10 * (1024**3) + available_bytes = host_mem.available - ten_gb + if requested_bytes > available_bytes: + raise ValueError( + f"Not enough host memory available. Requesting " + f"{requested_bytes / 1e9:.2f} GB but only have " + f"{available_bytes / 1e9:.2f} GB free. Please reduce the " + f"size of the hierarchical cache." + ) + logger.info( + f"Allocating {requested_bytes / 1e9:.2f} GB host memory for hierarchical KV cache." ) else: logger.info( - f"Allocating {requested_bytes / 1e9:.2f} GB host memory for hierarchical KV cache." + f"Allocating {requested_bytes / 1e9:.2f} GB host KV cache via a custom allocator " + f"(e.g. CP shared-L2 hugetlbfs); skipping the regular-memory check (the allocator owns " + f"its hugepage capacity check)." ) self.kv_buffer = self.init_kv_buffer() diff --git a/test/registered/unit/mem_cache/test_cp_shared_l2_pool.py b/test/registered/unit/mem_cache/test_cp_shared_l2_pool.py index 2c6d8b246..6ea165675 100644 --- a/test/registered/unit/mem_cache/test_cp_shared_l2_pool.py +++ b/test/registered/unit/mem_cache/test_cp_shared_l2_pool.py @@ -2836,5 +2836,26 @@ class TestCpSharedL2EightRankReserveDeterminism(unittest.TestCase): self.assertIsNotNone(a.get_range("cp_hicache_node:9", PAYLOAD_TARGET_KV)) +class TestCpSharedL2HugetlbfsCapacity(unittest.TestCase): + """The hugetlbfs capacity check fails LOUD on insufficient hugepages (clean message vs a cryptic mmap + ENOMEM). statvfs works on any mount, so the logic is exercised here against a tmpdir; the real hugetlbfs + semantics (f_frsize = hugepage size, f_bavail = free hugepages) validate on the live server.""" + + def test_free_bytes_and_capacity_check(self): + import os + import tempfile + + free_bytes_fn = _cp_shared_l2_pool.cp_shared_l2_hugetlbfs_free_bytes + check_fn = _cp_shared_l2_pool.check_cp_shared_l2_hugetlbfs_capacity + with tempfile.TemporaryDirectory() as d: + free, pages, page_bytes = free_bytes_fn(d) + self.assertGreater(free, 0) + self.assertGreater(page_bytes, 0) + self.assertEqual(free, pages * page_bytes) + check_fn(d, 4096) # tiny request fits -> no raise + with self.assertRaises(ValueError): # exceeds free capacity -> fail loud + check_fn(d, free + 10**12) + + if __name__ == "__main__": unittest.main()