From 6114c3e3e51b19626ba1392ce2a527c18981307d Mon Sep 17 00:00:00 2001 From: leavelet Date: Tue, 23 Jun 2026 10:17:33 +0000 Subject: [PATCH] HiCache host pool: check hugepages, not regular RAM, for the hugetlbfs-backed (CP shared-L2) pool The host-pool size check used psutil.virtual_memory().available -- regular RAM -- unconditionally. But hugepages are carved OUT of regular RAM, so reserving e.g. 1 TB of 2M hugepages drops psutil.available by ~1 TB, and the check then spuriously fails ("Not enough host memory ... only have 897 GB free") even though the slab is allocated from the hugepages, not regular RAM. It was inspecting the wrong pool. Two fixes: - memory_pool_host.py: gate the psutil check on the DEFAULT (regular-malloc) allocator. A custom host_tensor_allocator (the CP shared-L2 slab over hugetlbfs) owns its own capacity check, so skip the regular-RAM check for it. - cp_shared_l2_pool.py: add cp_shared_l2_hugetlbfs_free_bytes() + check_cp_shared_l2_hugetlbfs_capacity() (statvfs the hugetlbfs mount: f_frsize = hugepage size, f_bavail = free hugepages) and call it in create_cp_shared_host_slab before ftruncate/mmap. Now insufficient hugepages fail LOUD with a clean, actionable message ("insufficient hugepages in /dev/hugepages: need X GB (N x 2M pages) but only Y GB free; reserve more or reduce --hicache-size") instead of a cryptic mmap ENOMEM. Per-slab + sequential, so each payload's slab sees the free remaining after the prior ones. Unit-tested (capacity check: tiny fits, over-capacity fails loud). Pre-existing 2b L2-pooling code (not L3). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sglang/srt/mem_cache/cp_shared_l2_pool.py | 32 +++++++++++++++++ .../sglang/srt/mem_cache/memory_pool_host.py | 36 ++++++++++++------- .../unit/mem_cache/test_cp_shared_l2_pool.py | 21 +++++++++++ 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/python/sglang/srt/mem_cache/cp_shared_l2_pool.py b/python/sglang/srt/mem_cache/cp_shared_l2_pool.py index 5bc11f40c..5fa9cd0ea 100644 --- a/python/sglang/srt/mem_cache/cp_shared_l2_pool.py +++ b/python/sglang/srt/mem_cache/cp_shared_l2_pool.py @@ -1439,6 +1439,34 @@ def _extract_smaps_page_sizes(smaps_text: str) -> list[int]: return page_sizes +def cp_shared_l2_hugetlbfs_free_bytes(hugetlbfs_dir: str | os.PathLike[str]) -> tuple[int, int, int]: + """Free capacity of a hugetlbfs mount via statvfs. Returns (free_bytes, free_pages, page_bytes). On a + hugetlbfs mount f_frsize is the hugepage size and f_bavail is the free hugepage count, so this reflects the + reserved-but-unused hugepages backing the mount (NOT regular RAM, which psutil reports).""" + st = os.statvfs(os.fspath(hugetlbfs_dir)) + page_bytes = int(st.f_frsize or st.f_bsize) + free_pages = int(st.f_bavail) + return free_pages * page_bytes, free_pages, page_bytes + + +def check_cp_shared_l2_hugetlbfs_capacity( + hugetlbfs_dir: str | os.PathLike[str], requested_bytes: int +) -> None: + """Fail LOUD (clean message) if the hugetlbfs mount lacks enough free hugepages for requested_bytes -- + instead of the cryptic mmap/ftruncate ENOMEM you get otherwise. The shared-L2 slab is backed by hugepages, + so this is the CORRECT capacity check (the regular psutil.available check is wrong for this pool: hugepages + are carved out of regular RAM, so reserving them drops psutil.available).""" + free_bytes, free_pages, page_bytes = cp_shared_l2_hugetlbfs_free_bytes(hugetlbfs_dir) + if requested_bytes > free_bytes: + need_pages = -(-int(requested_bytes) // page_bytes) + _failfast( + f"insufficient hugepages in {os.fspath(hugetlbfs_dir)!r}: need {requested_bytes / 1e9:.2f} GB " + f"({need_pages} x {page_bytes // (1024 * 1024)}M pages) but only {free_bytes / 1e9:.2f} GB free " + f"({free_pages} pages). Reserve more hugepages (echo N | sudo tee " + f".../hugepages-2048kB/nr_hugepages) or reduce --hicache-size." + ) + + def create_cp_shared_host_slab( *, directory: str | os.PathLike[str], @@ -1456,6 +1484,10 @@ def create_cp_shared_host_slab( if not name or Path(name).name != name: _failfast("host slab name must be a single non-empty file name") mapped_nbytes = round_up_cp_shared_l2_host_slab_bytes(nbytes) + # Fail loud here (clean message) if the reserved hugepages can't hold this slab, rather than at the mmap + # below with a cryptic ENOMEM. Per-slab + sequential, so each call sees the free remaining after prior slabs. + if validate_production: + check_cp_shared_l2_hugetlbfs_capacity(hugetlbfs_dir, mapped_nbytes) path = str(Path(hugetlbfs_dir) / name) flags = os.O_RDWR | os.O_CREAT | os.O_EXCL diff --git a/python/sglang/srt/mem_cache/memory_pool_host.py b/python/sglang/srt/mem_cache/memory_pool_host.py index 6ebc3d878..ba4362a9d 100644 --- a/python/sglang/srt/mem_cache/memory_pool_host.py +++ b/python/sglang/srt/mem_cache/memory_pool_host.py @@ -1052,22 +1052,34 @@ class HostKVCache(abc.ABC): self.size > device_pool.size ), "The host memory should be larger than the device memory with the current protocol" - # Verify there is enough available host memory. - host_mem = psutil.virtual_memory() + # Verify there is enough available host memory -- ONLY for the default (regular-malloc) allocator. + # A custom host_tensor_allocator (e.g. the CP shared-L2 slab over hugetlbfs) allocates from HUGEPAGES, + # which are carved OUT of regular RAM -- so psutil.virtual_memory().available drops by the hugepage + # reservation and this check would spuriously fail (it inspects regular memory, not the hugepages the + # slab actually uses). The custom allocator owns its own capacity check: a hugetlbfs mmap fail-louds if + # the reserved hugepages are insufficient (the correct check for that pool). So skip the psutil check + # when a custom allocator is supplied. requested_bytes = self.size * self.size_per_token - # preserve at least 10GB for other usage - ten_gb = 10 * (1024**3) - available_bytes = host_mem.available - ten_gb - if requested_bytes > available_bytes: - raise ValueError( - f"Not enough host memory available. Requesting " - f"{requested_bytes / 1e9:.2f} GB but only have " - f"{available_bytes / 1e9:.2f} GB free. Please reduce the " - f"size of the hierarchical cache." + if host_tensor_allocator is None: + host_mem = psutil.virtual_memory() + # preserve at least 10GB for other usage + ten_gb = 10 * (1024**3) + available_bytes = host_mem.available - ten_gb + if requested_bytes > available_bytes: + raise ValueError( + f"Not enough host memory available. Requesting " + f"{requested_bytes / 1e9:.2f} GB but only have " + f"{available_bytes / 1e9:.2f} GB free. Please reduce the " + f"size of the hierarchical cache." + ) + logger.info( + f"Allocating {requested_bytes / 1e9:.2f} GB host memory for hierarchical KV cache." ) else: logger.info( - f"Allocating {requested_bytes / 1e9:.2f} GB host memory for hierarchical KV cache." + f"Allocating {requested_bytes / 1e9:.2f} GB host KV cache via a custom allocator " + f"(e.g. CP shared-L2 hugetlbfs); skipping the regular-memory check (the allocator owns " + f"its hugepage capacity check)." ) self.kv_buffer = self.init_kv_buffer() diff --git a/test/registered/unit/mem_cache/test_cp_shared_l2_pool.py b/test/registered/unit/mem_cache/test_cp_shared_l2_pool.py index 2c6d8b246..6ea165675 100644 --- a/test/registered/unit/mem_cache/test_cp_shared_l2_pool.py +++ b/test/registered/unit/mem_cache/test_cp_shared_l2_pool.py @@ -2836,5 +2836,26 @@ class TestCpSharedL2EightRankReserveDeterminism(unittest.TestCase): self.assertIsNotNone(a.get_range("cp_hicache_node:9", PAYLOAD_TARGET_KV)) +class TestCpSharedL2HugetlbfsCapacity(unittest.TestCase): + """The hugetlbfs capacity check fails LOUD on insufficient hugepages (clean message vs a cryptic mmap + ENOMEM). statvfs works on any mount, so the logic is exercised here against a tmpdir; the real hugetlbfs + semantics (f_frsize = hugepage size, f_bavail = free hugepages) validate on the live server.""" + + def test_free_bytes_and_capacity_check(self): + import os + import tempfile + + free_bytes_fn = _cp_shared_l2_pool.cp_shared_l2_hugetlbfs_free_bytes + check_fn = _cp_shared_l2_pool.check_cp_shared_l2_hugetlbfs_capacity + with tempfile.TemporaryDirectory() as d: + free, pages, page_bytes = free_bytes_fn(d) + self.assertGreater(free, 0) + self.assertGreater(page_bytes, 0) + self.assertEqual(free, pages * page_bytes) + check_fn(d, 4096) # tiny request fits -> no raise + with self.assertRaises(ValueError): # exceeds free capacity -> fail loud + check_fn(d, free + 10**12) + + if __name__ == "__main__": unittest.main()