ce3a20d11b
CP shared KV now keeps explicit L1 and host free-room targets so pressure is handled by planned eviction instead of repeated capacity-edge retries. The host allocator gains contiguous-preferred page reservation, L1 owner-lane allocation prefers contiguous physical pages, and CP HiCache metadata preserves pending backup safety for page-granular radix updates. Mooncake transfer stats and allocator microbenchmarks are included to make the remaining transfer bottlenecks measurable rather than inferred. Constraint: CP shared KV uses decode CP size 1 with all prefill CP ranks participating in transfer, so L1/L2 cache residency must remain page-granular and avoid extra collectives.\nConstraint: Production HiCache can be hundreds of GB, so allocator metadata overhead must be visible before enabling aggressive contiguous allocation broadly.\nRejected: Evict only the exact deficit | this keeps the cache at the cliff and causes repeated evict/allocate pressure.\nRejected: Rely on allocator scans alone for contiguity | remote microbenchmarks show fragmented 220GB-equivalent host metadata can make contiguous-preferred scans multi-ms.\nConfidence: medium\nScope-risk: moderate\nDirective: Do not increase L1/L2 free-room defaults or add new CP collectives without ETE evidence and transfer/allocator measurements.\nTested: python -m py_compile on touched runtime/test/benchmark files.\nTested: PYTHONPATH=. python -m pytest -q test/registered/unit/benchmark/test_cp_hicache_allocator_bench.py => 4 passed, 1 warning.\nTested: Remote g0034 log /mnt/beegfs/cjy/log/sglang_cp_hicache_20260601_233723.log shows active prefill process with L1/L2 free-room args, 702 HTTP 200 chat completions, 6272 prefill batches, and no fatal scheduler traceback in latest scan.\nTested: User-reported L1/L2 cache ETE validation passed on remote run.\nNot-tested: Full local pytest suite; local environment is missing several runtime dependencies.\nNot-tested: CUDA allocator microbenchmark during active production prefill process.\nNot-tested: Mooncake straggler fix; stats show transfer tail latency remains a separate bottleneck.
64 lines
2.0 KiB
Python
64 lines
2.0 KiB
Python
import unittest
|
|
from importlib import util
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
|
|
|
|
register_cpu_ci(est_time=1, suite="stage-a-test-cpu")
|
|
|
|
_UTILS_PATH = (
|
|
Path(__file__).resolve().parents[4]
|
|
/ "python"
|
|
/ "sglang"
|
|
/ "srt"
|
|
/ "disaggregation"
|
|
/ "common"
|
|
/ "utils.py"
|
|
)
|
|
_spec = util.spec_from_file_location("disaggregation_common_utils_under_test", _UTILS_PATH)
|
|
_utils = util.module_from_spec(_spec)
|
|
assert _spec.loader is not None
|
|
_spec.loader.exec_module(_utils)
|
|
contiguous_group_stats = _utils.contiguous_group_stats
|
|
|
|
|
|
class TestDisaggregationCommonUtils(unittest.TestCase):
|
|
def test_contiguous_group_stats_reports_fragmentation_shape(self):
|
|
src_indices = np.array([1, 2, 3, 10, 18, 19], dtype=np.int32)
|
|
dst_indices = np.array([101, 102, 103, 110, 118, 119], dtype=np.int32)
|
|
src_groups = [[1, 2, 3], [10], [18, 19]]
|
|
dst_groups = [[101, 102, 103], [110], [118, 119]]
|
|
|
|
stats = contiguous_group_stats(
|
|
src_indices, dst_indices, src_groups, dst_groups
|
|
)
|
|
|
|
self.assertEqual(stats["pages"], 6)
|
|
self.assertEqual(stats["groups"], 3)
|
|
self.assertEqual(stats["min_group_pages"], 1)
|
|
self.assertEqual(stats["max_group_pages"], 3)
|
|
self.assertAlmostEqual(stats["avg_group_pages"], 2.0)
|
|
self.assertEqual(stats["src_diff_head"], [1, 1, 7, 8, 1])
|
|
self.assertEqual(stats["dst_diff_head"], [1, 1, 7, 8, 1])
|
|
|
|
def test_contiguous_group_stats_handles_empty_transfer(self):
|
|
stats = contiguous_group_stats(
|
|
np.array([], dtype=np.int32),
|
|
np.array([], dtype=np.int32),
|
|
[],
|
|
[],
|
|
)
|
|
|
|
self.assertEqual(stats["pages"], 0)
|
|
self.assertEqual(stats["groups"], 0)
|
|
self.assertEqual(stats["avg_group_pages"], 0.0)
|
|
self.assertEqual(stats["src_diff_head"], [])
|
|
self.assertEqual(stats["dst_diff_head"], [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|