Reduce inactive NSA index-cache transfer safely
Centralize the IndexCache skip formula and thread the resulting active logical index layers into NSA KV pools. HiCache now skips only the indexer H2D/D2H payload for inactive target layers while preserving per-layer MLA KV transfer, keeping allocation shape unchanged for this phase. Constraint: P0-P2 must not compact device or host allocation yet; prefill/decode state transfer still has no logical layer-id metadata. Rejected: Recompute the skip formula separately in mem_cache | formula drift would corrupt cache or waste transfers when offset/pattern settings change. Rejected: Skip whole-layer HiCache load/backup | MLA KV remains required for every attention layer. Confidence: medium Scope-risk: moderate Directive: Before enabling compact state buffers or compact allocation, add layer-id metadata validation to PD transfer. Tested: Local py_compile for touched files; remote pytest in g0034 container: test_nsa_index_layers.py and TestNSAIndexerPageIndices, 20 passed. Not-tested: ETE replay/GSM8K with --nsa-index-topk-freq 4; PD state-transfer compaction remains unimplemented.
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.srt.configs.nsa_index_layers import (
|
||||
build_nsa_index_layer_plan,
|
||||
nsa_index_skip_flags,
|
||||
)
|
||||
|
||||
|
||||
def test_default_freq_one_marks_all_target_layers_active():
|
||||
cfg = SimpleNamespace(index_topk_freq=1)
|
||||
plan = build_nsa_index_layer_plan(cfg, 0, 6)
|
||||
assert plan.active_layer_ids == (0, 1, 2, 3, 4, 5)
|
||||
assert [nsa_index_skip_flags(cfg, i)[0] for i in range(6)] == [False] * 6
|
||||
|
||||
|
||||
def test_freq_four_without_offset_matches_current_model_formula():
|
||||
cfg = SimpleNamespace(index_topk_freq=4)
|
||||
plan = build_nsa_index_layer_plan(cfg, 0, 12)
|
||||
assert plan.active_layer_ids == (0, 1, 5, 9)
|
||||
assert [nsa_index_skip_flags(cfg, i)[0] for i in range(10)] == [
|
||||
False,
|
||||
False,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
]
|
||||
|
||||
|
||||
def test_freq_four_with_offset_one_uses_layers_zero_four_eight():
|
||||
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=1)
|
||||
plan = build_nsa_index_layer_plan(cfg, 0, 12)
|
||||
assert plan.active_layer_ids == (0, 4, 8)
|
||||
assert [nsa_index_skip_flags(cfg, i)[0] for i in range(9)] == [
|
||||
False,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
]
|
||||
|
||||
|
||||
def test_pattern_marks_non_shared_layers_active():
|
||||
cfg = SimpleNamespace(index_topk_freq=1, index_topk_pattern="CSSSCSS")
|
||||
plan = build_nsa_index_layer_plan(cfg, 0, len(cfg.index_topk_pattern))
|
||||
assert plan.active_layer_ids == (0, 4)
|
||||
|
||||
|
||||
def test_nextn_keeps_all_draft_layers_active_for_state_safety():
|
||||
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=1)
|
||||
plan = build_nsa_index_layer_plan(cfg, 0, 1, is_nextn=True)
|
||||
assert plan.active_layer_ids == (0,)
|
||||
assert nsa_index_skip_flags(cfg, 0, is_nextn=True) == (True, True)
|
||||
|
||||
|
||||
def test_nonzero_start_layer_preserves_logical_layer_ids():
|
||||
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=1)
|
||||
plan = build_nsa_index_layer_plan(cfg, 4, 13)
|
||||
assert plan.active_layer_ids == (4, 8, 12)
|
||||
assert plan.slot_for_layer(8) == 1
|
||||
|
||||
|
||||
def test_inactive_slot_lookup_fails_fast():
|
||||
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=1)
|
||||
plan = build_nsa_index_layer_plan(cfg, 0, 8)
|
||||
with pytest.raises(RuntimeError, match="inactive index layer requested"):
|
||||
plan.slot_for_layer(1)
|
||||
|
||||
|
||||
def test_invalid_offset_fails_before_layer_zero_can_skip_without_prior_topk():
|
||||
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=0)
|
||||
with pytest.raises(ValueError, match="index_skip_topk_offset"):
|
||||
nsa_index_skip_flags(cfg, 0)
|
||||
Reference in New Issue
Block a user