Avoid allocating indexer state for shared NSA layers

Target-model skip-topk layers reuse the previous active layer's top-k indices and should not run local indexer modules. Centralize the layer-needs-indexer decision, skip constructing indexers on shared target layers, and skip their checkpoint tensors during load while keeping nextn/draft layers conservative for state safety.

Constraint: index skip should reduce GPU memory in both prefill and decode without changing top-k propagation semantics
Constraint: nextn/draft layers report shared top-k behavior but still need local indexer state safety
Rejected: Loader-only filtering | parameters are already allocated during model construction
Rejected: Dummy indexer modules for skipped layers | preserves most of the memory cost this change removes
Confidence: high
Scope-risk: moderate
Directive: Do not reintroduce indexer execution on skip_topk target layers without proving prev_topk propagation and weight residency semantics
Tested: remote g0034 cjy-glm5-new PYTHONPATH=python python -m pytest -q test/registered/unit/speculative/test_spec_utils.py test/registered/unit/configs/test_nsa_index_layers.py test/registered/unit/models/test_deepseek_index_skip_weight_loading.py -> 19 passed
Tested: remote g0034 cjy-glm5-new py_compile for modified runtime files
Not-tested: full GLM5 model restart memory delta measurement
This commit is contained in:
laoyao0822
2026-06-21 05:24:05 +08:00
committed by leavelet
parent 187b700406
commit 9549b268d7
7 changed files with 139 additions and 28 deletions
@@ -4,6 +4,7 @@ import pytest
from sglang.srt.configs.nsa_index_layers import (
build_nsa_index_layer_plan,
nsa_indexer_layer_needs_weights,
nsa_index_skip_flags,
)
@@ -134,3 +135,18 @@ def test_invalid_offset_fails_before_layer_zero_can_skip_without_prior_topk():
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=0)
with pytest.raises(ValueError, match="index_skip_topk_offset"):
nsa_index_skip_flags(cfg, 0)
def test_indexer_weights_needed_only_for_active_target_layers():
cfg = SimpleNamespace(index_topk_freq=1, index_topk_pattern="FSF")
assert nsa_indexer_layer_needs_weights(cfg, 0) is True
assert nsa_indexer_layer_needs_weights(cfg, 1) is False
assert nsa_indexer_layer_needs_weights(cfg, 2) is True
def test_indexer_weights_kept_for_nextn_even_when_topk_is_shared():
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=1)
assert nsa_index_skip_flags(cfg, 0, is_nextn=True) == (True, True)
assert nsa_indexer_layer_needs_weights(cfg, 0, is_nextn=True) is True
@@ -0,0 +1,35 @@
from types import SimpleNamespace
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
DeepseekV2WeightLoaderMixin,
)
class TestDeepseekIndexSkipWeightLoading:
def _loader(self, pattern="FSF"):
loader = object.__new__(DeepseekV2WeightLoaderMixin)
loader.config = SimpleNamespace(index_topk_freq=1, index_topk_pattern=pattern)
return loader
def test_skip_layer_indexer_checkpoint_weights_are_ignored(self):
loader = self._loader()
assert not loader._should_skip_nsa_indexer_weight(
"model.layers.0.self_attn.indexer.wk.weight", is_nextn=False
)
assert loader._should_skip_nsa_indexer_weight(
"model.layers.1.self_attn.indexer.wk.weight", is_nextn=False
)
assert not loader._should_skip_nsa_indexer_weight(
"model.layers.2.self_attn.indexer.wk.weight", is_nextn=False
)
def test_non_indexer_and_nextn_weights_are_not_skipped(self):
loader = self._loader()
assert not loader._should_skip_nsa_indexer_weight(
"model.layers.1.self_attn.q_b_proj.weight", is_nextn=False
)
assert not loader._should_skip_nsa_indexer_weight(
"model.layers.1.self_attn.indexer.wk.weight", is_nextn=True
)