Avoid allocating indexer state for shared NSA layers
Target-model skip-topk layers reuse the previous active layer's top-k indices and should not run local indexer modules. Centralize the layer-needs-indexer decision, skip constructing indexers on shared target layers, and skip their checkpoint tensors during load while keeping nextn/draft layers conservative for state safety. Constraint: index skip should reduce GPU memory in both prefill and decode without changing top-k propagation semantics Constraint: nextn/draft layers report shared top-k behavior but still need local indexer state safety Rejected: Loader-only filtering | parameters are already allocated during model construction Rejected: Dummy indexer modules for skipped layers | preserves most of the memory cost this change removes Confidence: high Scope-risk: moderate Directive: Do not reintroduce indexer execution on skip_topk target layers without proving prev_topk propagation and weight residency semantics Tested: remote g0034 cjy-glm5-new PYTHONPATH=python python -m pytest -q test/registered/unit/speculative/test_spec_utils.py test/registered/unit/configs/test_nsa_index_layers.py test/registered/unit/models/test_deepseek_index_skip_weight_loading.py -> 19 passed Tested: remote g0034 cjy-glm5-new py_compile for modified runtime files Not-tested: full GLM5 model restart memory delta measurement
This commit is contained in:
@@ -4,6 +4,7 @@ import pytest
|
||||
|
||||
from sglang.srt.configs.nsa_index_layers import (
|
||||
build_nsa_index_layer_plan,
|
||||
nsa_indexer_layer_needs_weights,
|
||||
nsa_index_skip_flags,
|
||||
)
|
||||
|
||||
@@ -134,3 +135,18 @@ def test_invalid_offset_fails_before_layer_zero_can_skip_without_prior_topk():
|
||||
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=0)
|
||||
with pytest.raises(ValueError, match="index_skip_topk_offset"):
|
||||
nsa_index_skip_flags(cfg, 0)
|
||||
|
||||
|
||||
def test_indexer_weights_needed_only_for_active_target_layers():
|
||||
cfg = SimpleNamespace(index_topk_freq=1, index_topk_pattern="FSF")
|
||||
|
||||
assert nsa_indexer_layer_needs_weights(cfg, 0) is True
|
||||
assert nsa_indexer_layer_needs_weights(cfg, 1) is False
|
||||
assert nsa_indexer_layer_needs_weights(cfg, 2) is True
|
||||
|
||||
|
||||
def test_indexer_weights_kept_for_nextn_even_when_topk_is_shared():
|
||||
cfg = SimpleNamespace(index_topk_freq=4, index_skip_topk_offset=1)
|
||||
|
||||
assert nsa_index_skip_flags(cfg, 0, is_nextn=True) == (True, True)
|
||||
assert nsa_indexer_layer_needs_weights(cfg, 0, is_nextn=True) is True
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
|
||||
DeepseekV2WeightLoaderMixin,
|
||||
)
|
||||
|
||||
|
||||
class TestDeepseekIndexSkipWeightLoading:
|
||||
def _loader(self, pattern="FSF"):
|
||||
loader = object.__new__(DeepseekV2WeightLoaderMixin)
|
||||
loader.config = SimpleNamespace(index_topk_freq=1, index_topk_pattern=pattern)
|
||||
return loader
|
||||
|
||||
def test_skip_layer_indexer_checkpoint_weights_are_ignored(self):
|
||||
loader = self._loader()
|
||||
|
||||
assert not loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.0.self_attn.indexer.wk.weight", is_nextn=False
|
||||
)
|
||||
assert loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.1.self_attn.indexer.wk.weight", is_nextn=False
|
||||
)
|
||||
assert not loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.2.self_attn.indexer.wk.weight", is_nextn=False
|
||||
)
|
||||
|
||||
def test_non_indexer_and_nextn_weights_are_not_skipped(self):
|
||||
loader = self._loader()
|
||||
|
||||
assert not loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.1.self_attn.q_b_proj.weight", is_nextn=False
|
||||
)
|
||||
assert not loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.1.self_attn.indexer.wk.weight", is_nextn=True
|
||||
)
|
||||
Reference in New Issue
Block a user