Avoid allocating indexer state for shared NSA layers
Target-model skip-topk layers reuse the previous active layer's top-k indices and should not run local indexer modules. Centralize the layer-needs-indexer decision, skip constructing indexers on shared target layers, and skip their checkpoint tensors during load while keeping nextn/draft layers conservative for state safety. Constraint: index skip should reduce GPU memory in both prefill and decode without changing top-k propagation semantics Constraint: nextn/draft layers report shared top-k behavior but still need local indexer state safety Rejected: Loader-only filtering | parameters are already allocated during model construction Rejected: Dummy indexer modules for skipped layers | preserves most of the memory cost this change removes Confidence: high Scope-risk: moderate Directive: Do not reintroduce indexer execution on skip_topk target layers without proving prev_topk propagation and weight residency semantics Tested: remote g0034 cjy-glm5-new PYTHONPATH=python python -m pytest -q test/registered/unit/speculative/test_spec_utils.py test/registered/unit/configs/test_nsa_index_layers.py test/registered/unit/models/test_deepseek_index_skip_weight_loading.py -> 19 passed Tested: remote g0034 cjy-glm5-new py_compile for modified runtime files Not-tested: full GLM5 model restart memory delta measurement
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
|
||||
DeepseekV2WeightLoaderMixin,
|
||||
)
|
||||
|
||||
|
||||
class TestDeepseekIndexSkipWeightLoading:
|
||||
def _loader(self, pattern="FSF"):
|
||||
loader = object.__new__(DeepseekV2WeightLoaderMixin)
|
||||
loader.config = SimpleNamespace(index_topk_freq=1, index_topk_pattern=pattern)
|
||||
return loader
|
||||
|
||||
def test_skip_layer_indexer_checkpoint_weights_are_ignored(self):
|
||||
loader = self._loader()
|
||||
|
||||
assert not loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.0.self_attn.indexer.wk.weight", is_nextn=False
|
||||
)
|
||||
assert loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.1.self_attn.indexer.wk.weight", is_nextn=False
|
||||
)
|
||||
assert not loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.2.self_attn.indexer.wk.weight", is_nextn=False
|
||||
)
|
||||
|
||||
def test_non_indexer_and_nextn_weights_are_not_skipped(self):
|
||||
loader = self._loader()
|
||||
|
||||
assert not loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.1.self_attn.q_b_proj.weight", is_nextn=False
|
||||
)
|
||||
assert not loader._should_skip_nsa_indexer_weight(
|
||||
"model.layers.1.self_attn.indexer.wk.weight", is_nextn=True
|
||||
)
|
||||
Reference in New Issue
Block a user