B300 NextN fp4: also honor speculative MoE runner backend in keep-decision (+ debug log)
The NextN/MTP draft is built by the EAGLE draft worker, where get_moe_runner_backend() didn't resolve to flashinfer_trtllm, so the fp4 drop fired anyway -> 6144-vs-3072 crash in draft load. Check both the main and speculative runner backends (spec is AUTO when --speculative-moe-runner-backend unset); keep fp4 if either is a flashinfer fp4 backend. Temporary [NextN-fp4] warning logs the resolved backends. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -93,12 +93,28 @@ def _should_drop_nextn_modelopt_fp4_quant_config(
|
||||
"""
|
||||
if quant_config is None or quant_config.get_name() != "modelopt_fp4":
|
||||
return False
|
||||
backend = get_moe_runner_backend()
|
||||
keep_fp4 = (
|
||||
backend.is_flashinfer_trtllm()
|
||||
or backend.is_flashinfer_trtllm_routed()
|
||||
or backend.is_flashinfer_cutedsl()
|
||||
or backend.is_flashinfer_cutlass()
|
||||
# The NextN/MTP draft is built by the EAGLE draft worker, where the effective
|
||||
# MoE runner may be the *speculative* backend (which is AUTO when
|
||||
# --speculative-moe-runner-backend is unset), not the main runner. Keep fp4 if
|
||||
# EITHER the main or the speculative runner is a flashinfer fp4 backend.
|
||||
from sglang.srt.layers.moe.utils import get_speculative_moe_runner_backend
|
||||
|
||||
def _is_fp4_capable(b) -> bool:
|
||||
return (
|
||||
b.is_flashinfer_trtllm()
|
||||
or b.is_flashinfer_trtllm_routed()
|
||||
or b.is_flashinfer_cutedsl()
|
||||
or b.is_flashinfer_cutlass()
|
||||
)
|
||||
|
||||
main_backend = get_moe_runner_backend()
|
||||
spec_backend = get_speculative_moe_runner_backend()
|
||||
keep_fp4 = _is_fp4_capable(main_backend) or _is_fp4_capable(spec_backend)
|
||||
logger.warning(
|
||||
"[NextN-fp4] main_runner=%s spec_runner=%s keep_fp4=%s",
|
||||
main_backend,
|
||||
spec_backend,
|
||||
keep_fp4,
|
||||
)
|
||||
return not keep_fp4
|
||||
|
||||
|
||||
Reference in New Issue
Block a user