From 50fde834ae2566cfe516ffafabcef495504c9ce6 Mon Sep 17 00:00:00 2001 From: laoyao0822 Date: Tue, 9 Jun 2026 01:48:24 +0800 Subject: [PATCH] Keep CP compute padding out of sparse MoE CP shared-KV compute padding creates per-request lane slots, so valid rows are not a simple prefix/suffix mask. DeepEP MoE was still seeing dummy rows and using scalar non-padded semantics, which let padding participate in gate/topk and corrupted cache-hit tiny-extend inference.\n\nThe fix compacts CP-local valid rows before MoE dispatch and restores the compact output back to the compute-padded row layout before downstream layer communication. The local GSM8K investigation ledger is now removed from the tracked tree and ignored so future debug notes stay local.\n\nConstraint: CP shared-KV compute-padding layout must keep downstream communicator shapes stable.\nRejected: Disable bs>1/current reuse/cache-hit fast paths | hides the semantic bug and loses the intended performance path.\nRejected: Use num_token_non_padded for MoE under compute padding | valid rows are interleaved with dummy lane slots, not suffix-padded.\nConfidence: high\nScope-risk: moderate\nDirective: Do not feed compute-padded dummy rows into sparse MoE gate/topk; compact valid rows at the MoE boundary and restore shape afterward.\nTested: python -m py_compile python/sglang/srt/layers/attention/nsa/utils.py python/sglang/srt/models/deepseek_v2.py\nTested: remote focused CP utils tests passed, 4 tests.\nTested: remote GSM8K 50-question smoke accuracy 0.960; 200-question runs accuracy 0.955 and 0.965; full 1319-question run accuracy 0.952.\nNot-tested: Long-running production traffic beyond GSM8K after this commit. --- .gitignore | 3 + ...prefill_cp_gsm8k_cachehit_temp_findings.md | 2309 ----------------- .../sglang/srt/layers/attention/nsa/utils.py | 52 + python/sglang/srt/models/deepseek_v2.py | 25 + .../unit/layers/test_nsa_cp_utils.py | 50 + 5 files changed, 130 insertions(+), 2309 deletions(-) delete mode 100644 docs/advanced_features/nsa_prefill_cp_gsm8k_cachehit_temp_findings.md diff --git a/.gitignore b/.gitignore index 1488bf9b7..54d5170b5 100644 --- a/.gitignore +++ b/.gitignore @@ -280,3 +280,6 @@ sgl-kernel/csrc/**/*_musa/ # (kept on disk for local work, never committed) docs_internal/ tai-kernel/ + +# Local CP shared-KV/GSM8K debug ledger +docs/advanced_features/nsa_prefill_cp_gsm8k_cachehit_temp_findings.md diff --git a/docs/advanced_features/nsa_prefill_cp_gsm8k_cachehit_temp_findings.md b/docs/advanced_features/nsa_prefill_cp_gsm8k_cachehit_temp_findings.md deleted file mode 100644 index 8da835210..000000000 --- a/docs/advanced_features/nsa_prefill_cp_gsm8k_cachehit_temp_findings.md +++ /dev/null @@ -1,2309 +0,0 @@ -# CP shared-KV bs>1 GSM8K cache-hit 掉点临时排查记录 - -> 目的：上下文压缩后先读本文件，避免重复扫描远端大日志与重复审计同一批路径。 - -## 当前复现 - -- 远端日志：`g0034:/mnt/beegfs/cjy/log/sglang_cp_hicache_20260606_193039.log` -- 用户连续两次全量 GSM8K，同一 prefill 进程： - - 第 1 次：`Accuracy=0.953`, `Invalid=0.000`, `Latency=211.445s`, `Output throughput=654.008 token/s` - - 第 2 次：`Accuracy=0.692`, `Invalid=0.002`, `Latency=214.264s`, `Output throughput=654.376 token/s` - -## 日志已确认事实 - -- 无显式错误：`CP_SHARED_KV_FAIL_FAST=0`, `CP_SHARED_KV_FALLBACK=0`, `RuntimeError=0`, `Traceback=0`, `Health check failed=0`。 -- 无明显 L2 load-back：`HiCache-load=0`, `CacheCtrl-load=0`, `load_back=0`。 -- 第一轮并非全冷：CP0 典型 batch 已有 `#new-seq=5, #new-token=640/704/576, #cached-token=3200`，约每请求 640 cached tokens，仍然精度正常。 -- 第二轮典型变为 `#new-seq=5, #new-token=320/384, #cached-token=3520/3584/...`，约每请求比第一轮多复用 64 tokens 的 question-specific suffix page。 -- 第一轮 CP0 写入大量 non-page-aligned suffix node：如 `logical_len=95/81/140/66/86`, `owned_positions=64`。bs=5 时 CP0 常见 `valid_local_tokens=320`，即每请求首个 owner page。 -- 第二轮 CP0 `local_out_cache_loc` 仍按 valid rows 计算，例如 `split_tokens=63 out_cache_tokens=64 local_tokens=63 valid_local_tokens=63`，说明当前 write 侧没有直接把 padding token 当 valid row 写。 - -## 已排除 / 降低优先级 - -- 不是所有 cache-hit 都坏：第一轮已有共享 640-token cache-hit，精度仍正常。 -- 不是明显服务错误或 fallback 慢路径：日志无 fail-fast/fallback/crash。 -- 不是当前 prefill CUDA graph replay：用户确认 prefill 不走 cuda graph。 -- 单纯 “radix 把 698 向上暴露成 704” 不是最强解释：代码仍有 CP exact hit page-floor 方向，日志也可解释为第二轮额外命中了第一轮写入的每题 suffix 首 page。 - -## 当前主嫌 - -bs>1 下 first-run suffix page 的持久化 direct write 与 second-run prefix reuse 不等价： - -1. 第一轮当前请求使用 current K/V 或 current index 参与计算，答案正确。 -2. 同一轮把 suffix 首 page 写入 device/HiCache 持久 cache。 -3. 第二轮把这个 page 当 prefix 复用后精度掉点。 - -因此优先审计： - -- `get_cp_shared_kv_local_out_cache_loc()` -- `select_cp_local_valid_rows_for_cache_write()` -- MLA direct store (`_maybe_write_cp_shared_local_mla_kv`) -- NSA index direct store (`_store_cp_shared_local_index_k_cache`) -- prefix materialize/slot-remap 读取持久 page 时是否按相同 owner-lane order 解释。 - -## 下一步验证方向 - -1. 先补/跑 CPU 单测：bs=5, page_size=64, prefix=640, extend=[95,81,140,66,86]，验证 local valid row selector 输出顺序与 local physical out loc 顺序逐 request/page 对齐。 -2. 如果单测没暴露，加入 env-gated 限频 runtime validator：first-run suffix direct write 后读回刚写的 index/MLA buffer，与 current local rows 做 checksum/row-id 对比。 -3. 不先大范围禁用 current/partial reuse；这会掩盖 direct write 与 persistent reuse 不等价的问题。 - -## 2026-06-07 增量发现：CPU 侧 bs=5 row/loc 顺序探针 - -远端容器已跑过一个不依赖 CUDA 的顺序探针： - -- 脚本：`/sgl-workspace/sglang-tai/tmp_cp_bs5_order_probe.py` -- 参数：`page_size=64`, `cp_size=8`, `prefix=[640]*5`, `extend=[95,81,140,66,86]` -- 结论：Python planner / valid-row selector / local out loc 在该 GSM8K-like 形状下顺序一致。 - -输出摘要： - -```text -rank0: compute=320 valid_rows=320 locs=320 per_req=[64,64,64,64,64] -rank1: compute=320 valid_rows=136 locs=136 per_req=[31,17,64,2,22] -rank2: compute=320 valid_rows=12 locs=12 per_req=[0,0,12,0,0] -rank3-7: valid_rows=0 locs=0 -``` - -含义： - -- rank0 写每个 request 的 suffix 首 page，符合第二轮多命中约 64 tokens/request 的现象。 -- rank1/rank2 写剩余 valid tail；dummy compute-padding rows 没进入 persistent write rows。 -- 这降低了 “Python split/selector 把 valid rows 和 out loc 排错序” 的优先级。 -- 不能排除：TAI fused store、index fused store、或 persistent prefix materialize 对同一批 loc 的解释不一致。 - -若需复跑： - -```bash -ssh g0034 "docker exec sglang-glm5-dev-2 bash -lc 'cd /sgl-workspace/sglang-tai && PYTHONPATH=python python /sgl-workspace/sglang-tai/tmp_cp_bs5_order_probe.py'" -``` - -## 2026-06-07 增量发现：direct write 代码路径事实 - -MLA direct write： - -- 文件：`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py` -- 函数：`_maybe_write_cp_shared_local_mla_kv` -- 流程： - 1. `get_cp_shared_kv_local_out_cache_loc(forward_batch)` 取得 local logical locs。 - 2. `select_cp_local_valid_rows_for_cache_write(...)` 从 compute-padded local rows 中只取 valid rows。 - 3. 优先走 `try_tai_fused_mla_store(...)`，把 logical locs 交给 TAI fused store。 - 4. fallback 才通过 `get_cp_shared_kv_local_physical_out_cache_loc()` 转 physical loc 后调用 pool setter。 - -Index direct write： - -- 文件：`python/sglang/srt/layers/attention/nsa/nsa_indexer.py` -- 函数：`_store_cp_shared_local_index_k_cache` -- 流程： - 1. 取 local logical locs。 - 2. 用同一个 valid-row selector 取 `local_key`。 - 3. 转 physical locs。 - 4. `_store_index_k_cache(... out_loc_override=physical_out_loc)`。 - -当前判断： - -- Python 侧 valid row / local loc 长度与顺序在探针形状下是自洽的。 -- MLA TAI fused store 比 index store 更可疑：它接收 logical locs 并在 kernel 内自行解释 CP shared layout；index store 当前传的是 physical locs。 -- FP8 packed MLA buffer shape 需要重点确认。远端曾出现过 `kv_cache_dim=656` 相关问题，TAI fused MLA store 必须按真实 packed layout 写入，而不能按旧 bf16/compact 假设写。 - -## 2026-06-07 增量发现：日志噪音与非根因 - -最新日志里的 `Invalid` 计数主要来自环境变量拼接错误： - -```text -Invalid value for SGLANG_CP_SHARED_KV_BS_GT1_TIMING_LIMIT: -"-1SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1" is not a valid integer value, using default "256" -``` - -该问题会影响 timing limit/debug 配置，但不是 GSM8K 第二轮掉点的直接根因：同一进程第一轮准确率正常，第二轮 cache 命中更高后才掉点。 - -## 当前最小下一步 - -优先做一个 CUDA/remote 级验证，不继续靠日志猜： - -1. 在 `tai-kernel` 给 `fused_store_mla_kv` 补 GSM8K-like bs=5 非连续 logical loc + FP8 packed shape 的单测。 -2. 对比： - - TAI fused MLA store 写入 raw layer buffer； - - reference path：`CpSharedKVLayout.logical_locs_to_physical()` + 现有 pool setter / torch reference。 -3. 如果 MLA fused store 通过，再对 index fused store 做同形状验证。 - -该验证能直接回答：第一轮写入的 suffix page 是否已经在 persistent KV cache 中损坏。 - -## 2026-06-07 新假设：第二轮掉点可能来自 L2->L1 load，而不是写入 - -用户指出一个关键区分： - -- 第一轮的 cache-hit 可能主要来自同 prefix 的 radix/device cache，仍然正确； -- 第二轮多命中的 question-specific suffix page 可能来自第一轮 backup 到 L2/HiCache 后再 load 回 L1； -- 如果 L2 host layout、page_first_direct load kernel、或 load-back page-slot remap 错，现象就是第二轮掉点且不一定有显式 fallback/error。 - -这解释了为什么日志里“第一轮已有 cached-token 但正确”不能完全排除 cache 路径问题：第一轮命中的 cache 类型可能与第二轮额外命中的 cache 类型不同。 - -需要把验证拆成两类： - -1. **写入验证**：direct write 后立刻从 L1 persistent buffer 读回，确认 first-run suffix page 写入无损。 -2. **L2 roundtrip 验证**：L1 -> L2 backup -> 释放/覆盖 L1 -> L2 -> L1 load 后再读回，确认 host page_first_direct/direct backend 的 layout 与 L1 一致。 - -当前日志曾统计 `HiCache-load=0/CacheCtrl-load=0/load_back=0`，但这只能说明没有这些字符串；不能证明没有发生 HiCache L2 load。下一步需要按实际代码路径找 load/restore 日志点或补 env-gated validator。 - -### L2 roundtrip 单测结果 - -已新增并在远端容器验证： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_fp8_page_first_direct_roundtrip_preserves_kv_and_indexer_pages -s -``` - -结果： - -```text -1 passed, 3 warnings in 10.05s -``` - -该测试覆盖： - -- `NSATokenToKVPool(dtype=torch.float8_e4m3fn, kv_cache_dim=656)` -- `NSATokenToKVPoolHost(layout=page_first_direct, io_backend=direct)` -- 多层 KV bytes：L1 -> L2 backup -> L1 清零 -> L2 -> L1 load -- NSA indexer bytes 同步 roundtrip -- 非连续 page 映射：`src_pages=[1,2,5,8]`, `host_pages=[3,4,7,12]`, `dst_pages=[10,11,13,15]` - -含义： - -- 简单的 TAI `page_first_direct/direct` byte-copy kernel 不是当前最高优先级根因。 -- 仍未排除控制路径错误：HiCache metadata 中 host_indices/device_indices 选择、node split/page-floor 后 stale tail、或 prefix materialize 对 loaded pages 的解释仍可能出错。 - -## 2026-06-07 C: bs=5 valid-row FP8 L2 roundtrip test - -新增并远端验证： - -- `test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_fp8_page_first_direct_bs5_valid_rows_survive_l2_roundtrip` -- 形状：`cp_size=8,page_size=64,bs=5,prefix=[640]*5,extend=[95,81,140,66,86]` -- 覆盖：compute padding、只写 valid rows、FP8 packed MLA KV、NSA indexer page row、`page_first_direct/direct`、L1→L2 backup、L2→L1 load。 -- 远端命令： - `PYTHONPATH=python python -m pytest -q test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_fp8_page_first_direct_bs5_valid_rows_survive_l2_roundtrip -s` -- 结果：`1 passed, 3 warnings in 9.97s`。 - -结论： - -- 真实形状下，valid-row direct-write 之后再做 padded owned-row L2 backup/load，KV 与 index valid rows 均可逐字节保持。 -- 因此“TAI direct L2 raw copy 坏”以及“bs=5 valid-row/padded backup 基本顺序坏”目前不是最高优先级。 -- 剩余更可疑方向： - 1. 线上第二轮 GSM8K 是否真的触发了 `load_back`；此前日志没有 `[HiCache-load]`，需要用最新日志再确认。 - 2. radix/HiCache node metadata 在真实 split/page-floor/evict 后的 `visible_device_indices` 是否与 page table/current-reuse consume 侧一致。 - 3. cache-hit consume 侧（page table、current/partial reuse、index topk offset、MLA materialize）是否在第二轮复用时使用了错误 page/order，而不是 L2 copy 本身损坏。 - -## 2026-06-07 D: latest log still does not show L2 load_back - -复查 `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260606_193039.log`： - -- `[HiCache-load]`: 0 -- `load_back CP`: 0 -- `load_cp`: 0 -- `load_to_device`: 0 -- `CP_SHARED_KV_FAIL_FAST`: 0 -- `CP_SHARED_KV_FALLBACK`: 0 -- `Prefill batch`: 4384 -- `cached-token`: 4384 -- `partial`: 2048 -- `index_topk`: 5007 - -代码事实：`hiradix_cache.load_back()` 的 CP path 会打 `[HiCache-load] load_back CP...` info 日志。当前日志完全没有该前缀。 - -结论：这份 GSM8K 二轮掉点证据目前不支持“第二轮主要来自 L2 load-back 拷贝损坏”。更像是 device/radix cache-hit consume 侧，或 current/partial reuse/index/MLA materialize/page table 的复用语义问题。L2 仍不能完全排除，但优先级下降；如要继续验证，需要在新运行中显式制造 L1 evict 后再复测同一 prompts。 - -## 2026-06-07 E: actual TAI fused MLA store bs=5 test - -新增并远端验证： - -- `test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_fp8_fused_mla_store_matches_fallback_for_bs5_compute_padding_rows` -- 覆盖：`bs=5,prefix=[640]*5,extend=[95,81,140,66,86]`，compute padding 后通过 `get_cp_shared_kv_local_out_cache_loc()` 和 `select_cp_local_valid_rows_for_cache_write()` 得到真实 local valid rows，再调用实际 `runtime.try_tai_fused_mla_store()` 写 FP8 MLA KV。 -- 参考：`quantize_k_cache_separate()` + `set_mla_kv_buffer_triton()` fallback。 -- 远端结果：`1 passed, 3 warnings in 8.76s`。 - -结论：第一轮 prefill 的 TAI fused MLA direct-write 在该 bs=5/tiny/FP8 形状下与 fallback 字节一致。它不是当前最高优先级根因。 - -当前更可疑方向收敛为 cache-hit consume 侧： - -1. page table / visible device indices 在 page-floored cache hit 下是否按 request 边界传递。 -2. index topk 的 current/partial reuse 在 second-run cache-hit 下是否使用了错误 offset/layout。 -3. MLA/index materialize shared cache 时是否用错 logical page order 或 dense output slot。 - -## 2026-06-07 F: CP=8 bs=5 index partial-current compose rank-merge 单测 - -新增并远端验证： - -- `test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_cp8_index_partial_current_compose_matches_rank_merged_reference_for_bs5` -- 形状：`cp_size=8,page_size=64,bs=5,prefix=[640]*5,extend=[95,81,140,66,86]` -- 前缀页按相同 logical page 在 5 个 request 中重复，模拟同 prefix cache hit；suffix/current 页按 owner lane 分配，模拟 CP shared KV direct-write/current-reuse。 -- 测试方式：每个 CP rank 只 materialize 自己拥有的 page，patch collective 为 identity，然后在测试中手工把 8 个 rank 的输出相加，等价模拟 all-reduce 后的 dense page buffer。 -- 远端结果：`1 passed`。 - -结论：当前 CPU/reference index partial-current compose 在 CP=8、bs=5、重复 prefix + owner-lane current suffix 的情况下，rank-merge 后等价完整 dense reference。该路径暂未暴露 index page materialize/order 错误。 - -## 2026-06-07 G: CP=8 KV partial-current remote-current visibility 单测 - -新增并远端验证： - -- `test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_cp8_kv_partial_current_keeps_remote_current_valid_locs_after_reduce` -- 目的：确认本 rank 在 current slot all-reduce 后不会把远端 owner 的 valid current loc 错误 mask 成 `-1`。 -- 远端结果：`1 passed`。 - -结论：`build_current_page_mask()` 当前只 mask 本 rank `current_locs` 所在 page；远端 owner current page 的 valid loc 会保留，符合 all-reduce 后所有 rank 可见 current page 的语义。仍需注意：远端 owner current page 的 tail slack 也不会由本 rank mask，是否安全依赖 index/topk 侧正确使用 valid sequence length，不把 tail slack 选入 `logical_locs`。 - -## 2026-06-07 H: FP8/RAGGED MLA materialization domain mismatch suspect - -代码审计发现一个比 L2 copy 更具体的风险： - -- `get_topk_transform_method()` 在 FP8 + `flashmla_sparse` prefill 下返回 `RAGGED`。 -- RAGGED topk 的 `page_table_1/topk_indices` 是 ragged flattened request/KV 坐标，用于索引 `page_table_1_flattened`，不是 token_to_kv_pool 的 raw logical loc。 -- 但当前 CP shared-KV MLA generic materialize/current-reuse block 在进入 `flashmla_sparse` attention 分支前，无条件把 `page_table_1` 当 `logical_locs` 传给 `materialize_prefix_and_reuse_current_kv_page_slots()` / `materialize_shared_token_kv_buffer()`。 -- 这在 cache-hit/prefix sharing 下会把“ragged 坐标”误当 “KV logical loc”，非常符合：第一轮部分 cache-hit 仍正常、第二轮更高 prefix/suffix cache-hit 后精度掉点。 - -下一步按 TDD 补 source/behavior guard 单测：FP8 RAGGED 路径必须绕过 generic CP MLA materialize，交给后面的 RAGGED `page_table_1_flattened` materialize/dequantize 路径处理；后续再做基于 flattened locs 的 partial-current 优化。 - -## 2026-06-07 I: RAGGED generic MLA materialize guard 已存在 - -继续审计后确认当前本地代码已经有 guard： - -```python -if ( - forward_batch.uses_cp_shared_kv - and topk_transform_method == TopkTransformMethod.PAGED -): -``` - -因此 FP8/`flashmla_sparse` 的 RAGGED topk 不会进入 generic CP MLA materialize/current-reuse block，避免了把 ragged flattened topk 坐标误当 raw KV logical loc。已补回归测试锁住这个 guard。 - -结论：H 是合理风险，但在当前版本已被代码规避；它不是这次第二轮 GSM8K 掉点的剩余根因。继续排查应转向 RAGGED 分支内部的 `page_table_1_flattened` materialize/dequantize、radix visible loc、或 cache-hit 后的 req_to_token/page_table 内容。 - -## 2026-06-07 J: 用户提出的 L2 区分需要更高层单测 - -用户指出：第一轮的 cache-hit 可能主要是同 prefix/device radix cache，第二轮额外 cache-hit 才可能来自 L2/HiCache load。因此仅有 raw L1↔L2 byte-copy roundtrip 还不够，需要补一个更贴近真实 consume 的测试： - -1. first-run 形状写入 FP8 packed MLA KV + NSA index valid rows； -2. 备份 full padded owned pages 到 page_first_direct/direct host； -3. 清空/污染 L1 后从 host load 到新的 L1 pages； -4. 用 CP shared-KV `materialize_shared_token_kv_buffer()` 按 RAGGED `page_table_1_flattened` 风格 logical locs 读取 loaded pages； -5. 对比 materialize 后 dense loc 对应 bytes 与 first-run 写入 bytes。 - -目的：直接验证“L2 load 本身正确，但 load 后被 RAGGED/MLA materialize 或 logical→physical remap 错读”的可能性。若该测试通过，L2 load 作为 GSM8K 二轮掉点根因的优先级继续降低，后续应重点看真实运行中的 page_table/topk offset/current reuse 控制流。 - -### L2-loaded suffix + RAGGED materialize 单测结果 - -新增并在远端验证： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_fp8_l2_loaded_bs5_suffix_materializes_from_ragged_logical_locs -s -``` - -结果：`1 passed, 3 warnings in 9.82s`。 - -覆盖内容： - -- GSM8K-like `bs=5,prefix=[640]*5,extend=[95,81,140,66,86]`； -- CP=8 owner-lane suffix valid rows； -- FP8 packed MLA KV (`kv_cache_dim=656`)； -- `page_first_direct/direct` L1→L2 backup，再 L2→L1 load 到新的 logical pages； -- 之后按 RAGGED `page_table_1_flattened` 风格的 token logical locs 调 `materialize_shared_token_kv_buffer()`； -- 手工合并 8 个 rank 的 local materialize 输出，验证 dense loc 对应 bytes 与 first-run 写入 bytes 一致。 - -结论：在该单测覆盖的控制面里，L2-loaded suffix page 被 RAGGED MLA materialize 读取是逐字节正确的。当前 GSM8K 二轮掉点更不像 raw L2 copy 或 basic RAGGED logical-loc materialize 错；下一步应看真实运行中的 `page_table_1_flattened/topk_indices_offset` 是否在 bs>1/cache-hit 后按 request 正确构造，以及 decode transfer/FP8 dtype 协议是否一致。 - -## 2026-06-07 K: RAGGED fused topk offset 语义验证 - -新增并在远端验证： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/layers/test_nsa_topk_transform.py::TestNSATopkTransform::test_ragged_topk_transform_offsets_are_request_relative_with_row_starts -s -``` - -结果：`1 passed, 5 warnings in 9.77s`。 - -验证点：`fast_topk_transform_ragged_fused` 在传入 `row_starts` 时，会把选中的 compact column 转成： - -```text -request_offset + (selected_compact_col - row_start) -``` - -而不是： - -```text -request_offset + selected_compact_col -``` - -结论：当前 `_get_topk_ragged_with_cp()` 给 batch compact path 传 `topk_indices_offset_override=request_kv_base` 的方向是对的；不需要改成 `request_kv_base - k_base`。这降低了“RAGGED topk offset 基本语义错误”作为二轮 GSM8K 掉点根因的优先级。 - -## 2026-06-07 L: TAI batched index MQA prepare GSM8K-like 形状验证 - -新增并在远端验证： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_tai_batched_index_mqa_prepare_matches_getk_gets_reference_gsm8k_bs5 -s -``` - -结果：`1 passed, 3 warnings in 10.00s`。 - -覆盖内容： - -- `bs=5`, 每请求 `seq_len=704`, `q_start=640`, `q_len=64`； -- `page_size=64`, `index_head_dim=128`； -- 实际调用 `try_tai_prepare_cp_mqa_index_batch()`； -- 与 reference 路径逐项对比：`index_buf_accessor.GetK/GetS`、`ks`、`ke_offset`。 - -结论：至少在 GSM8K-like cache-hit/tiny-extend 形状下，TAI batched index prepare kernel 的 K/S compact 与 range descriptor 与 Python reference 一致。当前二轮掉点不应优先归因于该 kernel 的基本 gather/descriptor 错误。 - -## 2026-06-07 M: L2 与 persistent index cache 需要区分验证 - -用户指出第一轮 GSM8K 里的 cache hit 可能主要来自相同 prefix 的 L1/device radix cache，而第二轮新增的更高命中率可能来自 L2/HiCache load；因此不能只用第一轮正常来排除 L2/load 路径。 - -当前已覆盖并通过的单测降低了以下路径优先级：raw page_first_direct L1↔L2 copy、L2-loaded FP8 MLA KV 经 RAGGED logical loc materialize、RAGGED topk offset、TAI batched index MQA prepare。下一步需要补更贴近第二轮 consume 的 persistent NSA index cache 写入/读取单测，尤其是 `fused_store_index_k_cache` 在 bs=5/tiny/FP8/physical loc direct-write 下是否与 fallback `act_quant + SetKAndS` 等价。 - -原因：第一轮 topk 可以使用 freshly computed `current_index_kv/local_key`，即使 persistent index cache 写坏也不一定立即影响；第二轮高 cache-hit 时会更依赖持久化 index cache，这与“第一轮 0.95、第二轮 0.69”的症状吻合。 - -### M 追加验证结果：persistent index fused store 基本路径通过 - -新增并在远端验证： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_fp8_index_fused_store_persistent_pages_survive_bs5_materialize -s -``` - -结果：`1 passed, 3 warnings in 10.22s`。 - -覆盖内容：bs=5/tiny extend、CP=8、compute padding、`get_cp_shared_kv_local_out_cache_loc()`、`select_cp_local_valid_rows_for_cache_write()`、`logical_locs_to_physical()`、`fused_store_index_k_cache()`、`materialize_shared_paged_buffer()` rank merge、`GetK/GetS` consume。 - -结论：不经过 L2 的 persistent NSA index fused-store + paged materialize 基本路径未复现行顺序/page offset 错误。单测仍保留，因为它能锁住第一轮写 persistent index cache 的核心合同。 - -## 2026-06-07 N: L2-loaded persistent index cache 单测 - -新增并在远端验证： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_fp8_l2_loaded_index_pages_materialize_after_fused_store_bs5 -s -``` - -结果：`1 passed, 3 warnings in 10.33s`。 - -覆盖内容： - -- 第一轮形状用 `fused_store_index_k_cache()` 写入 persistent index pages； -- `page_first_direct/direct` 备份到 L2 host； -- 污染目标 L1 pages 后从 L2 load 到新 L1 pages； -- 对 loaded index pages 走 `materialize_shared_paged_buffer()` + rank merge； -- 通过 `GetK/GetS` 按 request page table 读取并反量化，对比原始 valid key rows。 - -结论：在单测覆盖的 GSM8K-like bs=5/tiny/FP8 形状中，persistent index cache 经 fused store → L2 backup/load → paged materialize → GetK/GetS 后仍可还原 valid rows。用户提出的“第二轮来自 L2 cache load 的错误”目前没有被 KV 或 index 的直接单测支持；下一步应把排查焦点上移到真实运行中的 radix/req_to_token/page_table 可见范围、chunked/full prompt cache-hit 后 valid length 与 page table 的组合、以及 prefill→decode transfer 元数据。 - -## 2026-06-07 O: 下一层 runtime 证伪点应放在 batch topk membership - -前面单测已经覆盖并通过：FP8 KV/index page_first_direct L2 roundtrip、persistent index fused-store、L2-loaded persistent index materialize、RAGGED topk offset、TAI batched index MQA prepare。当前还没有证据支持 raw L2 copy 或基本 persistent cache 布局损坏。 - -新的高优先级可证伪点是 `_get_topk_in_seq_cp_pair_batch()` 产出的 `topk_indices` 是否仍严格落在对应 request 的可见前缀内。这个检查能直接区分： - -- topk/index/range/offset/scatter 侧跨 request 或未来 token； -- MLA KV / prefill→decode transfer 侧错误。 - -计划：复用现有 `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG`，在 batch topk scatter 后增加 fail-fast validator。对每个 compact span，允许 `-1`，其余值必须满足： - -```text -request_base <= topk < request_base + absolute_query_position + 1 -``` - -其中 `request_base=sum(seq_lens_cpu[:req_id])`。若违反，抛： - -```text -[CP_SHARED_KV_FAIL_FAST][index_topk] reason=batch_gt1_topk_membership -``` - -该 validator 是 debug-gated，预期只在 ETE 定位时打开；不会新增 collective，也不会改变正常 hot path。 - -### O 追加实现/验证结果 - -已实现 debug-gated runtime validator：`_validate_cp_shared_kv_batch_topk_membership()`。 - -接入点：`_get_topk_in_seq_cp_pair_batch()` compact topk scatter 后，且仅在 `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1` 且非 CUDA graph capture 时运行。 - -验证：先在远端只同步测试，确认新测试因 helper 缺失失败；随后实现 helper 与接入后远端通过： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/layers/test_nsa_topk_transform.py -s -``` - -结果：`7 passed, 5 warnings in 7.29s`。 - -下一轮 ETE 如果仍然掉点但没有 `reason=batch_gt1_topk_membership`，就可以进一步降低 batch topk membership/cross-request/future-token 的优先级，把焦点移到 page table flatten 与 prefill→decode transfer metadata。 - -## 2026-06-07 P: Mooncake KV page-count 有 fail-fast，但 NSA state page-count 仍可能静默错配 - -继续审查 prefill→decode transfer runtime 发现：KV chunk 路径已经调用 `validate_transfer_page_count_or_raise()`，但 `maybe_send_extra()` 里的 `state_type in ["swa", "nsa"]` 分支仍只有旧逻辑：当 `prefill_state_indices` 少于 `dst_state_indices` 时只 warning，然后继续走 transfer。 - -对 CP shared-KV + NSA 来说，`state_indices` 是 index/state page 的 companion payload；如果 KV pages 数量正确但 state pages 错配，decode 侧仍可能拿到错误/缺失的 NSA index cache。这个路径与“第一轮主要 fresh current，第二轮更依赖 persisted/cached index/state”症状相容，需要 fail-fast 而不是 warning。 - -计划：复用已有 `validate_transfer_page_count_or_raise()`，在 CP shared-KV state transfer（调用方传入 `dst_state_indices`，即按 logical positions 选择后的目标 pages）时对 `prefill_state_indices` 和 `dst_state_indices` 做同样的 page-count fail-fast，path 标为 `mooncake_state`。非 CP shared-KV 暂不改变旧行为，避免扩大影响面。 - -### P 追加实现/验证结果 - -已在 `MooncakeKVManager.maybe_send_extra()` 的 `state_type in ["swa", "nsa"]` 分支补 CP shared-KV state page-count fail-fast：当调用方传入 `dst_state_indices`（即 CP shared-KV 已按 logical page positions 选择目标 state pages）时，`prefill_state_indices` 与 `dst_state_indices` 数量必须完全一致，否则抛： - -```text -[CP_SHARED_KV_FAIL_FAST][mooncake_transfer_page_count_mismatch] path=mooncake_state -``` - -TDD 验证：先同步新测试到远端，确认旧代码只 warning、不 raise；实现后远端通过： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/disaggregation/test_cp_shared_kv_transfer_mapping.py -s -``` - -结果：`10 passed, 3 warnings in 5.27s`。 - -这还不是 root-cause 结论，但它补掉了一个真实 silent-corruption 风险：KV page-count 正确时，NSA state/index companion pages 仍可能静默错配。 - -## 2026-06-07 Q: CP shared-KV NSA state buffer-count mismatch 也不应静默截断 - -继续审查 `maybe_send_extra()`：除 page-count mismatch 外，`src_state_data_ptrs` 与 `dst_state_ptrs` buffer 数量不一致时，当前逻辑只 warning，然后截断到 `min(src,dst)` 个 buffer 继续传输。 - -在 CP shared-KV + EAGLE/draft/shared NSA state 场景，这同样属于 silent corruption 风险：page 数量一致也不能保证 layer/state buffer 数量一致。若少传部分 state buffer，decode 侧可能继续运行但使用不完整的 NSA/draft companion state。 - -计划：仍只收窄到 CP shared-KV state transfer（`dst_state_indices is not None`），将 buffer-count mismatch 改为 fail-fast；非 CP shared-KV 旧路径暂不扩大影响。 - -### Q 追加实现/验证结果 - -已在 CP shared-KV state transfer 中把 state buffer-count mismatch 从 warning+truncate 改为 fail-fast： - -```text -[CP_SHARED_KV_FAIL_FAST][mooncake_state_buffer_count_mismatch] -``` - -TDD 验证：先同步新测试到远端，确认旧代码只 warning、不 raise；实现后远端通过组合测试： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/layers/test_nsa_topk_transform.py \ - test/registered/unit/disaggregation/test_cp_shared_kv_transfer_mapping.py -s -``` - -结果：`18 passed, 5 warnings in 7.21s`。 - -这进一步收窄了 decode 侧 silent corruption 风险：CP shared-KV 下 state pages 数量和 state buffer 数量都不允许静默截断。 - -## 2026-06-07 R: L1↔L2 路径当前测试缺口收敛到 controller 级批量 transfer - -继续按“不在生产热路径随意加 check”的约束阅读代码后，当前 L1→L2/L2→L1 真实路径是： - -- 写入/backup：`HiRadixCache.prepare_write_backups_for_reqs()` 为 bs>1 每个 request 单独 `reserve_write_cp()`，随后 `HiCacheController.submit_write_cp_per_layer(catch_up_all_layers=False)` 注册；真正 D2H 在 `on_layer_end()` 中通过 `_submit_write_cp_layer_states()` 把多个 reservation 的 `host_indices/physical_device_indices` concat 成一次 per-layer transfer。 -- 读取/load：`HiRadixCache.load_back()` 调 `HiCacheController.load_cp()`，按 metadata.page_owners 用 `alloc_pages_with_owners()` 复现 owner pattern，随后 `start_loading()` 通过 `CacheOperation.merge_ops()` 把多个 queued load op concat 成一次 per-layer H2D transfer。 -- 既有 CUDA 单测覆盖了 raw `NSATokenToKVPoolHost.backup_from_device_all_layer()` / `load_to_device_per_layer()`、L2-loaded MLA/index materialize、fused store 等；但缺少 controller 级“多个 CP HiCache node 的 per-layer grouped D2H + merged H2D”回归。 - -因此下一步单测不新增生产检查，而是构造 bs=5/GSM8K-like node reservations，走真实 `reserve_write_cp()` → grouped `on_layer_end()` backup → `load_cp()` × 多 node → merged `start_loading()`，验证 valid rows 与 index rows 在 L1→L2→L1 后仍按 request/node 顺序正确。 - -## 2026-06-07 S: controller-level L2 roundtrip test first run hit test-allocation artifact - -新增 controller 级 grouped bs=5 L1→L2→L1 单测后，远端首次运行在 host pool 初始化后的下一次 CUDA op 报： - -```text -torch.AcceleratorError: CUDA error: part or all of the requested memory range is already mapped -``` - -定位：这是测试环境里 `NSATokenToKVPoolHost(pin_memory=True)` 默认走 `cudaHostRegister`，同一 pytest 进程/循环中 CPU allocator 可能复用已注册地址导致的注册冲突；不是业务 L1/L2 数据不一致证据。既有测试已有模式：在这类 pin-memory host-pool 单测里临时把 `ALLOC_MEMORY_FUNCS["cuda"]` 切到 `alloc_with_pin_memory`，结束后恢复。 - -下一步只修正单测分配方式，不改生产路径。 - -## 2026-06-07 T: controller-level grouped bs=5 L1→L2→L1 path verified - -新增 controller 级单测并在远端通过： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_cp_hicache_controller_grouped_bs5_l2_roundtrip_preserves_valid_rows -s -``` - -结果：`1 passed, 3 warnings in 8.10s`。 - -覆盖的真实控制路径： - -- `CPSharedPagedTokenToKVPoolAllocator.alloc_extend_compute_owner()` 生成真实 bs=5 out_cache_loc； -- `HiCacheController.reserve_write_cp()` 对每个 request 做 page padding、owner-lane physical mapping、host slot reservation； -- `submit_write_cp_per_layer(catch_up_all_layers=False)` 注册多个 node； -- `on_layer_end()` 把多个 reservation grouped 后 per-layer D2H 写入 host； -- 覆盖 L1； -- `load_cp()` 多 node 排队，`start_loading()` 通过 `CacheOperation.merge_ops()` 合并后 per-layer H2D load 回 L1； -- 校验每个 node 的 valid owned KV/index rows。 - -结论：在 GSM8K-like bs=5、FP8 packed KV、page_first_direct/direct、controller grouped backup/load 条件下，L1→L2→L1 数据 roundtrip 没有暴露损坏。因此当前掉点 root cause 继续从 radix/HiCache node 元数据、cache-hit consume 侧、或 prefill→decode transfer 语义排查；不继续往生产热路径加常态 check。 - -## 2026-06-07 U: 定位阶段不保留新增 runtime validator/fail-fast - -根据最新约束：“不能随便引入 check，这些冗余操作会导致正式推理时性能下降”。本轮未提交的 runtime validator / state fail-fast 已从本地和远端同步回退，当前只保留： - -- 临时排查文档； -- 不影响生产 hot path 的 L1↔L2 controller grouped CUDA 单测。 - -原则：后续继续用单测/benchmark 暴露问题；如果需要 runtime 证据，只能走已有 debug env 且明确限频，不加入默认路径。 - -## 2026-06-07 V: L2→L1 load_back exact-capacity path 仍触发同步 free-room eviction - -远端组合运行： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_cp_hicache_controller_grouped_bs5_l2_roundtrip_preserves_valid_rows \ - test/registered/unit/mem_cache/test_cp_hicache_load_back_owner_lanes.py -q -``` - -结果：controller grouped L1↔L2 单测通过，但既有 CPU 单测失败： - -```text -TestCpHiCacheLoadBackOwnerLanes.test_load_back_does_not_synchronously_evict_for_l1_free_room_when_exact_capacity_fits -expected evicted_device_indices=[] -got [tensor([8, 9, 10, 11])] -``` - -含义：`HiRadixCache.load_back()` 在 owner-lane exact capacity 已满足、`deficit_by_owner=[0,...]` 时，仍然因为 `free_room_deficit_by_owner` 做同步 L1 eviction。这会把 L2→L1 load-back 的 scheduler/control path 变重，和当前“不能引入冗余热路径操作”的约束冲突。 - -修复方向：load-back 的 L1 free-room 信号应保持 advisory；exact capacity 已满足时不为了补 free-room 在 load_back 中同步 evict。真正容量不足仍走 exact deficit eviction。 - -### V 追加实现/验证结果 - -已做最小修复：删除 `HiRadixCache.load_back()` 中 exact owner-lane capacity 已满足时的同步 L1 free-room eviction，只保留 stage 记录为 advisory。容量不足时的 `deficit_by_owner` eviction 路径不变。 - -远端验证通过： - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_cp_hicache_controller_grouped_bs5_l2_roundtrip_preserves_valid_rows \ - test/registered/unit/mem_cache/test_cp_hicache_load_back_owner_lanes.py -q -``` - -结果：`12 passed, 3 warnings`。 - -结论：L1↔L2 数据 roundtrip 正确；L2→L1 control path 发现并修复了一个同步 eviction 过重问题。该修复是删除冗余热路径操作，不是新增 check。 - -## 2026-06-07 W: 增加默认关闭的 CP HiCache cache-hit 定位日志 - -当前 L1↔L2 controller grouped bs=5 roundtrip 单测通过，仍未解释 GSM8K 第二轮 cache hit 后掉点。为避免继续盲查，本轮只增加默认关闭、受 `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG` 和 `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT` 控制的限频日志，不加入默认 runtime check。 - -新增观测点： - -- `match_prefix_result`：每次 CP HiCache match 返回的 device 命中长度、host hit 长度、last_device_node、last_host_node、deferred node、rid（schedule_policy 已把 req 传给 MatchPrefixParams）。用于区分第二轮掉点是 L1 resident hit、L2 load_back hit，还是 pending-backup defer。 -- `match_floor_valid_tail`：valid-tail 被 page floor 的节点、原始 prefix、floor 后 prefix、host_len/padded_len/pending_write。用于确认 cache 以 page 为最小单位时是否仍暴露了 sub-page tail。 -- `init_load_back_start/loaded/unloaded` 和 `load_back_plan`：L2→L1 load_back 的 rid、节点链、owner-lane required/available/deficit/free-room。用于确认第二轮 hit 是否真的从 L2 加载，以及 exact/free-room 行为。 -- `prepare_write_backup`、`attach_prepared_backup`、`commit_pending_backup`：写入侧 node_id、logical_len、padded_len、host_indices、owned_positions、page owners。用于对齐“写入的 node 元数据”和“后续 match/load 消费的 node 元数据”。 - -启动方式示例： - -```bash -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1 \ -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT=256 \ -... -``` - -风险控制：日志全部默认关闭，限频；不做 tensor 内容校验，不引入 H2D/D2H 冗余检查，不改变 cache 行为。 - -### W 追加验证 - -远端已同步 `hiradix_cache.py`、`schedule_policy.py` 和本文档，并验证： - -```bash -python -m py_compile python/sglang/srt/mem_cache/hiradix_cache.py \ - python/sglang/srt/managers/schedule_policy.py - -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1 \ -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT=8 \ -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_nsa_pool_host_unit.py::TestNSAHiCacheTransfer::test_cp_hicache_controller_grouped_bs5_l2_roundtrip_preserves_valid_rows \ - test/registered/unit/mem_cache/test_cp_hicache_load_back_owner_lanes.py::TestCpHiCacheLoadBackOwnerLanes::test_load_back_does_not_synchronously_evict_for_l1_free_room_when_exact_capacity_fits -q -``` - -结果：`2 passed, 3 warnings`。debug env 打开时不会破坏对应 L1↔L2 / load_back 单测路径。 - -## 2026-06-07 X: GSM8K 两轮后仍掉点，当前日志暴露的问题 - -用户在同一进程上连续两轮完整 GSM8K： - -- 第一轮：Accuracy `0.955`，Invalid `0.001`，Latency `208.152s`，throughput `669.457 token/s`。 -- 第二轮：Accuracy `0.684`，Invalid `0.003`，Latency `216.646s`，throughput `658.218 token/s`。 - -远端最新日志：`/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_103959.log`。 - -### X1 debug env 实际未生效 - -日志中没有真正的 `[CP_SHARED_KV_BS_GT1_DEBUG][hicache]` 事件，只有如下 warning 重复出现： - -```text -Invalid value for SGLANG_CP_SHARED_KV_BS_GT1_TIMING_LIMIT: "-1SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1" is not a valid integer value -``` - -说明启动命令里 `SGLANG_CP_SHARED_KV_BS_GT1_TIMING_LIMIT=-1` 和 `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1` 中间缺少空格/续行，导致 debug env 被拼进 timing limit；本轮无法用新增日志判断 match/load_back 细节。 - -正确写法需要类似： - -```bash -SGLANG_CP_SHARED_KV_BS_GT1_TIMING_LIMIT=-1 \ -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1 \ -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT=512 \ -... -``` - -### X2 普通日志未见显式 crash/fallback，但第二轮大量 cache hit - -普通日志无 `Traceback` / `RuntimeError` / `FALLBACK` / `FAIL_FAST`。按 CP0 的 `Prefill batch` 聚合： - -```text -minute cp0_batches cp0_newseq cp0_newtok cp0_cached cached_batches -2026-06-07 10:45 38 182 23744 115200 36 -2026-06-07 10:46 77 385 49216 245760 77 -2026-06-07 10:47 78 390 50112 248320 78 -2026-06-07 10:48 74 368 48576 234240 74 -2026-06-07 10:49 65 317 40832 241024 64 -2026-06-07 10:50 77 385 25024 269312 77 -2026-06-07 10:51 77 383 41088 269312 77 -2026-06-07 10:52 51 244 16128 169856 49 -``` - -第二轮缓存命中比例更高，符合“cache hit 后掉点”的现象。 - -### X3 可疑现象：短 extend 写入时只有 CP0 拥有 padded page - -日志中大量短 suffix： - -```text -node_id=6493 logical_len=8 CP0 owned_positions=64, CP1-7 owned_positions=0 -node_id=6494 logical_len=37 CP0 owned_positions=64, CP1-7 owned_positions=0 -node_id=6495 logical_len=21 CP0 owned_positions=64, CP1-7 owned_positions=0 -node_id=6496 logical_len=28 CP0 owned_positions=64, CP1-7 owned_positions=0 -``` - -这可能是合法的 page-minimum ownership，也可能说明 batch/page padding 后 owner 选择使用了 relative page id，导致短 suffix 的第一个 padded page 总是 owner 0，而不是按全局 page/sequence position 的 owner lane。需要回到 `build_in_seq_page_compute_owners()`、`alloc_extend_compute_owner()`、`reserve_write_cp()`、`get_cp_shared_kv_local_out_cache_loc()` 的 prefix/global-page 语义确认。 - -下一步：修正启动 debug env 后再跑一轮或小规模重复 GSM；同时从代码侧验证“短 suffix padded page 是否应总是 CP0 owner”。 - -## 2026-06-07 X: 最新 GSM8K 两轮日志复查（`20260607_103959`） - -用户再次连续跑全量 GSM8K，同一 prefill 进程： - -- 第 1 轮：`Accuracy=0.955`, `Invalid=0.001`, `Latency=208.152s`, `Output throughput=669.457 token/s` -- 第 2 轮：`Accuracy=0.684`, `Invalid=0.003`, `Latency=216.646s`, `Output throughput=658.218 token/s` - -远端日志：`g0034:/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_103959.log`。 - -复查事实： - -- `FALLBACK=0`, `FAIL_FAST=0`, `RuntimeError=0`。 -- `HiCache-load=0`, `load_back=0`, `CacheCtrl-load=0`；这份日志仍没有显式 L2->L1 load-back 证据。 -- `CP_SHARED_KV_BS_GT1_DEBUG` 没有真正生效。启动环境里出现拼接错误： - - ```text - Invalid value for SGLANG_CP_SHARED_KV_BS_GT1_TIMING_LIMIT: - "-1SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1" is not a valid integer value - ``` - - 因此日志里没有 `[CP_SHARED_KV_BS_GT1_DEBUG][hicache] match_prefix/load_back/...` 事件，不能用本轮日志判断真实 radix match/load_back 元数据。 - -- CP0 `Prefill batch` 按分钟统计显示两轮之间 cache-hit 确实上升： - - ```text - 10:45-10:48: cached/new 约 115200/23744 -> 248320/50112 - 10:49-10:52: cached/new 约 241024/40832 -> 269312/41088 -> 169856/16128 - ``` - -- 10:50 左右 timing 日志显示大量 tiny suffix batch：`split_tokens=89/126/163/175/...`，`compute_padding=True`，CP0 持有几乎全部 valid local rows，CP1 只有少量 tail row，符合当前 page-minimal valid-owner 设计。 -- 两轮结束后出现 detokenizer health failure 与 503 shutdown： - - ```text - 11:00:34 Health check failed. Server couldn't get a response from detokenizer for last 20 seconds. - 11:00:37 Shutdown: finished 0 streaming requests normally, aborted 2 requests with 503. - 11:00:37 serving_chat.py:818 AttributeError: 'int' object has no attribute 'name' - ``` - -判断： - -- 精度掉点仍然与第二轮更高 cache-hit 强相关。 -- 本轮日志不能证明 L2 load-back 参与；没有 `[HiCache-load]`。当前更像 L1/radix cache-hit consume 侧或 prefill->decode transfer/decoder consume 侧语义问题。 -- 新暴露的 detokenizer hang/503 shutdown 是独立稳定性问题，会污染尾部请求，但不能单独解释第二轮 `0.684` 的大幅掉点。 -- 下一轮若要靠 runtime 日志定位，必须修正 env 换行： - - ```bash - SGLANG_CP_SHARED_KV_BS_GT1_TIMING_LIMIT=-1 \ - SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1 \ - SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT=512 \ - ... - ``` - - -## 2026-06-07 Y: 重启后 debug 生效，掉点仍然集中在 L1 device suffix page hit - -远端最新日志：`g0034:/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_112229.log`，prefill PID `538216`。 - -启动环境已确认包含： - -```text -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1 -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT=-1 -SGLANG_CP_SHARED_KV_CURRENT_REUSE=1 -SGLANG_CP_SHARED_KV_FUSED_INDEX_MQA_PREPARE=1 -SGLANG_CP_SHARED_KV_FUSED_MLA_STORE=1 -SGLANG_CP_DRAFT_SHARED_KV=1 -SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER=1 -``` - -### Y1 日志计数结论 - -```text -Traceback: 0 -RuntimeError: 0 -FAIL_FAST: 0 -FALLBACK: 0 -Health check failed: 0 -HiCache-load/load_back/CacheCtrl-load: 0 -match_prefix_result: 46464 -match_floor_valid_tail: 512 -send_kv_chunk: 21344 -prefill_result_handoff: 4480 -``` - -结论：本轮 debug 生效，且没有显式 fallback/crash；也没有 L2->L1 load_back 证据。GSM8K 第二轮掉点不能继续优先归因于 L2 load-back 数据损坏。 - -### Y2 两轮 GSM8K 的 CP0 prefill 形态 - -按 CP0 `Prefill batch` 聚合，第一轮约 `11:34-11:38`，第二轮约 `11:43-11:47`： - -```text -第一轮 11:34-11:38: newtok≈171392, cached≈843520 -第二轮 11:43-11:47: newtok≈86016, cached≈928896 -``` - -第二轮 new token 明显下降、cached token 上升，符合“cache hit 后精度下降”的现象。 - -### Y3 关键差异：第二轮大量命中 first-run 写入的 device suffix page - -过滤 CP0 且 `rid != None` 的 `match_prefix_result` 后： - -- 第一轮绝大多数真实请求只命中公共 prompt 的 `640` token device prefix；`last_device_host_len=640`。 -- 第二轮大量请求命中 `704/768/832` token device prefix；其中新增的 `64/128/192` token 来自第一轮写入的 suffix page node，`last_device_host_len=64/128/192`。 -- `host_hit_length` 始终为 `0`。 - -代表样例： - -```text -# 第一轮 -rid=09be43... key_len=697 device_tokens=640 last_device_node=8 last_device_host_len=640 - -# 第二轮 -rid=47c543... key_len=734 device_tokens=704 last_device_node=2650 last_device_host_len=64 -rid=5df882... key_len=779 device_tokens=768 last_device_node=2653 last_device_host_len=128 -``` - -### Y4 send_kv_chunk 与 transfer 侧没有暴露 page count mismatch - -第二轮 `send_kv_chunk` 的 `kv_page_count` 与 `state_page_count` 一致，且没有 mooncake transfer fail-fast/fallback。第二轮 extend 变得很短： - -```text -11:43 ext_min_avg_max=(1, 27.4, 65) -11:44 ext_min_avg_max=(2, 28.0, 65) -11:45 ext_min_avg_max=(2, 27.2, 65) -11:46 ext_min_avg_max=(2, 27.4, 65) -``` - -当前 root-cause 方向收窄为：第一轮写入并留在 L1 的 suffix page node，在第二轮作为 device cache prefix 被消费时语义错误。优先检查： - -1. radix/HiCache split 后 `node.value` 与 `cp_hicache` metadata 是否仍表示同一 token span； -2. page-floor 后 suffix node 的 `owned_positions` 是否相对 node-local 正确偏移； -3. cache-hit consume 时 `prefix_indices` / `req_to_token` 是否按全局 token 顺序拼接这些 suffix page； -4. first-run insert of `640 + short suffix` 到 second-run match `704/768` 的最小单测。 - -不要再重复优先排查 L2 load-back；本轮日志没有该路径参与证据。 - -## 2026-06-07 Z: root-cause direction refined to L1 persistent direct-write/read - -Latest debug-enabled run (`20260607_112229`) shows: - -- no `HiCache-load` / `load_back` / `CacheCtrl-load` events; -- no `FALLBACK` / `FAIL_FAST` / `RuntimeError`; -- second GSM8K run has far more device cache hits and short extends; -- second run consumes first-run suffix pages as L1 device prefix (`device_tokens=704/768/832`, `host_hit_length=0`). - -This makes L2 load-back unlikely for the accuracy drop. A stronger hypothesis is: - -1. first run computes prompt/current suffix correctly because attention/index paths use freshly produced current KV/index rows in partial-current compose; -2. first run also writes those rows into the persistent L1 CP shared-KV pool through direct-write/fused-store; -3. second run hits those persisted device pages and reads them as prefix; -4. if direct-write layout/offset/FP8 packing is wrong, first run can remain accurate while second run drops. - -Next inspection target: direct-write store/read roundtrip for `page_first_direct`, `fp8_e4m3`, batch/compute-padding, especially MLA KV fused store and NSA index K/scale store. Do not spend more cycles on L2 load-back for this run unless new logs show `HiCache-load`. - -## 2026-06-07 AA: cp_floor_exact false is internal refresh; transfer confirms second run sends fully cached prompts - -Latest log recheck for `20260607_112229`: - -- `cp_floor_exact=False` `match_prefix_result` events all have `rid=None`: - - `('False', rid_is_None=True, device_tokens_not_page_aligned)=20760` - - `('True', rid_is_None=False, device_tokens_page_aligned)=25336` -- Scheduler-visible request matches (`rid != None`) are page-aligned. The non-page results are from the post-insert/cache refresh path and are not direct scheduler planning inputs. -- First GSM8K run CP0 send shape: - - 1318/1319 requests transfer with `fill_len - extend_input_len = 640`. - - Average extend sent/computed ≈ 92 tokens. -- Second GSM8K run CP0 send shape: - - prefix distribution from `fill_len - extend_input_len`: `704:1154`, `768:82`, `640:81`, `832:2`. - - Average extend ≈ 27 tokens. -- `send_kv_chunk` runs after cache insertion refresh; its logged `prefix_len=len(req.prefix_indices)` often equals full `fill_len` in the second run, so decode is receiving page tables that can be entirely first-run cached prompt pages, not just freshly computed tail rows. - -Implication: the current strongest path is not L2 load-back and not scheduler non-page prefix planning. The second run transfers and consumes first-run persisted L1 prompt pages (question suffix pages) far more aggressively. Root-cause search should focus on persistent L1 writes/readback/transfer for bs>1 compute-padding FP8 CP shared KV: - -1. MLA KV fused persistent store -> cached prefix materialize / transfer. -2. NSA index fused persistent store -> cached prefix materialize / top-k. -3. Radix post-insert refresh replacing freshly computed tail with full cached prompt before transfer. - -Need a focused test that writes bs=5 compute-padded valid rows into persistent L1, then reads/materializes the same pages as a later cache-hit prefix. Existing tests cover index store+materialize and MLA store-vs-fallback, but do not yet prove FP8 MLA persistent pages survive the second-run cache-hit materialize/transfer shape. - -## 2026-06-07 AB: added MLA persistent-page cache-hit unit; collective materialize path passes - -Added `test_fp8_mla_persistent_pages_survive_bs5_cache_hit_materialize` to cover the missing MLA side of the second-run cache-hit shape: - -- bs=5, `prefix_lens=[640]*5`, `extend_lens=[95,81,140,66,86]`; -- writes valid rows through TAI fused MLA persistent store after compute-padding selection; -- materializes only full cached suffix pages as a later page-floored cache-hit prefix; -- simulates CP owner merge by summing per-rank dense buffers with IPC disabled. - -Remote CUDA result on `g0034` container: - -```text -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_fp8_mla_persistent_pages_survive_bs5_cache_hit_materialize -1 passed, 3 warnings in 8.76s -``` - -This weakens the hypothesis that the ordinary fused MLA store + local materialize + all-reduce path corrupts bs=5 FP8 persistent L1 pages. Remaining likely areas: - -1. runtime path difference not covered by the unit, especially TAI IPC materialize when a single prefix span is used; -2. decode/disaggregation transfer consuming cached prompt pages after `cache_finished_req` refreshes `req_to_token` to full cached prompt; -3. draft/target transfer parity for cached-prefix pages; -4. index runtime path still has a CUDA persistent-page test, but it disables IPC and may not cover the exact online materialized shared-buffer path with all environment flags. - -## 2026-06-07 AC: new process ready; raw-output based failure classification required - -New remote prefill log is `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_131732.log`. - -Startup evidence: - -- server reached `The server is fired up and ready to roll!` at log line 4334; -- no startup `Traceback` / `RuntimeError` / `FAIL_FAST` / `FALLBACK` marker before readiness; -- fp8 KV cache still warns that no scaling factors were provided and scaling defaults to 1.0 on all ranks. This warning alone cannot explain the second-run-only accuracy drop because first-run fp8 accuracy was normal, but it remains part of the runtime context. - -Existing raw result file found: - -- `/mnt/beegfs/cjy/gsm8k_bench_sglang_full_raw_20260606_132651.jsonl`: 1319 rows, accuracy 0.955; its failures are ordinary wrong GSM8K reasoning/answers, not obvious repetitive gibberish. No matching second-run raw file was found yet, so cache-hit drop must be probed with new raw-result files. - -Next diagnostic: run two small GSM8K passes (50-200 questions) with `--raw-result-file`, then compare first vs second failures and prefill `match_prefix_result` / `prefill_result_handoff` / `send_kv_chunk` around that window. - -## 2026-06-07 AD: 200-question raw probe reproduces second-run drop and shows prompt/request contamination pattern - -Probe command shape: two sequential GSM8K passes with `--num-questions 200 --parallel 64 --raw-result-file`, on the new process/log `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_131732.log`. - -Results: - -```text -pass1 raw=/mnt/beegfs/cjy/gsm8k_bs_gt1_probe_20260607_132646_pass1_raw.jsonl -Accuracy: 0.965, Invalid: 0.000 - -pass2 raw=/mnt/beegfs/cjy/gsm8k_bs_gt1_probe_20260607_132646_pass2_raw.jsonl -Accuracy: 0.700, Invalid: 0.000 -``` - -Raw transition summary over the same 200 prompt ids: - -```text -(True, True): 139 -(False, False): 6 -(True, False): 54 -(False, True): 1 -``` - -Failure classification: - -- pass1 failures: 7/7 normal wrong reasoning/answers; -- pass2 failures: 55 normal-looking wrong answers + 5 very-short outputs; -- pass2 newly bad ids: `[13, 14, 15, 18, 20, 25, 29, 31, 33, 35, 38, 39, 44, 45, 61, 62, 63, 64, 65, 75, 85, 89, 90, 93, 94, 100, 105, 108, 110, 115, 120, 124, 128, 129, 130, 134, 135, 143, 144, 154, 155, 159, 160, 164, 169, 170, 173, 175, 189, 190, 192, 193, 198, 199]`. - -Important qualitative signal: second-run failures are usually not random gibberish. Some outputs answer a different problem or echo a few-shot/example prompt fragment: - -- id=33 prompt asks Gretchen coins, output starts with the standard Janet-ducks GSM8K example and answers `18`; -- id=45 prompt asks Meredith blogging hours, output starts with `20\nKylie makes and sells scarves...` and answers `180`; -- id=62 prompt asks Marcy 30-year pension, output switches to a 25-year pension variant and answers `31250`; -- id=75 prompt asks average square footage, output answers a similar total-square-footage problem. - -Prompt-echo marker count: - -```text -pass1 total_echo=1, fail_echo=0 -pass2 total_echo=18, fail_echo=15 -``` - -This points away from pure fp8 numerical drift and toward request/prompt metadata contamination or offset/order mismatch under bs>1 cache-hit. The model is often coherent but conditioned on the wrong cached prompt slice / wrong hidden/topk / wrong transferred page table. - -Prefill log for the two-pass window: - -```text -Traceback=0 RuntimeError=0 FAIL_FAST=0 FALLBACK=0 HealthCheckFail=0 -HiCache-load=0 load_back=0 CacheCtrl-load=0 - -CP0 handoff bs distribution around probe: - mostly bs=5; no L2 load path. - -Second pass CP0 handoff prefix_top: - 640:176, 704:167, 768:16 -extend range: - min=2 avg≈56.9 max=165 -``` - -Conclusion update: root cause is likely in the bs>1 cache-hit online path where per-request cached prefix pages, output hidden/topk, and decode transfer metadata must stay aligned. Continue checking: - -1. `process_batch_result_disagg_prefill`: `batch.reqs`, `result.next_token_ids`, and `batch.spec_info.{hidden_states,topk}` order agreement; -2. `cache_unfinished_req` side effects before `send_kv_chunk` (it can refresh `req_to_token`/`prefix_indices` to a fuller cached prompt); -3. `send_kv_chunk` target/draft `page_indices/state_indices` consistency with the hidden/topk assigned to that same req; -4. any batch-level plan that reorders/splits valid rows without carrying request ids through to logits/sample/spec tensors. - -## 2026-06-07 AE: failing unit test isolates compute-padding all-gather rerange source-offset bug - -Added a targeted CPU unit test for this exact dataflow: - -- request 0 is a tiny extend that enables compute padding; -- request 1 follows it in the same bs>1 rank-major all-gather buffer; -- valid output must drop request-0 synthetic rows, but source offsets must still skip those synthetic rows. - -Remote failing evidence before fix: - -```text -test_batch_in_seq_all_gather_rerange_uses_compute_offsets_for_padded_source: FAILED -``` - -Root-cause candidate is now concrete: `_torch_batch_in_seq_all_gather_rerange()` used `request_split_lists` (valid rows) to compute source offsets inside the rank-major all-gather payload. Under compute padding, the source payload is laid out by `request_compute_split_lists`, while output must remain `request_split_lists`. This can make the second request read the first request's padded mirror rows or another request's rows after cache-hit/tiny-extend batching, matching the observed GSM8K prompt-fragment contamination. - -Required contract: - -- source rank offsets: compute split lists when `compute_padding_enabled`; -- output length/order: valid split lists; -- copy length per segment: valid segment length; -- no synthetic compute-padding row may appear in restored hidden/index/MLA rows. - -Fix applied: - -- `_torch_batch_in_seq_all_gather_rerange()` now separates `source_split_lists` from `output_split_lists`; -- source rank offsets and mirror offsets use `request_compute_split_lists` when compute padding is enabled; -- output allocation, output prefixes, and copy lengths still use valid `request_split_lists`; -- fail-fast validates that every compute segment covers the corresponding valid segment. - -Verification: - -```text -remote container: -PYTHONPATH=python python -m pytest -q test/registered/unit/layers/test_nsa_cp_utils.py -k batch_in_seq_all_gather_rerange -3 passed - -remote container: -PYTHONPATH=python python -m pytest -q test/registered/unit/layers/test_nsa_cp_utils.py -93 passed -``` - -ETE implication to verify after restart: if this was the second-run cache-hit accuracy root cause, a 200-question two-pass GSM8K probe should no longer show pass2 dropping to ~0.70 nor show prompt-fragment echo spikes. - -## 2026-06-07 AF: rerange 修复后第一次 ETE probe 被 decode/prefill fingerprint mismatch 阻断 - -修复 `_torch_batch_in_seq_all_gather_rerange()` 后，prefill 在 g0034 重新启动，日志 `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_135544.log` 显示： - -- server ready； -- `Traceback=0`、`RuntimeError=0`、`FAIL_FAST=0`、`FALLBACK=0`、`Health check failed=0`； -- container 内 `/sgl-workspace/sglang-tai/python/.../nsa/utils.py` 已包含 `source_split_lists` 修复。 - -随后启动 200 条 GSM8K 两轮 probe，但 pass1 卡在 0/200。prefill 日志只看到 `/get_model_info` 和 health check，没有实际 `/v1/chat/completions` 请求进入 prefill。 - -检查 decode 日志 `/mnt/beegfs/cjy/log/decode{0,1}_20260607_1317*.log`，decode 在 14:04:20 退出，根因是 runtime fingerprint fail-fast： - -```text -Runtime source fingerprint mismatch between prefill and decode. -prefill_fingerprint=96939b022cf93b42c8e1231832cafd4e35528e268a79db3814236dc63a889d94 -decode_fingerprint=47b3db5225b34a3f22fded3af198d2eb1d79ef71c63f2bb01f344aabd5ed95fd -Restart both prefill and decode from the same source tree. -``` - -这不是新的数据正确性结论；它说明这轮 ETE 没有真正验证 rerange 修复。fingerprint guard 正确阻止了 prefill/decode 源码版本不一致下的 PD KV transfer。 - -当前状态： - -- g0034/g0035/g0036 host `/mnt/beegfs/cjy/sglang-dev` 中关键文件 sha256 一致； -- g0034/g0035/g0036 container `/sgl-workspace/sglang-tai` 中关键文件 sha256 也一致； -- decode 失败发生在旧 decode 进程与新 prefill 进程交互时；需要重启 decode，使其重新加载同一份源码 fingerprint，再重跑 50-200 条两轮 GSM8K probe。 - -被中止的 probe base： - -```text -/mnt/beegfs/cjy/gsm8k_bs_gt1_probe_after_rerange_fix_20260607_140414 -``` - -该 probe 未产生有效精度结果，不应用于判断 rerange 修复效果。 - -## 2026-06-07 AG: rerange 修复后仍复现 pass2 cache-hit 掉点 - -有效 probe base： - -```text -/mnt/beegfs/cjy/gsm8k_bs_gt1_probe_after_rerange_fix_20260607_142514 -``` - -结果： - -```text -pass1: Accuracy=0.955, Invalid=0.000, Latency=34.272s, Output throughput=613.860 token/s -pass2: Accuracy=0.720, Invalid=0.000, Latency=34.599s, Output throughput=627.594 token/s -``` - -因此 AE 的 `_torch_batch_in_seq_all_gather_rerange()` source/output split 修复是必要的，但不是完整 root cause。 - -pass1/pass2 raw 对比： - -```text -pass1: ok=191 bad=9 -pass2: ok=144 bad=56 -new regressions: 48 -regression ids first: [15, 18, 19, 29, 34, 35, 39, 44, 45, 50, 54, 58, 59, 64, 65, 75, 85, 90, 93, 94] -``` - -失败形态：pass2 多数输出是语义连贯但解错，且常表现为“读到相似但不完全相同的问题”： - -- id=15：原题增长率是 `2.5% / 1.2%`，pass2 输出按 `2% / 1%` 解； -- id=18：原题 `4 weeks`，pass2 输出按 `4 days` 解； -- id=19：原题要求 average speed=4mph 下 remaining speed，pass2 输出解成同速完成； -- id=34：原题 Aaron has `5 more than half`，pass2 输出误成 `half * 5`。 - -Prefill log 观察： - -```text -Traceback=0 RuntimeError=0 FAIL_FAST=0 FALLBACK=0 HealthCheckFail=0 -pass1 mostly: #new-token≈576-704, #cached-token=3200 -pass2 mostly: #new-token=320, #cached-token≈3456-3584 -``` - -这更像“缓存命中后，命中的部分 prompt KV 与当前请求 prompt 不完全一致”，而不是 decode 崩溃或 fp8 随机数值漂移。 - -新增关键线索：debug log 在 health check / tiny extend 路径上显示 compute padding 即使 bs=1 也会启用： - -```text -batch_plan bs=1 extend_lens=[1] valid_pages=[1] compute_pages=[8] padding_tokens=[511] -split_tensor:1d:compute input_tokens=8 expected_tokens=1 target_lens=[512] local_rows=64 static_padded=8 -``` - -这本身未直接证明 GSM8K 失败，但说明 current/prefix/tiny cache-hit 路径仍大量依赖 compute-padding contract。下一步不要继续盲改，需要沿 L1 cache-hit 使用的 KV 读取路径检查： - -1. radix/HiCache 命中是否会把非 page-aligned 的 logical prefix floor 到正确 page boundary，且不会返回跨请求 stale tail； -2. L1 cached prefix 的 `req_to_token_pool` / `out_cache_loc` 是否只包含真实 page slots，不包含 compute padding mirror rows； -3. direct write 时 valid rows 写入的 physical page slot 顺序，是否与后续 cache-hit page table 读取顺序一致； -4. index KV 与 MLA KV 是否都遵守同一 valid-page contract，尤其是 second-pass `new-token=320` 的 partial-current/cache-hit组合。 - -## 2026-06-07 AH: decode prebuilt out_cache_loc slices prefix instead of suffix - -Root-cause candidate found in `python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py::prepare_for_prebuilt()`. - -Code facts: - -- `input_ids = fill_ids[len(prefix_indices):]` correctly makes decode prebuilt process only the suffix after cached prompt prefix. -- `seq_len - pre_len == req.extend_input_len` asserts the same suffix length contract. -- But `out_cache_loc` was built from: - - ```python - req_to_token_pool.req_to_token[req.req_pool_idx][: req.extend_input_len] - ``` - - which selects the first `extend_input_len` prompt locations, i.e. prefix page slots, not the suffix locations at `pre_len : pre_len + extend_input_len`. - -Observed failing unit test before fix: - -```text -TestDecodeQueueCompaction.test_prepare_for_prebuilt_uses_suffix_cache_locs_after_cache_hit -expected out_cache_loc=[2000,2001,2002] -actual out_cache_loc=[1000,1001,1002] -``` - -Why this matches GSM8K pass2 drop: - -- In pass1, many requests have low cache hit / larger prefill suffix, so the wrong prebuilt suffix mapping is less concentrated on short cache-hit tails. -- In pass2, `pre_len` is much larger and `extend_input_len` is often tiny; using `[:extend_input_len]` makes decode update/read the beginning of the prompt instead of the actual tail. -- Raw outputs looked like the model saw the first part of the question but missed final constraints (`per week`, `$3 fee + tip`, `4 weeks`, etc.), which is consistent with the tail KV being mapped to the wrong cache locs. - -Planned fix: slice decode prebuilt locs with `pre_len : pre_len + req.extend_input_len`, matching the existing input-id and assertion contract. This is not CP-specific; it is a general cache-hit prebuilt correctness bug that CP shared-KV bs>1 exposed by making second-pass cache hits much deeper/shorter. - -### AH 追加实现/验证结果 - -已按 TDD 做最小修复：`prepare_for_prebuilt()` 的 `out_cache_loc` 改为从 decode 端 `req_to_token_pool` 的 suffix 区间读取： - -```python -req_to_token_pool.req_to_token[req.req_pool_idx][pre_len : pre_len + req.extend_input_len] -``` - -同时 debug `out_chunk_start` 改为 `pre_len`，与 `expected_suffix_start` 保持一致。 - -验证： - -```bash -# RED: 同步测试到远端、旧代码失败 -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/disaggregation/test_decode_queue_compaction.py::TestDecodeQueueCompaction::test_prepare_for_prebuilt_uses_suffix_cache_locs_after_cache_hit -q -# expected [2000,2001,2002], actual [1000,1001,1002] - -# GREEN: 修复后 -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/disaggregation/test_decode_queue_compaction.py \ - test/registered/unit/layers/test_nsa_cp_utils.py -q -# 108 passed -``` - -下一步 ETE 验证要求：需要重启 decode（该 bug 在 decode 进程路径），然后重复 200 条 GSM8K 两轮 probe。若 root cause 命中，第二轮准确率应不再从约 `0.95` 跌到约 `0.70`，并且 raw 输出里的 prompt-tail omission / prompt-fragment echo 应显著减少。 - -## 2026-06-07 AI: restart 后 200 条 probe 仍失败，问题仍在 L1 cache-hit page order - -用户重启后重新跑 200 条 GSM8K 两轮 probe： - -```bash -python benchmark/gsm8k/bench_sglang.py \ - --host g0034 --port 17100 --backend srt \ - --data-path /mnt/beegfs/cjy/data/gsm8k_test.jsonl \ - --num-questions 200 --parallel 64 \ - --temperature 0.0 --top-p 1.0 --max-new-tokens 512 -``` - -结果目录：`/mnt/beegfs/cjy/gsm8k_retest_20260607_231950` - -结果： - -```text -pass1 Accuracy: 0.685 Invalid: 0.000 Latency: 35.842 s Output throughput: 614.422 token/s -pass2 Accuracy: 0.720 Invalid: 0.000 Latency: 34.502 s Output throughput: 609.073 token/s -``` - -说明 decode prebuilt suffix slicing 修复仍不足；当前进程内 cache 已经被命中/复用，第一轮也不是干净 cold baseline。 - -prefill log：`/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_145743.log` - -关键证据仍然是 `send_kv_chunk` 前 `req_to_token_pool` 的 page 顺序错误。示例： - -```text -event=send_kv_chunk rid=aaad28fd98184ef5bdd0b6e6a733c737 -start_idx=0 end_idx=721 page_size=64 -pages=head=[33,2,3,4,5,6,7,8] -state_pages=head=[33,2,3,4,5,6,7,8] -prefix_len=721 host_hit_length=0 extend_input_len=17 fill_len=721 origin_input_len=721 -``` - -同一批次多个 rid 都出现相同 pattern：`state_pages` 头部为新 suffix page（例如 33），后面才是公共 prefix page `[2,3,4,...]`。这不是 L2 load-back：`host_hit_length=0`，属于 L1/device hit 后的 page table 顺序问题。 - -当前排除项： - -1. `write_cache_indices()` 独立 CUDA/unit probe 已验证 prefix-first + suffix-after-prefix 的写入 contract 正确。 -2. `CPSharedPagedTokenToKVPoolAllocator.alloc_extend_compute_owner()` 简单 cache-hit suffix 分配 probe 已验证返回 suffix loc 是 request order。 -3. prefill/decode 日志没有新的 `FAIL_FAST` / `CP_SHARED_KV_FALLBACK` / `Traceback`。 - -下一步定位点不要再查上述两个 helper；应在真实 runtime 的以下边界记录 page-order： - -1. `alloc_for_extend()` / `write_cache_indices()` 后：记录每个 req 的 prefix pages、suffix/out pages、最终 `req_to_token[:seq_len]` pages。 -2. `cache_unfinished_req()` / `cache_finished_req()` 前后：记录插入 radix 的 `values` pages、match 后 `new_indices` pages、以及最终写回 `req_to_token` 的 pages。 -3. 如果 `alloc_for_extend()` 后顺序正确而 `send_kv_chunk()` 前错误，root cause 在 radix cache insertion / re-match / page-floor tail pruning；如果前者已经错误，则继续回溯 prefix_indices 来源。 - -## 2026-06-08 00:04 retest after reported restart - -Command: - -```bash -cd /sgl-workspace/sglang -python benchmark/gsm8k/bench_sglang.py \ - --host g0034 --port 17100 --backend srt \ - --data-path /mnt/beegfs/cjy/data/gsm8k_test.jsonl \ - --num-questions 200 --parallel 64 \ - --temperature 0.0 --top-p 1.0 --max-new-tokens 512 \ - --raw-result-file /mnt/beegfs/cjy/gsm8k_debug_retest_20260608_000457/pass1_raw.jsonl \ - --result-file /mnt/beegfs/cjy/gsm8k_debug_retest_20260608_000457/pass1_result.jsonl -``` - -Result: - -- Accuracy: 0.695 -- Invalid: 0.000 -- Latency: 35.829 s -- Output throughput: 588.436 token/s - -Notes: - -- The prefill process observed before the retest was PID 627606, start time `Sun Jun 7 23:48:54 2026`, log `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_154854.log`. -- This means the retest was against an already warm cache from earlier GSM8K runs in the same prefill process, not a cold-cache first pass. -- The result remains consistent with the cache-hit correctness regression: warm-cache GSM8K accuracy is around 0.69-0.72 while cold/first-pass accuracy is around 0.95-0.97. - -## 2026-06-08 remote retest after syncing origin+cjy fixes - -Remote code synced to g0034 `/mnt/beegfs/cjy/sglang-dev` and container path `/sgl-workspace/sglang-tai`. - -Full GSM8K two-pass run directory: -`/mnt/beegfs/cjy/gsm8k_retest_20260607_164904` - -Results: -- pass1: Accuracy 0.958, Invalid 0.000, Latency 214.254s, Output throughput 646.402 token/s. -- pass2: Accuracy 0.690, Invalid 0.002, Latency 226.338s, Output throughput 624.138 token/s. - -Conclusion: warm-cache/cache-hit correctness regression still reproduces after syncing per-layer transfer branch plus local fixes. For future iterations, do not start with full 1319-question runs. Run a 200-question GSM8K canary first for fast signal; only run full GSM8K after the 200-question warm-cache second pass stays near expected accuracy. - -## 2026-06-08 AJ: 当前线上复现未开启 disagg per-layer transfer - -用户确认当前线上版本没有开启 prefill→decode per-layer transfer。因此本轮 GSM8K warm-cache 掉点不应继续优先归因于 `CP_PER_LAYER_TRANSFER` 的异步/分组路径。 - -纠正：此前把 per-layer transfer block grouping 列为风险点只是代码层可能性；对当前线上日志/复现不成立。当前应回到 monolithic Mooncake transfer、radix/req_to_token page-order、NSA state/index companion transfer、以及 decode 端接收/consume 元数据路径。 - -已知日志事实仍保持：最新 debug 日志没有 `HiCache-load/load_back`，第二轮主要是 L1/device cache-hit 后 transfer/consume 更深 cached prefix；输出像“prompt/request 片段串了”，不是乱码。 - -## 2026-06-08 AK: 当前更强证据指向 radix/scheduler 过度 prefix hit，不是 per-layer transfer - -纠正上一轮 page-order 解释：`send_kv_chunk pages=head=[25,2,3,...]` 不能单独证明 suffix-before-prefix。冷启动/无 cache 的 first pass 也会出现类似非单调 page id，这是 CP owner-lane allocator 的物理 page id 分配结果，数值顺序不等于 token 顺序。 - -新证据：用容器内 GLM-5.1 tokenizer 对 `/mnt/beegfs/cjy/gsm8k_retest_20260607_164904/pass1_raw.jsonl` 前 200 条 prompt 计算两两公共前缀： - -```text -prompt0 prefix cnt [(671, 198), (672, 1)] minmax 671 672 -pair floored common top [(640, 19900)] -examples >=704 [] -``` - -也就是说，真实 GSM8K prompt 两两 page-floor 后只有 640 token 公共前缀，没有自然 704/768 公共前缀。 - -但 prefill 日志在 warm-cache pass 中出现： - -```text -match_prefix_result ... key_len=735 device_tokens=735 ... last_host_node=2645 last_host_host_len=64 cp_floor_exact=False -batch_plan ... extend_lens=[32,18,3,13,23] prefix_lens=[704,704,704,768,704] -``` - -这说明 runtime 里某些请求被暴露了 704/768 的 device prefix，而从真实 prompt token 序列看这些 prefix 不应该互相匹配。当前 root-cause 排查优先级应转向： - -1. scheduler / waiting_queue radix match 是否用全 prompt 或临时树产生了过长 prefix； -2. CP HiCache exact valid-tail floor 规则是否只在 `Req.adjust_max_prefix_ids()` 生效，而 `schedule_policy._compute_prefix_matches()` / in-batch prefix path 仍可能把非 page-aligned tail 当作命中； -3. `cache_unfinished_req()` 内部 `match_prefix(cp_floor_exact=False)` 是当前请求自刷新用，不能让后续 scheduler-visible match 继承未 floor 的 valid-tail。 - -当前不再优先排查 disagg per-layer transfer；用户确认线上未开启该路径。 - -## 2026-06-08 AL: 排除 health check 后修正 AK 的过度 prefix-hit 判断 - -用户提醒：GSM8K 后面的请求很可能是 health check，不应纳入 cache-hit 掉点判断。后续日志分析只统计 benchmark 时间窗内的 `/v1/chat/completions` / scheduler prefill batch，不再用后续 health-check 长尾请求解释 GSM8K 精度下降。 - -同时修正 AK：pass2 是对同一批 GSM8K prompt 的重复请求，因此 `prefix_lens=704/768` 本身并不证明不同 prompt 之间发生过度匹配。用 tokenizer 计算得到的 `floor(prompt_len)` 也显示许多 GSM8K prompt 自身长度就是 704/768 页对齐前缀；warm-cache pass2 命中这些完整 prompt 页是预期行为。 - -因此当前更准确的判断是： - -1. **不能再把 `704/768` prefix 命中单独作为 scheduler/radix overmatch 证据。** 只有当某个具体 request 的 `prefix_len` 超过其自身 `floor(prompt_len)`，或输出明显来自另一条 prompt 且能绑定到同一 request，才算 overmatch/串 cache 证据。 -2. **当前 warm-cache 掉点仍然成立**：full GSM8K pass1 约 `0.958`，pass2 约 `0.690`，且输出样例有 prompt 内容串用现象。 -3. **优先排查方向回到 cache-hit consume/transfer correctness**：L1 device cache hit 后的 CP shared KV page table、index/topk offset、MLA materialize/current reuse、monolithic Mooncake transfer/metadata consume。当前线上未开启 disagg per-layer transfer；health check 请求不作为证据。 - -下一步日志分析必须把 benchmark result/raw 文件与 prefill 日志时间窗绑定，必要时只跑 50/200 条 canary，并在 debug log 中输出 request/rid 与 prompt index 的可关联字段，否则很难把某条错误输出反推到具体 cache path。 - -## 2026-06-08 AM: 增加 GSM8K cache-hit 对齐用 debug log - -新增 debug 只挂在既有 `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG` / `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT` 下，不新增开关，默认关闭。 - -目的：把 benchmark raw prompt 与 prefill/transfer 日志绑定起来，避免继续只能按时间和 request 顺序猜测。 - -新增字段/事件： - -1. `scheduler_prefill_batch` 增加 `rids` 与 `origin_sigs`。 - - `origin_sig` 是 origin token ids 的稳定 hash + head/tail + len，可由 raw prompt 在同 tokenizer 下复算。 -2. `send_kv_chunk` 增加 `origin_sig` 与 `fill_sig`。 - - 用于确认 send 前请求身份、fill_ids 是否多了输出 token，以及同一 prompt pass1/pass2 是否绑定到预期 rid。 -3. Mooncake prefill sender 增加 `sender_filter`。 - - 记录 CP shared-KV 过滤前 logical request pages、过滤后 physical pages、logical page positions、state/indexer pages。 -4. Mooncake worker 增加 `transfer_worker_kv` / `transfer_worker_state`。 - - 记录实际发送到 decode dst pages 的映射，区分 KV 与 NSA state/indexer companion metadata。 - -这些日志用来验证：同一个 GSM8K prompt 在 pass2 cache-hit 时，prefill 端 page table、CP owner filter、decode dst page selection 是否一致；如果输出串 prompt，下一步应按 `origin_sig` 精确定位对应 rid/room，而不是再把 health check 或无关长尾请求纳入判断。 - -## 2026-06-08 AN: 200-question canary on new debug log - -User started a new debug-log run and requested a 50-200 question GSM8K canary. I ran two 200-question passes against the existing prefill process/log: - -- prefill PID: `666430`, started `Mon Jun 8 02:01:57 2026` -- log: `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_180157.log` -- result dir: `/mnt/beegfs/cjy/gsm8k_debug_retest_20260607_183407` -- time window: `2026-06-07 18:34:07` to `18:35:29` - -Command shape: - -```bash -cd /sgl-workspace/sglang -python benchmark/gsm8k/bench_sglang.py \ - --host g0034 --port 17100 --backend srt \ - --data-path /mnt/beegfs/cjy/data/gsm8k_test.jsonl \ - --num-questions 200 --parallel 64 \ - --temperature 0.0 --top-p 1.0 --max-new-tokens 512 \ - --raw-result-file /mnt/beegfs/cjy/gsm8k_debug_retest_20260607_183407/pass${PASS}_raw.jsonl \ - --result-file /mnt/beegfs/cjy/gsm8k_debug_retest_20260607_183407/pass${PASS}_result.jsonl -``` - -Results: - -- pass1: Accuracy `0.725`, Invalid `0.000`, Latency `35.364s`, Output throughput `601.398 token/s`. -- pass2: Accuracy `0.710`, Invalid `0.000`, Latency `34.390s`, Output throughput `606.721 token/s`. - -Important interpretation: this process/log was already warm before the canary, so pass1 is not a cold baseline. Both passes are warm-cache and both are already bad. This confirms the warm-cache correctness failure remains, but this particular canary does not measure cold→warm drop. - -Bound scheduler/raw correlation through `origin_sigs`: - -- 200 prompt IDs mapped to scheduler log entries. -- Each prompt appeared exactly twice in the benchmark window. -- CP0 scheduler batches: `bs=5` for 78 batches, `bs=4` for 2 batches, `bs=1` for 4 batches. -- Prefix top counts: `704` x334, `640` x34, `768` x32, `0` x2. -- Extend range: min `2`, max `65`, average `28.1` tokens. -- `prefix_floor_bad=0`: no scheduler occurrence had `prefix_len > floor(prompt_len/page_size)*page_size`, and `extend_len == prompt_len - prefix_len` for all mapped prompts. -- No `CP_SHARED_KV_FAIL_FAST`, `CP_SHARED_KV_FALLBACK`, `Traceback`, `RuntimeError`, or `Health check failed` in the benchmark window. - -Accuracy by prompt tail length (`prompt_len % 64`) on this warm-cache process: - -```text -pass1: - 00-16: 63/70 = 0.900 - 17-32: 49/62 = 0.790 - 33-48: 16/34 = 0.471 - 49-63: 17/34 = 0.500 -pass2: - 00-16: 61/70 = 0.871 - 17-32: 47/62 = 0.758 - 33-48: 15/34 = 0.441 - 49-63: 19/34 = 0.559 -``` - -Current evidence-based direction: - -1. Do not re-open the earlier `704/768 overmatch` hypothesis for this window: prompt-to-log matching shows no over-prefix relative to each prompt's own page floor. -2. Do not prioritize L2 load-back for this window: there were no `HiCache-load/load_back` markers in the canary window, and every mapped request was already L1/device cache-hit plus short current suffix. -3. The strongest current signal is tail-size correlation under warm L1 cache hit: failures rise sharply when the current suffix tail is larger. This points to cache-hit current-tail consume/compose, especially FP8/RAGGED MLA or index partial-current handling under compute padding, rather than radix overmatch or monolithic Mooncake transfer. -4. Next code reading should focus on whether the FP8/RAGGED `nsa_backend` path materializes current tail tokens from persistent KV before they are actually stored, or otherwise omits the explicit partial-current compose used by the PAGED path. - -## 2026-06-08 AO: RAGGED FP8 MLA cache-hit path can read current suffix through full materialize - -New code-reading finding after the 200-question warm-cache canary: - -- Current launch uses FP8 + `flashmla_sparse` RAGGED prefill path for CP shared KV. -- `nsa_backend.forward_extend()` only enters the existing MLA current/partial-current compose block when `topk_transform_method == TopkTransformMethod.PAGED`. -- The RAGGED branch later handles prefix sharing by calling `materialize_shared_token_kv_buffer(kv_cache, logical_locs=page_table_1_flattened, ...)` and then `dequantize_k_cache_paged(...)`. -- For cache-hit requests, `page_table_1_flattened` includes both cached prefix tokens and the just-computed current suffix tokens. The current suffix was direct-written into each CP rank's persistent KV earlier in the layer. -- If TAI IPC materialize is enabled, full materialize can read current suffix pages directly from peer ranks without an inter-rank ordering point proving those peer direct-write kernels have completed. The old collective materialize path incidentally provided such ordering; IPC gather removes it. - -This matches the observed shape: cold/current-only first-pass can be correct, but warm L1 cache-hit requests with short current suffixes lose accuracy, and failure rate correlates with current tail size. It also explains why no L2 load-back markers or radix over-prefix evidence appeared in the canary window. - -Planned test/fix direction: - -1. Add a regression/source-contract test showing the RAGGED CP shared-KV prefix-hit path must use page-slot partial-current compose, not full materialize of prefix+current through IPC. -2. Update RAGGED path to materialize prefix slots and fill/reduce current slots via `materialize_prefix_and_reuse_current_kv_page_slots`, mirroring the PAGED partial-current contract, then dequantize the resulting dense page-slot buffer. -3. Keep the test narrow; if this does not recover GSM8K warm-cache accuracy, add bounded RAGGED materialize debug under existing `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG`. - -## 2026-06-08 AP: RAGGED fix exposed wrong current-row selector assumption - -Remote 200-question canary after the first RAGGED partial-current fix killed prefill before accuracy could be measured. - -Observed prefill exception in `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_192001.log`: - -```text -RuntimeError: [CP_SHARED_KV_FAIL_FAST][cache_write_valid_selector_shape] -CP shared-KV cache write tensor must contain local compute rows before valid-row selection. -local_rows=473 expected_compute_rows=320 valid_rows=13 -``` - -and on another rank: - -```text -local_rows=473 expected_compute_rows=320 valid_rows=140 -``` - -Call stack: - -```text -nsa_backend.forward_extend -> RAGGED cache-hit compose -> -select_cp_local_valid_rows_for_cache_write(forward_batch, k/k_rope) -``` - -Interpretation: - -- `select_cp_local_valid_rows_for_cache_write()` is only valid for tensors whose first dimension is the CP-local compute-padded row count (`expected_compute_rows`). -- In the `flashmla_sparse` RAGGED attention branch, `k/k_rope` can have a different row contract (`local_rows=473` in the crash), so blindly reusing the cache-write selector there is wrong. -- The actual direct KV write path in `forward_mla._maybe_write_cp_shared_local_mla_kv()` has already performed the compute-padding valid-row selection before writing to persistent KV. The RAGGED attention compose path only needs a current-row source for attention materialization; it must select rows based on the actual tensor contract, not assume cache-write local compute rows. - -Next fix direction: - -1. Factor a helper for RAGGED/PAGED current-row source selection that accepts either: - - already local valid rows (`rows == local_out_cache_loc.numel()`), or - - CP-local compute-padded rows (`rows == expected_compute_rows`), using `select_cp_local_valid_rows_for_cache_write()`, or - - global valid current rows (`rows == current_extend_kv_rows_for_reuse(...)`), using `cp_split_and_rebuild_data()`. -2. Add a unit/source-contract test covering the `rows == local_out_cache_loc.numel()` fast path before production code change. - -## 2026-06-07 G: retained-cache 200x2 run is not a cold/warm comparison - -用户提醒本轮服务未重启，cache 仍保留。因此下面结果不能作为“冷启动第一轮 vs warm-cache 第二轮”对照，只能作为已有热缓存状态下的复现： - -- 输出目录：`/mnt/beegfs/cjy/gsm8k_debug_20260607_195303` -- pass1：`Accuracy=0.695`, `Invalid=0.000`, `Latency=35.049s`, `Output throughput=615.881 token/s` -- pass2：`Accuracy=0.695`, `Invalid=0.000`, `Latency=34.227s`, `Output throughput=632.346 token/s` - -结论：当前服务 cache 已处于错误热缓存状态；继续重复跑 GSM8K 不会提供新的冷启动证据。该结果仍证明 warm-cache consume 路径存在稳定精度损坏。后续不要把这组数据误解为首轮冷启动掉点。 - -## 2026-06-08 AM: retained-cache 200x2 probe is warm/warm, not cold/warm - -用户提醒本次服务没有重启，cache 仍然保留。复查 `/mnt/beegfs/cjy/gsm8k_debug_20260607_195303` 与日志 `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_193843.log` 后确认： - -- pass1 Accuracy `0.695`，pass2 Accuracy `0.695`；两轮都已经是热 cache 状态。 -- 通过 `origin_sig` 关联 CP0 `scheduler_prefill_batch` 与 200 条 GSM8K prompt，pass1/pass2 的 prefix/extend 基本完全一致。例如： - - pid0: `(prefix=704, extend=32)` 在 19:53:08 和 19:53:50 各出现一次； - - pid1: `(prefix=640, extend=59)` 两轮一致； - - pid4: `(prefix=768, extend=13)` 两轮一致。 -- CP0 prefix 统计：`704:334`, `640:34`, `768:32`，对应 200 条 prompt 的两轮 warm/warm 调度。 - -结论：这轮结果只能证明“保留 cache 后 warm-cache 精度稳定在约 0.69”，不能用于判断某个修复是否改善了 cold→warm 掉点，也不能作为第一轮 cold baseline。后续需要冷/warm 对比时必须重启 prefill/decode 或显式清 cache；若不重启，就只把它当热 cache 定位样本。 - -## 2026-06-08 AM: tensor dump deferred - -User correction: tensor dump should be the last-resort step, only after code/log review cannot locate the divergence boundary. Current next step stays source-level/runtime-log investigation around prefill cache-hit compute and send metadata. - -Do not add tensor dump instrumentation by default. If later needed, keep it opt-in and scoped first to prefill handoff / `send_kv_chunk()`; decode-side dump should remain out of scope unless prefill-side evidence matches but output still diverges. - -## 2026-06-08 AQ: index partial-current 尚未排查干净，补充限频元数据日志 - -结论：index 的 partial/current 方向还不能排除。 - -依据： - -- 既有单测已覆盖 CP=8、bs=5、FP8、prefix+current page-slot compose、TAI batched index MQA prepare、persistent index fused-store、L2 roundtrip 等基本合同；这些降低了“单个 kernel/基础布局必错”的优先级。 -- 但最新 warm-cache 复现仍显示：没有 L2 load-back、没有 fallback/fail-fast，失败率与 cache-hit 后 current tail 大小相关。 -- 线上日志里 index 路径大量走 `has_current_index=True` + `has_shared_index_buffer=True` 的 partial-current compose；现有日志只能确认分支被使用，不能确认 request 级 current rows、slot spans、compact topk segments 与 request metadata 一致。 - -本轮新增默认关闭的元数据日志，复用已有开关： - -```bash -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG=1 \ -SGLANG_CP_SHARED_KV_BS_GT1_DEBUG_LIMIT= -``` - -新增事件： - -- `index_current_reuse_prepare_detail`：记录 current-index reuse 的 row source、prefix/extend、local valid loc、rank-local offsets/tokens。 -- `index_partial_current_compose_detail`：记录 prefix/current slot spans、current rows、logical page-table shape、是否命中 prefetcher。 -- `index_topk_batch_compact_detail`：记录 batch topk compact 前后的 request q/kv 长度、valid q、cp_index 与 scatter spans。 -- `index_topk_cp_segments_detail`：记录 `_get_topk_ragged_with_cp()` 内部 segment_records、k/q bases、request_kv_bases、topk offset 列表摘要。 -- `index_direct_write` 增加 rids、prefix/extend、logical/physical loc 摘要。 - -这些日志只在 debug env 打开时执行；默认路径不新增 runtime check，也不做 tensor dump。下一轮应使用 50/200 条 warm-cache canary，把 raw prompt 的 `origin_sig/rid` 与上述 index 事件绑定，判断是否存在某个 request 的 current rows、slot span 或 topk offset 跨 request/错位。 - -## 2026-06-08 AR: 50 条 GSM8K 三段隔离，问题收敛到 bs>1 + warm-cache - -远端服务健康，日志：`/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_204531.log`。 - -测试目录：`/mnt/beegfs/cjy/gsm8k_50_bs_probe_20260607_205120`。 - -三段命令等价为： - -1. 50 条并发 `parallel=64`：`pass1_parallel64` -2. 同一 50 条再次并发 `parallel=64`：`pass2_parallel64` -3. 同一服务不清 cache，50 条顺序 `parallel=1`：`pass3_parallel1` - -结果： - -```text -pass1_parallel64: Accuracy=0.980, Invalid=0.000, Latency=12.365s -pass2_parallel64: Accuracy=0.760, Invalid=0.000, Latency=12.398s -pass3_parallel1: Accuracy=0.960, Invalid=0.000, Latency=94.405s -``` - -raw 对比： - -```text -p1 bad ids: [37] -p2 bad ids: [10,13,15,18,19,20,28,29,33,37,44,45] -p3 bad ids: [5,12] -p1 -> p2 regress: [10,13,15,18,19,20,28,29,33,44,45] -p1 -> p3 regress: [5,12], recover: [37] -``` - -prefill/scheduler 统计： - -```text -p1_parallel64: - scheduler bs: {5:72, 4:8, 1:8} - prefix top: 640 x392, 0 x8 - extend avg: 102.62, min=59, max=736 - CP0 prefill: newtok=7040, cached=31360 - -p2_parallel64: - scheduler bs: {5:72, 4:8, 1:8} - prefix top: 704 x352, 640 x24, 768 x24 - extend avg: 25.82, min=3, max=61 - CP0 prefill: newtok=3200, cached=35200 - -p3_parallel1: - scheduler bs: {1:408} - prefix top: 704 x352, 640 x24, 768 x24, 0 x8 - extend avg: 25.33, min=1, max=61 - CP0 prefill: newtok=3264, cached=35200 -``` - -关键结论： - -- `pass2_parallel64` 和 `pass3_parallel1` 都是 warm-cache、短 extend、prefix 命中 704/768；但 `parallel=64/bs>1` 明显掉点，`parallel=1/bs=1` 基本恢复。 -- 因此 **cache-hit 本身不是充分条件**；问题收敛到 **bs>1 batching 与 warm-cache/current-tail 组合**。 -- `p2` 与 `p3` 均大量走 index partial-current：`has_current_index=True`、`has_shared_index_buffer=True`、`compute_padding=True`。但 bs=1 通过，说明不是“index partial-current 基础逻辑必错”，而更可能是 batch-aware compact/scatter/request-offset/slot-span 跨 request 语义问题。 -- 下一步优先审计 batch-aware index/MLA consume 路径，而不是 L2 load-back 或单 request current reuse。 - -## 2026-06-08 AS: parallel=20 小批量 probe 强化 bs>1 触发信号 - -用户指出 `parallel=5` 可能没有充分叠 batch，因此直接跑 20 条、`parallel=20` 的 warm-cache canary。 - -测试目录：`g0034:/mnt/beegfs/cjy/gsm8k_tiny20_p20_probe_20260607_210046`，日志仍为：`/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_204531.log`。 - -结果： - -```text -warm_parallel20_pass1: Accuracy=0.750, Invalid=0.000, Latency=6.370s -warm_parallel20_pass2: Accuracy=0.750, Invalid=0.000, Latency=6.539s -bad prompt ids: [10, 14, 15, 18, 19] in both passes -``` - -对照同一热 cache 状态下较低并发/顺序小样本： - -```text -parallel=5, 20 questions: Accuracy=0.850, bad=[5,7,12] -parallel=1, 20 questions: Accuracy=0.900, bad=[5,12] -``` - -用 `origin_sig` 绑定 prompt id 到 CP0 `scheduler_prefill_batch` 后，parallel=20 中失败请求稳定落在 bs>1 batch 内，且集中在 batch 后半段： - -```text -21:00:53 bs=5 pids=[6,7,9,8,10] bad=[10] prefix=[704,704,704,768,704] extend=[10,34,26,6,27] -21:00:54 bs=5 pids=[11,12,13,15,14] bad=[15,14] prefix=[704,704,704,704,704] extend=[30,35,26,59,18] -21:00:55 bs=4 pids=[16,17,18,19] bad=[18,19] prefix=[704,704,640,704] extend=[19,21,61,33] - -21:01:06 bs=5 pids=[5,7,8,9,10] bad=[10] -21:01:06 bs=5 pids=[12,11,13,14,15] bad=[14,15] -21:01:07 bs=4 pids=[17,16,18,19] bad=[18,19] -``` - -结论： - -- `parallel=20` 比 `parallel=5` 更清楚地触发了 bs>1 错误；20 条样本即可稳定复现 `0.75` accuracy。 -- 失败不是简单 cache-hit 本身：相同热 cache 下 `parallel=1` 明显更好。 -- 失败也不是随机：同一批 prompt 在两轮 parallel=20 中坏 id 一致，并且集中在 bs=5/bs=4 batch 的后半 request。下一步优先检查 batch-aware compact/scatter/output cursor/request offset 是否在后续 request 上错位，尤其是 index topk、RAGGED MLA cache-hit compose、以及 prefill handoff 的 per-request输出对齐。 - -## 2026-06-08 AT: parallel=20 输出显示跨 request 语义串扰 - -继续检查 parallel=20 raw 输出后，发现失败样本不是普通推理错误，而是明显的 request 语义串扰。 - -例子：parallel=20 中 `prompt_id=18` 的 question 是鸡蛋/dozen 问题，但输出包含： - -```text -60-15-25=... cups of feed -#### 20 -``` - -该推理内容对应 GSM8K `prompt_id=4` 的鸡饲料题。也就是说，后半 request 的 prefill/forward 结果中混入了另一个 request 的语义，而不是单纯 cache hit 后数值漂移。 - -对比：同一热 cache 状态下 `parallel=1/5` 中 `prompt_id=10/14/15/18/19` 正确，`parallel=20` 中这些 id 稳定错误。因此当前首要假设是 **bs>1 flatten/compact/scatter/request-offset 对齐错误**，不是 L2 load-back 基础错误，也不是 decode 单独错误。 - -## 2026-06-08 AU: slot remap 不是 unique/dedup，排除一个旧假设 - -检查 `build_slot_page_remap()`： - -- 输入是 `logical_pages` 的 flattened slot 表。 -- 输出 dense page id 是 flattened slot id + 1；每个 page-table slot 都有独立 dense slot。 -- 它不是按 logical page 去重的 unique/remap。 - -因此先前“prefix/current spans 可能索引进 dedup 后 unique pages，导致 request slot 串位”的假设不成立。 - -仍需注意：`page_inverse.scatter_()` 对重复 logical page 会保留最后一个 slot，但共享 prefix 的重复 slot 理论上内容相同，当前没有证据表明这是 root cause。 - -## 2026-06-08 AV: RAGGED index batch path走 TAI packed gather，暂未证实 page-slot/packed 错配 - -检查 `_get_topk_ragged_with_cp()` 和 TAI `cp_index_mqa_prepare.cu` 后确认： - -- RAGGED batch index 路径构造 `k_bases/q_bases/request_kv_bases` 等 packed descriptor。 -- `try_tai_prepare_cp_mqa_index_batch()` 的 CUDA gather 通过 `block_tables[batch_idx, logical_page]` 读取 page，再写入 `k_out[k_base + token_base]`。 -- 也就是说 index K buffer 的最终产物是按 request packed 的 dense token buffer，而不是直接把 page-slot buffer 当 packed buffer 使用。 - -因此“index topk 直接把 page-slot dense buffer 当 request-packed buffer”的简单假设暂时排除。仍需继续验证 packed topk output 与 MLA ragged KV dequant 的 packed token layout 是否完全一致。 - -## 2026-06-08 AW: 错误在 prefill handoff 前已经出现，不是 decode-only - -在 parallel=20 的 `bs=4 pids=[16,17,18,19]` batch 中，CP0 日志显示： - -```text -event=prefill_result_handoff bs=4 -extend_lens=[19,21,61,33] -prefix_lens=[704,704,640,704] -next_token_ids=[576,5512,220,1416] -hidden_shape=(4,6144) topk_shape=(4,1) -``` - -其中 `prompt_id=18` 对应 batch index 2，prefill 首 token 是 `220`（空格），后续输出转向了另一个题目的语义。由于 prefill handoff 已经携带错误 next token，问题不应优先归因到 decode-only。 - -下一步优先检查： - -1. `cp_collect_last_token_hidden()` 在 compute padding + bs>1 下是否选错每个 request 的 last hidden row。 -2. RAGGED MLA cache-hit compose/dequant 的 packed KV 与 index topk 的 packed token id 是否一致。 -3. logits/topk handoff 的 per-request row 顺序是否仍等于 scheduler batch request 顺序。 - -## 2026-06-08 AX: p20 tiny-extend last-token collect 合同通过，暂时排除 prefill handoff 行选择 - -针对 parallel=20 中稳定失败的真实 batch 形态补了单测： - -```text -extend_lens=[19,21,61,33] -prefix_lens=[704,704,640,704] -cp_size=8,page_size=64,cp_rank=0 -request_compute_rank_local_offsets=[0,64,128,192] -request_last_token_local_offset=[18,20,60,32] -request_last_token_owner=[0,0,0,0] -``` - -验证 `cp_collect_last_token_hidden()` 会从每个 request 的 64-row compute-padded slot 中取正确 last-token 行，并按 batch request 顺序 all-gather 回收。 - -远端验证： - -```text -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_collect_last_token_hidden_matches_parallel20_tiny_extend_layout -# 1 passed -``` - -结论：当前证据不支持 “prefill handoff 的 last hidden row 选择把 request 行串了” 作为 root cause。下一步继续查 RAGGED MLA/topk 的 packed token layout 对齐，特别是 cache-hit compose/dequant 后的 KV token 顺序是否和 index topk 的 request-local packed token id 一致。 - -## 2026-06-08 AY: p20 tiny-extend batch rerange 合同通过，暂时排除 compute-padding rerange 串行 - -补充 cp=8、parallel=20 真实形态的 `_torch_batch_in_seq_all_gather_rerange()` 单测： - -```text -extend_lens=[19,21,61,33] -prefix_lens=[704,704,640,704] -compute rows: 每 req 64 row，rank0 有 valid current，rank1-7 只有 dummy padding -``` - -测试在 rank0 的每个 64-row compute slot 里填 valid rows + poison padding，在 rank1-7 填 poison，期望 rerange 后只保留每个 request 的 valid rows，且按 request 顺序拼接。 - -远端验证： - -```text -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_batch_in_seq_all_gather_rerange_matches_parallel20_tiny_extend_layout \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_collect_last_token_hidden_matches_parallel20_tiny_extend_layout -# 2 passed -``` - -结论：当前证据不支持 “cp_all_gather_rerange_output 在 p20 tiny compute padding 下把 dummy rows 混入 valid 输出” 作为 root cause。 - -同时检查 `fast_topk_transform_ragged_fused` 后确认：GSM8K 700 token 级别 `kv_len < topk=2048`，ragged topk fast path 返回 `0..length-1 + topk_indices_offset`，不依赖 index logits 内容。因此 p20 掉点优先级应从 index logits/partial-current 降到 MLA KV cache-hit compose/dequant/token-content 对齐。 - -## 2026-06-08 AZ: CP8 multi-request KV compose 的 Python/reference 合同通过 - -新增 runtime 单测 `test_cp8_batch_kv_partial_current_keeps_request_packed_layout`： - -- `cp_size=8`，4 个 request； -- prefix pages 分散在不同 CP owner； -- current pages 全部构造为 rank0-owned； -- 每个 request/token 写唯一值； -- 对 8 个 rank 分别执行 `materialize_prefix_and_reuse_current_kv_page_slots()`，再把 rank 输出求和模拟 all-reduce； -- 用返回的 `mixed_locs` 按 request-packed 顺序读取 `merged_kv`。 - -远端验证： - -```text -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_cp8_batch_kv_partial_current_keeps_request_packed_layout -# 1 passed -``` - -结论：Python/reference 的 page-slot compose + current fill + request-packed mixed_locs 合同目前没有暴露串 request。下一步优先检查生产 CUDA/TAI current-slot fill、FP8 paged dequant、以及 FlashMLA sparse consume 的接口形状，而不是继续怀疑 planner/rerange 的基础 CPU 逻辑。 - -## BA. parallel=20 reproducer confirms max bs=5 stacking under current scheduler gate - -2026-06-08: ran GSM8K first 50 twice with `--parallel 20` against the running remote service, without restart: - -- Result dir: `/mnt/beegfs/cjy/gsm8k_p20_probe_20260608_055637` -- pass1: `Accuracy: 0.740`, bad prompt ids `[10,14,15,18,19,20,28,29,33,37,39,44,45]` -- pass2: `Accuracy: 0.760`, bad prompt ids `[10,14,15,18,20,28,29,33,37,39,44,45]` -- Prefill log confirms `parallel=20` is sufficient to hit the configured scheduler cap: many batches are `#new-seq: 5`, `#new-token: 320`, cached-token around `3456-3584`. - -Conclusion: `parallel=5` can under-drive batching; use `parallel=20` for the small GSM8K reproducer. The current failure is not merely warm-cache drift: p20 immediately exposes the bs>1 tiny-extend path with poor accuracy on first pass as well. - -Next focus remains the production CUDA/TAI MLA current-slot compose/dequant/FlashMLA consume path; CPU/reference batch rerange and slot-compose contracts already passed. - -## BB. Same-cache parallel=1 recovers accuracy, isolating failure to bs>1 consume path - -2026-06-08: after the p20 reproducer above, ran the same first 50 GSM8K questions on the same running service/cache with `--parallel 1`: - -- Result dir: `/mnt/beegfs/cjy/gsm8k_p1_after_p20_probe_20260608_060435` -- Result: `Accuracy: 0.960`, `Invalid: 0.000` - -This is a strong isolation result: the cache contents are not globally corrupted by prior p20 traffic. If L2/L1 cache storage or load-back had generally poisoned the cached prefix, p1 warm-cache should also stay bad. Instead p1 recovers to expected accuracy while p20 remains bad. - -Conclusion: continue focusing on bs>1 runtime consume/metadata semantics. L2 cache backup/load-back remains lower priority for this accuracy bug. - -Additional diagnostic test: - -- `test_tai_current_slot_fill_matches_torch_for_parallel20_tiny_fp8_rows` compares TAI current-slot fill against torch reference for the p20 tiny-extend shape, `page_size=64`, `prefix_lens=[704,704,640,704,704]`, `extend_lens=[29,9,60,9,17]`, row width 656 bytes. -- Remote result: `1 passed`. - -Conclusion: current evidence does not support TAI current-slot fill byte-copy as the root cause for p20 accuracy loss. - -## BC. FP8 paged dequant p20 slot-loc contract passed - -Added CUDA diagnostic test `test_fp8_paged_dequant_matches_reference_for_parallel20_slot_locs`: - -- Shape mirrors p20 tiny warm-cache requests: `prefix_lens=[704,704,640,704,704]`, `extend_lens=[29,9,60,9,17]`, `page_size=64`. -- Builds slot-dense FP8 MLA rows (`dim=656`) and request-packed `mixed_locs` through the same slot-remap helpers. -- Compares `dequantize_k_cache_paged()` output with a direct per-loc reference dequant. - -Remote result: `1 passed`. - -Conclusion: current evidence does not support FP8 paged dequant or slot-loc remapping as the root cause. Remaining high-priority suspects are now narrower: - -1. FlashMLA sparse consuming bs>1 compute-padded query rows and topk rows with an ordering/shape mismatch. -2. Index/topk batch path producing topk rows whose row order is not exactly the same as the q rows consumed by FlashMLA sparse. -3. Later layer pipeline preserving compute-padded rows through attention/MoE but selecting valid rows only at handoff; if an intermediate op reorders or compacts rows without preserving the compute-padded contract, p1 can pass while p20 fails. - -## BD. FlashMLA sparse dummy-row contract passed - -Added CUDA diagnostic test `test_flashmla_sparse_valid_rows_match_compact_with_compute_padding`: - -- Builds p20-like compute-padded q rows: 5 requests, 64 rows/request, valid rows `[29,9,60,9,17]`. -- Fills dummy rows' sparse indices with `-1`. -- Runs `flash_mla_sparse_fwd()` once on padded rows and once on compact valid-only rows. -- Compares only valid-row outputs. - -Remote result: `1 passed`. - -Conclusion: FlashMLA sparse itself handles compute-padded dummy rows with `-1` indices for this contract. The remaining likely bug is not the sparse kernel independently, but a production metadata/layout mismatch among: - -- `q_all` row order consumed by FlashMLA, -- `topk_indices` row order/offset returned by batch indexer, -- dequantized KV request-packed row order from `page_table_1_flattened`. - -## BE. parallel=20 isolates a compute-padding semantic hazard - -2026-06-08: `--parallel 20` is now the preferred small GSM8K reproducer because it reaches the current scheduler cap (`bs=5`). The p20 first-50 run produced `Accuracy: 0.740/0.760`, while the same cache with `--parallel 1` recovered to `Accuracy: 0.960`. - -Additional log evidence from `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_204531.log`: - -```text -event=batch_plan cp_rank=0 bs=5 extend_lens=[29,9,60,9,17] -valid_pages=[1,1,1,1,1] compute_pages=[8,8,8,8,8] -valid_local_tokens=[29,9,60,9,17] compute_local_tokens=[64,64,64,64,64] -compute_padding=True padding_tokens=[483,503,452,503,495] - -cp_rank=1..7 valid_local_tokens=[0,0,0,0,0] -compute_local_tokens=[64,64,64,64,64] -``` - -This means tiny requests are not just padded to the physical tail page. They are inflated from one valid page to `cp_size` compute pages, and every CP rank runs 64 synthetic rows per request through the full model stack. That is semantically risky with MoE/DeepEP because dummy rows can enter routing/dispatch/capacity/accounting even if cache writes later drop them. It also explains why targeted tests for current-slot fill, FP8 dequant, and FlashMLA sparse dummy indices passed while p20 accuracy still drops: those tests cover attention/cache consume locally, not full-layer MoE interaction with synthetic rows. - -Current hypothesis: compute-padding-to-`cp_size` is not a safe correctness mechanism for tiny bs>1. The next fix should prefer valid page-granular splits with zero-lane support over fabricating full-model dummy compute rows. If zero-lane support is incomplete, fail-fast at the precise unsupported boundary rather than silently padding through MoE. - -## BF. next evidence target: layer0 tensor dump for compute-padding validation - -当前 p20 失败已收敛到 bs>1 full-layer consume 语义，而不是单个低层 copy/kernel 单测独立失败。下一步不再继续猜测修复，改为在 target model layer0 加窄门控 tensor dump： - -- dump p1 与 p20 的 layer0 valid-path tensor； -- 对比阶段：layer input、prepare_attn 后、self_attn 后、prepare_mlp 后、mlp 后/postprocess 后； -- 若 layer input/prepare_attn valid rows 一致而 self_attn 或 MLP 后开始偏离，则可以定位到 attention/cache consume 或 MoE/DeepEP 与 compute-padding 的交互； -- 若 layer input 已不一致，则问题在 scheduler/split/position/input metadata 进入 layer 前。 - -实现约束：只用 `DUMPER_ENABLE` 触发；默认生产路径零行为变化；跳过 warmup 极短请求；每个 rank/stage 只 dump 首个 bs=1 与首个 bs>1 目标层 batch，避免日志/磁盘爆炸。 - -## BG. layer0 dump probe data file prepared - -为避免 layer0 dump 被前几个正常样本消耗，本轮准备了只包含已知 p20 稳定失败样本的 5 条 GSM8K 子集： - -```text -g0034:/mnt/beegfs/cjy/data/gsm8k_bad_probe_10_14_15_18_19.jsonl -orig ids: [10, 14, 15, 18, 19] -``` - -建议采集顺序：在启用 `DUMPER_ENABLE=1` 的新 prefill 进程中，先 `parallel=20` 打这 5 条形成一个 bs=5 batch，再 `parallel=1` 打同一文件。当前 dump 代码会跳过 warmup，并分别保存首个 `bs_gt1` 与首个 `bs1` target layer0 batch 的各阶段 tensor。 - -## BH. first layer0 dump attempt failed due dumper ctx duplicate layer_id - -首次启用 `DUMPER_ENABLE=1` 后，远端日志 `/mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_223657.log` 出现： - -```text -CP shared-KV layer0 tensor dump failed at stage=layer_input: dict() got multiple values for keyword argument 'layer_id' -``` - -原因：dumper 的全局 ctx 已经可能包含 `layer_id`，本次新增 dump tags 又显式传入 `layer_id`，触发 dumper 内部 `dict(..., **extra_kwargs, **global_ctx)` 重复 keyword。该问题只影响采证 dump，不影响模型推理语义。 - -修正：layer0 dump tags 不再传 `layer_id`，仍然通过 stage/cp_rank/batch_kind/compute_padding 区分文件；必要时从 dumper 全局 ctx 或 meta 读取层号。 - -## BI. layer0 dump window widened for p20 first-20 reproducer - -只打 `[10,14,15,18,19]` 五条子集时，远端返回 `Accuracy=1.000`，没有复现 p20 串扰。这说明错误可能依赖前序 batch/队列状态，而不是单独这五个 request 本身。 - -采证策略修正： - -- `bs_gt1`：每个 CP rank/stage dump 前 4 个 layer0 batch；覆盖此前 p20 first-20 中第 2/3/4 个稳定失败 batch。 -- `bs1`：每个 CP rank/stage dump 前 20 个 layer0 request；用于同一 first-20 prompt 顺序对照。 - -dump 文件增加 `dump_batch_index` tag，便于按 batch 顺序匹配。 - -## BJ. first p20 pass only populates cache; dump quota must cover second p20 warm pass - -用户指出 layer0 dump 必须先跑一轮填 cache。验证结果： - -```text -first p20 first-20: Accuracy=0.950 -second p20 first-20: Accuracy=0.750 -``` - -因此第一轮 p20 不能作为失败样本；它会消耗原先 `bs_gt1` 前 4 个 batch 的 dump quota，导致第二轮 warm-cache 失败 batch 不再 dump。 - -修正：`bs_gt1` dump 窗口从 4 个 batch 扩大到 10 个 batch，用来覆盖“第一轮 populate-cache + 第二轮 warm-cache”。`bs1` 仍保留 20 个 request 用于顺序对照。当前进程已经消耗旧 quota，需要重启后重新采集。 - -## BK. layer0 dump quota increased for router probes and two-pass warm-cache capture - -用户提醒 router 可能发送 health check / probe 请求。当前代码已经在计数前跳过 `total_extend_tokens < 16` 的短请求，但为了避免额外调度或非 GSM8K 请求消耗窗口，采证 quota 继续放大： - -- `bs_gt1` 与 `bs1` 均保留前 40 个有效 layer0 batch/request； -- 短 warmup/health 请求仍不会消耗 quota。 - -这会多占 dump 空间，但仅在 `DUMPER_ENABLE=1` 下生效，用于本轮 root-cause 取证。 - -## BL. layer0 dump evidence: matched valid rows are identical between p20 warm-hit and p1 - -采集流程：新进程开启 `DUMPER_ENABLE=1`，先跑 p20 first-20 填 cache，再跑 p20 first-20 复现，再跑 p1 first-20 对照。 - -结果： - -```text -p20 pass1: Accuracy=1.000 -p20 pass2: Accuracy=0.850 -p1 pass3: Accuracy=0.900 -Dump dir: /ssd/dump/cp_layer0_20260607_225959 -Log: /mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_225959.log -``` - -对 p20 第二轮 warm-hit 的 `bs_gt1 dump_batch_index=4..7` 与 p1 顺序请求按 `origin_sig` 对齐，在 CP0 上比较 valid rows： - -- 比较阶段：`layer_input`, `after_prepare_attn`, `after_self_attn`, `after_prepare_mlp`, `after_mlp`, `layer_output`。 -- 已匹配 request 的所有阶段 `max=0, mean=0, rel=0`。 -- positions 也未发现 mismatch。 - -样例： - -```text -BATCH 6 ext [30,35,26,18,59] prefix [704,704,704,704,704] -req0..req4: layer_input/after_prepare_attn/after_self_attn/after_prepare_mlp/after_mlp/layer_output all max=0 - -BATCH 7 ext [21,19,61,33] prefix [704,704,640,704] -req0..req3: all stages max=0 -``` - -解释：这是一条反证。至少在 target model layer0、CP0 valid rows 上，compute padding 没有导致 p20 与 p1 的语义差异；dummy rows 也没有通过 layer0 attention/MLP/DeepEP 影响这些 valid rows。当前不能把 root cause 直接归因于 layer0 compute padding 污染。 - -仍未排除： - -1. 后续层才出现 divergence； -2. 其他 CP rank 的 valid rows（非 CP0 owner）或跨层 all-gather/rerange 有问题； -3. EAGLE/draft 路径或 prefill handoff/logits/topk 在 layer0 之后出现 batch 对齐错误； -4. 少数 p1 对照没有 dump，因为当前 dump 仍跳过 `total_extend_tokens < 16`，导致 ext=3/6/10/13 等短请求无法比较。 - -下一步更有效的证据点应从 layer0 后移：dump final collected hidden/logits 或在 selected later layers 做同样 valid-row 对比，而不是继续只盯 layer0。 - -## BM. Next evidence boundary: keep layer0 and add final-layer/final-output dumps - -User direction: focus on layer0 and the last layer, i.e. beginning and end. - -Current evidence from BL shows CP0 valid rows at target layer0 are bitwise identical between bs>1 warm-cache and bs=1 matched prompts. Therefore continuing to only inspect layer0 is unlikely to isolate the accuracy drop. - -Planned opt-in evidence points: - -- keep existing target layer0 stage dumps; -- add target last-layer input/output dumps only, not every middle layer; -- add final model hidden dumps around norm and CP last-token collect; -- add compact logits top-k/argmax dump after `LogitsProcessor`, avoiding full-vocab tensor dumps unless later needed. - -All points must stay gated by existing dumper enablement. To work with the current remote `DUMPER_FILTER="name.startswith('cp_shared_kv_layer0')"`, new dump records intentionally keep the `cp_shared_kv_layer0_*` prefix while encoding `dump_scope` and `stage` in metadata/tags. - -## BN. Implemented final-boundary dump hooks for the next cache-hit accuracy probe - -Implemented opt-in dump hooks in `python/sglang/srt/models/deepseek_v2.py`: - -- existing layer0 dumps remain unchanged in semantics; -- last target layer now dumps `last_layer_input` and `last_layer_output`; -- final model path dumps `model_final_pre_norm`, `model_final_post_norm_pre_collect`, and post CP collect/rerange hidden; -- logits path dumps only top-16 indices/values plus argmax at `logits_processor_next_token_topk`, not full-vocab logits. - -All dumps are still gated by existing dumper enablement and preserve the `cp_shared_kv_layer0_*` filename prefix so the current remote filter keeps capturing them. Metadata now includes `dump_scope` (`layer0`, `last_layer`, `model_final`, `logits`) and `target_layer_id` where applicable. - -Verification so far: local `python -m py_compile python/sglang/srt/models/deepseek_v2.py` passed. Runtime evidence requires service restart because the running process has the old module loaded. - -## BO. Prepared boundary comparison script for layer0/final-layer probes - -Remote helper script prepared at: - -```text -/mnt/beegfs/cjy/compare_cp_boundaries.py -``` - -It parses dump tags and scheduler `origin_sig`, then compares bs>1 requests with bs=1 requests for the same prompt signature on CP rank 0. It supports the new boundary scopes: - -- `layer0/layer_input`, `layer0/layer_output`; -- `last_layer/last_layer_input`, `last_layer/last_layer_output`; -- `model_final/model_final_pre_norm`, `model_final/model_final_post_norm_pre_collect`, post collect/rerange compact hidden; -- `logits/logits_processor_next_token_topk` top-index equality and top-value drift. - -Usage after a restarted service captures new dumps: - -```bash -python /mnt/beegfs/cjy/compare_cp_boundaries.py \ - --dump-dir /ssd/dump/ \ - --log /mnt/beegfs/cjy/log/ \ - --max-batches 12 -``` - -The script can run against old layer0-only dumps, but last-layer/model-final/logits comparisons require the new `deepseek_v2.py` hooks and therefore a service restart after code sync. - -## BP. Mooncake version/path triage for current bs>1 cache-hit accuracy issue - -User note: another per-layer development environment upgraded Mooncake to `0.3.11.post1`; check whether current issue is Mooncake-related. - -Findings on `g0034` container: - -- `import mooncake` succeeds from `/usr/local/lib/python3.12/dist-packages/mooncake`. -- `mooncake.__version__`, `mooncake.version`, and `mooncake.VERSION` are all `None`. -- `pip show mooncake` reports `Package(s) not found: mooncake`; the installed package does not expose normal wheel metadata, so the exact pip version cannot be confirmed from Python metadata. -- `mooncake/engine.cpython-312-x86_64-linux-gnu.so` is present and logs normal Transfer Engine startup (`Using RDMA transport`, topology discovery, etc.). - -Current service path: - -- launch args show `disaggregation_transfer_backend='mooncake'`, so ordinary prefill->decode KV transfer uses Mooncake; -- code gate for per-layer transfer is `SGLANG_CP_SHARED_KV_PER_LAYER_TRANSFER`, default `False`; -- current launch args/env do not include `SGLANG_CP_SHARED_KV_PER_LAYER_TRANSFER=1`; -- latest log contains no `CP_PER_LAYER_TRANSFER` registration/finish lines, so current reproduction is **not** exercising the per-layer Mooncake transfer path. - -Root-cause relevance: - -- Existing evidence shows some bad cases already have suspect `prefill_result_handoff` next-token behavior before decode consumes transferred KV. -- Therefore Mooncake can still affect decode continuation / KV integrity / hangs, but it is unlikely to be the primary root cause of the current bs>1 warm-cache GSM8K accuracy drop if final prefill logits/next-token are already divergent. -- The next boundary dump should decide this cleanly: if `model_final_post_cp_collect` or `logits_processor_next_token_topk` diverges in prefill, Mooncake is outside the first failing boundary. If prefill logits match but decode output diverges, then Mooncake transfer/version becomes a high-priority suspect. - -Action: keep Mooncake in the risk ledger, but do not change Mooncake or per-layer transfer while investigating the prefill final-boundary divergence. - -## BQ. Boundary dump localizes warm-cache bs>1 divergence after layer0 and before final target layer - -Latest restarted service with final-boundary dumps: - -```text -Log: /mnt/beegfs/cjy/log/sglang_cp_hicache_20260607_231847.log -Dump dir: /ssd/dump/cp_layer0_20260607_231847 -Probe: /mnt/beegfs/cjy/gsm8k_boundary_probe_20260607_232341 -Compare: /mnt/beegfs/cjy/compare_cp_boundaries_20260607_232341.txt -``` - -Probe results: - -```text -p20 populate: Accuracy=1.000 -p20 warm: Accuracy=0.800 -p1 warm: Accuracy=0.900 -``` - -Boundary comparison for p20 warm-cache batches (`dump_batch_index=4..7`) against p1 same-origin prompts on CP0: - -- `layer0/layer_input`: max diff 0 for matched valid rows. -- `layer0/layer_output`: max diff 0 for matched valid rows. -- `last_layer/last_layer_input`: already diverged for several requests. -- Divergence persists through `last_layer_output`, `model_final_post_norm`, `model_final_post_cp_collect`, and logits top-k. - -Representative rows from the comparison: - -```text -batch4 req3 ext=18 prefix=704: layer0 max=0, last_layer_input max=11.9844, post_collect max=3.71484 -batch5 req2 ext=34 prefix=704: layer0 max=0, last_layer_input max=27.625, post_collect max=6.59717, logits arg0 differs 92983 vs 5512 -batch6 req3 ext=18 prefix=704: layer0 max=0, last_layer_input max=20.1562, logits arg0 differs 62792 vs 220 -batch7 req2 ext=61 prefix=640: layer0 max=0, last_layer_input max=22.7188, logits arg0 differs 220 vs 2932 -``` - -Conclusion: - -- The current accuracy regression is not caused by layer0 input/output corruption on the compared CP0 valid rows. -- The first observed failing boundary is between target layer0 output and target last-layer input. -- Because final prefill hidden/logits are already divergent, ordinary Mooncake prefill->decode transfer is not the first failing boundary for this specific GSM8K warm-cache bs>1 correctness issue. - -Next evidence step: - -- Add an opt-in configurable intermediate-layer dump list (for example `SGLANG_CP_SHARED_KV_DUMP_LAYERS=0,20,40,60,77`) and binary-search the first layer where bs>1 warm-cache diverges from bs=1 same-origin prompts. -- Keep dumps gated and sparse; do not add always-on runtime checks or production overhead. - -## BR. Scope correction: do not investigate Mooncake for current prefill KV correctness probe - -User clarified that the local environment is not suitable for Mooncake investigation and that per-layer transfer is not enabled in the current run. - -Current debugging scope is therefore narrowed to prefill-side correctness only: - -- do not spend more time on Mooncake version/path checks for this issue; -- verify whether CP shared-KV/HiCache warm-cache bs>1 produces or consumes incorrect KV inside prefill forward; -- use opt-in dumper evidence rather than adding production-time checks; -- focus on KV cache content at the attention consume boundary for layer0/intermediate/final layers. - -Rationale: prior boundary dumps show layer0 hidden rows match but final prefill hidden/logits diverge. Since per-layer transfer is disabled and final prefill output is already suspect, the next useful boundary is the paged KV cache consumed by target attention, not prefill->decode transfer. - -## BS. Current pass: stop remote/dump, unit-test intermediate prefill invariants first - -User direction for this pass: - -- remote service is stopped; -- do not add another dump yet; -- do not investigate Mooncake/per-layer transfer for this issue; -- continue reading the prefill middle path and add unit tests for suspicious invariants. - -Current localized evidence remains: - -- warm-cache bs>1 and bs=1 matched prompts are bitwise equal on CP0 target layer0 input/output valid rows; -- divergence is first observed later, before the final target layer input; -- final prefill hidden/logits already diverge, so decode transfer is not the first failing boundary. - -Most useful next invariant to test without runtime dumps: - -1. RAGGED MLA cache-hit compose must produce a compact KV row space whose row IDs match the row space used by NSA index/top-k offsets for every request in a bs>1 batch. -2. Persistent cache writes must use only valid rows and preserve request boundaries across layers; compute-padding/page-tail rows must not shift later-layer request-local coordinates. -3. The cached remap for page-slot materialization must be batch/request-layout scoped enough that a later layer cannot reuse a stale row mapping from another request shape. - -This pass will prioritize small CPU/unit tests around those mapping contracts before any code fix. - -## BT. Suspicious invariant: compute-padding rows are not a suffix before sparse/MoE top-k - -New read-only evidence for the current no-dump pass: - -- GLM-5.1 FP8 config on `g0034:/ssd/models/GLM-5.1-FP8/config.json`: - - `num_hidden_layers=78` - - `first_k_dense_replace=3` - - `moe_layer_freq=1` - - `n_routed_experts=256` - - `num_experts_per_tok=8` -- This matches the previous boundary dump: target layer0 can match exactly while later layers diverge, because layer0 is dense and sparse/MoE begins at layer3. - -Code path evidence: - -- `build_batch_page_aligned_in_seq_split_plan()` enables compute padding when a request has fewer physical pages than `cp_size`. -- For tiny extends, each CP rank receives compute rows, but many of those rows are synthetic page-fill rows. In the common 1-page suffix case, non-owner ranks can have compute rows with zero valid rows. -- `DeepseekV2DecoderLayer.forward()` passes the full `hidden_states` tensor through `self.mlp(...)` after `prepare_mlp()`; there is no CP shared-KV valid-row compaction before sparse/MoE. -- `DeepseekV2MoE.forward_deepep()` calls `self.topk(..., num_token_non_padded=forward_batch.num_token_non_padded, ...)`. -- `forward_batch.num_token_non_padded` is a scalar based on total original input tokens. It can only represent a contiguous valid prefix/suffix-padded layout. CP compute-padding invalid rows are per-request/per-rank and can be interleaved inside the local tensor, so the scalar cannot mask them correctly. - -Concrete suspected failure shape: - -- For a bs>1 warm-cache batch with tiny suffixes like `extend=[29,9,60,9,17]`, `page_size=64`, `cp_size=8`, compute padding creates `64` local rows per request on every CP rank. -- On non-owner ranks, valid local current rows for those requests can be `0`, but the local compute tensor still has `5 * 64 = 320` rows. -- The scalar `num_token_non_padded=sum(extend_lens)=124` or similar would leave the first 124 synthetic local rows active in MoE routing, rather than masking all zero-valid rows on that rank. - -Hypothesis: - -- The cache-hit accuracy drop is caused by compute-padding rows entering sparse/MoE routing/dispatch/capacity as real tokens in bs>1 tiny-extend batches. -- This explains why layer0 input/output valid rows match while final prefill hidden/logits diverge: layer0 is dense; the first likely semantic contamination point is sparse layer3. - -Next unit-test target: - -- Add a CPU/unit contract test showing that CP compute-padding creates non-suffix invalid rows and that a single scalar `num_token_non_padded` is insufficient for sparse/MoE top-k masking. This should become the guardrail before choosing the final fix (compact-before-MoE, row-mask-capable MoE, or disabling compute padding for sparse layers). - -### BT unit tests added - -Added CPU/unit guardrails in `test/registered/unit/layers/test_nsa_cp_utils.py`: - -- `test_compute_padding_tiny_batch_valid_rows_are_not_suffix_maskable_for_moe` - - Builds the p20-like tiny batch `extend=[29,9,60,9,17]`, `prefix=[704,704,640,704,704]`, `page_size=64`, `cp_size=8`, `cp_rank=0`. - - Confirms compute rows are `320`, valid rows are `124`, and valid rows are not a contiguous tensor prefix. Therefore a scalar suffix mask (`num_token_non_padded`) cannot express the valid-row layout. -- `test_compute_padding_non_owner_rank_scalar_non_padded_would_unmask_dummy_rows` - - Same batch on `cp_rank=1`. - - Confirms compute rows are `320`, valid rows are `0`, while the global non-padded count is positive. Reusing the global scalar would unmask dummy rows on a zero-valid rank. - -Verification: - -```bash -ssh g0034 "docker exec sglang-glm5-dev-2 bash -lc 'cd /sgl-workspace/sglang-tai && PYTHONPATH=python python -m pytest -q test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_compute_padding_tiny_batch_valid_rows_are_not_suffix_maskable_for_moe test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_compute_padding_non_owner_rank_scalar_non_padded_would_unmask_dummy_rows'" -``` - -Result: `2 passed, 5 warnings`. - -Local verification remains blocked by missing `orjson` in `/root` Python environment; remote container has the required dependencies. - -Additional related verification for the same invariant group: - -```bash -ssh g0034 "docker exec sglang-glm5-dev-2 bash -lc 'cd /sgl-workspace/sglang-tai && PYTHONPATH=python python -m pytest -q \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_batch_plan_compute_padding_only_pads_tiny_request_in_mixed_batch \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_batch_plan_compute_padding_is_per_request_not_batch_total \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_compute_padding_tiny_batch_valid_rows_are_not_suffix_maskable_for_moe \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_compute_padding_non_owner_rank_scalar_non_padded_would_unmask_dummy_rows \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_collect_last_token_hidden_matches_parallel20_tiny_extend_layout \ - test/registered/unit/layers/test_nsa_cp_utils.py::TestNSAInSeqCPUtils::test_batch_in_seq_all_gather_rerange_matches_parallel20_tiny_extend_layout'" -``` - -Result: `6 passed, 5 warnings`. - -## 2026-06-08 BU. Agent2 page_inverse duplicate-alias hypothesis - -Question checked: whether bs>1 dense materialize/current-reuse can corrupt cache-hit requests because `page_inverse` is batch-global and keyed only by logical page id. - -Evidence: - -- `python/sglang/srt/layers/attention/nsa/cp_shared_kv_runtime.py:2902-2931` - `build_slot_page_remap()` intentionally preserves every flattened page-table slot and documents that duplicate logical pages may become duplicate dense slots. -- `python/sglang/srt/layers/attention/nsa/cp_shared_kv_runtime.py:2943-2974` - `build_slot_page_inverse()` builds a single `logical_page -> dense_slot` map with `scatter_`. Duplicate logical pages cannot be represented; the later slot wins in the Python fallback. -- `python/sglang/srt/layers/attention/nsa/cp_shared_kv_runtime.py:2977-3012` - `remap_logical_locs_to_slot_dense_locs()` maps all token locs through that one inverse, so all occurrences of the same logical page id target the same dense slot. -- `python/sglang/srt/layers/attention/nsa/cp_shared_kv_runtime.py:3584-3716` - MLA prefix+current compose materializes prefix slots, then remaps `logical_locs` and fills current rows via the same `slot_remap.page_inverse`. -- `python/sglang/srt/layers/attention/nsa/cp_shared_kv_runtime.py:3772-3905` - Index prefix+current compose has the same inverse-based slot contract. -- `python/tai_kernel/nsa_prefill/csrc/cp_shared_kv_materialize.cu:163-176` and `202-226` - TAI kernels assign `page_inverse[lp] = dense_page` without atomic/duplicate handling. Duplicate logical pages are therefore last-writer/race dependent on GPU. This is not only a TAI issue: the Python fallback also collapses duplicates by construction. -- `python/tai_kernel/nsa_prefill/csrc/cp_index_mqa_prepare.cu:79-116` and `374-444` - The TAI index-MQA prepare path is segment/request-aware through `batch_indices`, `kv_lens`, `k_bases`, and `q_bases`; it does not rely only on a global logical-page inverse. - -Inference: - -- The hypothesis is structurally valid: the current slot-remap contract says duplicate logical pages can have duplicate dense slots, but its inverse cannot address duplicate slots. Any consumer that needs request-concat positional identity can be redirected to another request's slot. -- This fits the observed shape: bs=1 has no cross-request duplicate slots; warm-cache bs>1 with shared prefixes/current reuse can create repeated logical page ids; second GSM8K pass loses accuracy while first pass is mostly clean. -- The strongest corrupting case is duplicate logical page ids that are not semantically identical, or current/suffix locs that collide. If duplicates are true shared-prefix pages with identical KV content, content aliasing alone may be safe, but positional/dequant/top-k consumers can still be wrong if they expect one dense slot per request slot. -- Agent2's `index-GetK path has k_bases, so it is safe` is only true for the fused index-MQA prepare/GetK concat path. The separate index page-slot materialize path still uses `page_inverse` and should be checked under bs>1 cache-hit. - -Unknown / next discriminating checks: - -1. Add a unit test with two bs>1 requests whose `remap_logical_pages` contains a repeated positive logical page and whose downstream `logical_locs` require per-request slot identity. Current code should show both requests remap to one dense slot. -2. Add a production-shape test for RAGGED cache-hit compose where prefix pages are shared but current pages differ; verify whether dequant/top-k consumers require duplicate dense slots or can safely dedupe shared-prefix content. -3. Under existing `SGLANG_CP_SHARED_KV_BS_GT1_DEBUG`, count duplicate positive pages in `metadata.real_page_table`, classified by prefix slots vs current slots. Current/suffix duplicates should be fail-fast; prefix duplicates require a clear dedupe-or-slot contract. - -Status: hypothesis has medium-high confidence as a real correctness risk; root cause of the GSM8K warm-cache drop is not fully proven until the duplicate class and consumer contract are verified. - -### BU reproduction test added - -Added a focused RED unit test: - -- `test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_token_slot_remap_duplicate_pages_keep_request_slot_identity` - -Shape: - -- `page_size=4`, `cp_size=1`. -- `remap_logical_pages=[[1,2],[1,3]]`: two requests share logical page 1 as their first slot. -- Slot layout says dense pages should be `[1,2,3,4]` for the four request slots. -- Desired request-slot-preserving dense locs: `[[4,8],[12,16]]`. - -Observed failure on `g0034` remote container: - -```text -E AssertionError: Lists differ: [[12, 8], [12, 16]] != [[4, 8], [12, 16]] -``` - -Interpretation: - -- `slot_logical_pages=[1,2,1,3]` is preserved, but `page_inverse[1]` points to the second occurrence's dense slot. -- Req0's logical loc on page 1 is remapped from expected row `4` to row `12`, aliasing req1's page-1 slot. -- This reproduces the duplicate-page alias at the CPU/fallback contract level. It is therefore not only a TAI-kernel issue; TAI adds non-atomic/race behavior on top of an already lossy inverse contract. - -Verification command: - -```bash -ssh g0034 "docker exec sglang-glm5-dev-2 bash -lc 'cd /sgl-workspace/sglang-tai && PYTHONPATH=python python -m pytest -q test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_token_slot_remap_duplicate_pages_keep_request_slot_identity'" -``` - -Result: expected RED, `1 failed` with the alias shown above. - -### BU fix: row-scoped slot inverse for bs>1 duplicate logical pages - -Root cause fixed in this pass: - -- The dense slot layout is request-slot based: duplicate logical pages are allowed because two batch rows can reference the same logical page id at different request-local positions. -- The old `page_inverse` was batch-global (`logical_page -> dense_page`) and therefore lossy under duplicates. Python `scatter_` deterministically kept one occurrence; TAI kernels used non-atomic writes and could be nondeterministic. -- Cache-hit bs>1 materialize/current-reuse paths then remapped token/page indices through that lossy inverse, so one request could alias another request's dense slot. - -Implemented contract: - -- Keep the existing global `page_inverse` only for legacy/no-row-context paths. -- Add `page_inverse_by_row` for rectangular bs>1 page tables: `request_row, logical_page -> dense_page`. -- Add row-aware token/page remap helpers. If row context exists, remap through `page_inverse_by_row`; otherwise fall back to the old global inverse. -- Pass flattened request row ids into the FP8 RAGGED MLA cache-hit compose path, where `logical_locs` is flattened but the page table is request-shaped. -- Update MLA/index prefetch consume paths to use the shared row-aware helpers when the cached remap carries row-scoped inverse metadata. - -Verification on `g0034` remote container: - -```bash -# RED test now GREEN -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_token_slot_remap_duplicate_pages_keep_request_slot_identity -# result: 1 passed - -# Related materialize/current-reuse regression set -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_build_shared_token_kv_slot_remap_reuses_slot_layout \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_token_slot_remap_duplicate_pages_keep_request_slot_identity \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_materialize_batch_prefix_span_and_reuse_current_kv_page_slots \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_cp8_batch_kv_partial_current_keeps_request_packed_layout \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_fp8_ragged_mla_uses_page_slot_current_compose_for_cache_hit \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_index_prefetch_partial_current_compose_fills_current_page_slots \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_cp8_index_partial_current_compose_matches_rank_merged_reference_for_bs5 -# result: 7 passed -``` - -Remaining risk: - -- The TAI full-materialize kernels still expose only a global inverse. The Python runtime now avoids that path when row-scoped remap is required, but the CUDA kernel contract should eventually be upgraded to row-scoped descriptors instead of relying on the Python fallback for bs>1 duplicate-page cases. -- `page_inverse_by_row` memory is `O(batch_size * logical_page_capacity)`. Current production gates use small bs, so this is acceptable as a correctness fix; if bs grows substantially, replace it with a compact per-row descriptor/kernel. - -### BU follow-up: replace dense row inverse with compact row-slot descriptor - -Correction after review: - -- The first row-scoped fix used a dense CUDA tensor shaped `[batch_size, logical_page_capacity]`. -- That was correctness-safe but not acceptable for production memory use, especially with `bs` up to 10 and long contexts. - -Updated implementation contract: - -- Removed the dense `page_inverse_by_row` allocation entirely. -- Store only compact descriptors with the same shape as the actual batch page table: - - `slot_sorted_logical_pages_by_row`: `[batch_size, actual_pages_per_row]` - - `slot_sorted_dense_pages_by_row`: `[batch_size, actual_pages_per_row]` -- Row-aware remap loops over request rows and uses `torch.searchsorted` on that row's compact sorted page list. -- Space is now `O(actual_batch_pages)`, not `O(batch_size * logical_page_capacity)`. - -Tradeoff: - -- This adds a small per-row remap compute cost versus direct dense inverse indexing. -- Current production gate keeps `bs` small enough that this is acceptable. If this path becomes a CPU/GPU launch bottleneck, the next step is a TAI row-aware remap kernel over the same compact descriptors. - -Verification on `g0034` remote container: - -```bash -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py::TestCpSharedKVRuntimeHelpers::test_token_slot_remap_duplicate_pages_keep_request_slot_identity -# result: 1 passed - -PYTHONPATH=python python -m pytest -q test/registered/unit/mem_cache/test_cp_shared_kv_runtime.py -# result: 126 passed, 2 subtests passed - -PYTHONPATH=python python -m pytest -q \ - test/registered/unit/layers/test_nsa_cp_utils.py \ - test/registered/unit/mem_cache/test_cp_shared_kv_layout.py -# result: 137 passed -``` - -### BV cleanup: remove opt-in tensor dump hooks and temporary index trace logs - -Cleanup status for the current code state: - -- Removed the temporary `deepseek_v2.py` tensor-dump runtime hooks used for layer0/last-layer/model-final/logits boundary evidence. -- Removed temporary `nsa_indexer.py` request-origin/top-k/direct-write detail logs that were only intended for the GSM8K bs>1 investigation. -- Kept semantic regression tests, fail-fast checks, and explicit fallback/warning paths. -- Historical dump findings above remain as an investigation ledger only; they should not be read as active runtime instrumentation. - -Directive: do not reintroduce tensor dumps or high-cardinality per-request logs into the default runtime path. If another boundary probe is needed, make it opt-in, narrow, and remove it after collecting evidence. diff --git a/python/sglang/srt/layers/attention/nsa/utils.py b/python/sglang/srt/layers/attention/nsa/utils.py index 735d5fdb5..098115d9d 100644 --- a/python/sglang/srt/layers/attention/nsa/utils.py +++ b/python/sglang/srt/layers/attention/nsa/utils.py @@ -1192,6 +1192,58 @@ def select_cp_local_valid_rows_for_cache_write( return selected +def restore_cp_local_valid_rows_for_moe( + forward_batch, + compact_tensor: torch.Tensor, + local_compute_reference: torch.Tensor, +) -> torch.Tensor: + """Restore compact CP valid rows to the local compute-padded row layout. + + CP shared-KV compute padding pads each request to a full CP lane set so + attention/index kernels keep a page/lane-stable layout. Sparse MoE must not + route those dummy rows: they are not suffix padding and cannot be represented + by ``num_token_non_padded``. The MoE path therefore runs on compact valid + rows and scatters the result back to the original local compute layout before + the layer communicator sees it. + """ + + plan = get_cp_shared_kv_batch_plan(forward_batch) + if plan is None or not bool(getattr(plan, "compute_padding_enabled", False)): + return compact_tensor + + indices, expected_compute_rows = _get_cp_local_valid_row_indices_cache( + forward_batch, + plan, + compact_tensor.device, + ) + local_rows = int(local_compute_reference.shape[0]) + valid_rows = int(indices.numel()) + compact_rows = int(compact_tensor.shape[0]) + if local_rows != expected_compute_rows: + raise RuntimeError( + "[CP_SHARED_KV_FAIL_FAST][moe_restore_compute_rows_shape] " + "CP shared-KV MoE restore reference must contain local compute rows. " + f"local_rows={local_rows} expected_compute_rows={expected_compute_rows} " + f"valid_rows={valid_rows}" + ) + if compact_rows != valid_rows: + raise RuntimeError( + "[CP_SHARED_KV_FAIL_FAST][moe_restore_valid_rows_shape] " + "CP shared-KV MoE compact output must contain local valid rows. " + f"compact_rows={compact_rows} valid_rows={valid_rows} " + f"local_rows={local_rows}" + ) + if valid_rows == local_rows: + return compact_tensor + restored = local_compute_reference.new_zeros( + (local_rows, *compact_tensor.shape[1:]), + dtype=compact_tensor.dtype, + ) + if valid_rows > 0: + restored.index_copy_(0, indices, compact_tensor) + return restored + + def select_cp_current_valid_rows_for_reuse( forward_batch, current_tensor: torch.Tensor, diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 720cbe3c3..a6ecbf5cc 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -60,9 +60,11 @@ from sglang.srt.layers.attention.nsa.utils import ( cp_collect_last_token_hidden, cp_split_and_rebuild_data, cp_split_and_rebuild_position, + restore_cp_local_valid_rows_for_moe, is_nsa_enable_prefill_cp, nsa_use_prefill_cp, prepare_input_dp_with_cp_dsa, + select_cp_local_valid_rows_for_cache_write, ) from sglang.srt.layers.communicator import ( LayerCommunicator, @@ -755,6 +757,22 @@ class DeepseekV2MoE(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: + local_compute_hidden_states = None + if nsa_use_prefill_cp(forward_batch): + plan = getattr( + getattr(forward_batch, "nsa_cp_metadata", None), + "batch_plan", + None, + ) + if plan is not None and bool( + getattr(plan, "compute_padding_enabled", False) + ): + local_compute_hidden_states = hidden_states + hidden_states = select_cp_local_valid_rows_for_cache_write( + forward_batch, + hidden_states, + ) + shared_output = None sbo_enabled_flag = self._fuse_shared_experts_inside_sbo and not self.is_nextn sbo_overlap_dispatch_flag = ( @@ -959,6 +977,13 @@ class DeepseekV2MoE(nn.Module): ): final_hidden_states *= self.routed_scaling_factor + if local_compute_hidden_states is not None: + final_hidden_states = restore_cp_local_valid_rows_for_moe( + forward_batch, + final_hidden_states, + local_compute_hidden_states, + ) + return final_hidden_states def _forward_shared_experts( diff --git a/test/registered/unit/layers/test_nsa_cp_utils.py b/test/registered/unit/layers/test_nsa_cp_utils.py index cacc82485..c0979726b 100644 --- a/test/registered/unit/layers/test_nsa_cp_utils.py +++ b/test/registered/unit/layers/test_nsa_cp_utils.py @@ -1905,6 +1905,56 @@ class TestNSAInSeqCPUtils(unittest.TestCase): self.assertEqual(selected.tolist(), [[50.0, 51.0]]) + def test_restore_cp_local_valid_rows_for_moe_keeps_dummy_rows_zero(self): + import torch + + from sglang.srt.layers.attention.nsa.utils import ( + restore_cp_local_valid_rows_for_moe, + ) + + plan = build_batch_page_aligned_in_seq_split_plan( + extend_lens=[29, 9, 60, 9, 17], + prefix_lens=[704, 704, 640, 704, 704], + page_size=64, + cp_size=8, + cp_rank=0, + ) + forward_batch = SimpleNamespace( + nsa_cp_metadata=NSAContextParallelMetadata( + batch_size=5, + batch_plan=plan, + ) + ) + local_compute_rows = torch.zeros( + (sum(plan.request_compute_rank_local_tokens), 2), dtype=torch.float32 + ) + compact_valid = torch.arange( + sum(plan.request_valid_rank_local_tokens) * 2, dtype=torch.float32 + ).view(-1, 2) + + restored = restore_cp_local_valid_rows_for_moe( + forward_batch, + compact_valid, + local_compute_rows, + ) + + self.assertEqual(tuple(restored.shape), tuple(local_compute_rows.shape)) + valid_rows = select_cp_local_valid_rows_for_cache_write( + forward_batch, + restored, + ) + self.assertTrue(torch.equal(valid_rows, compact_valid)) + + valid_mask = torch.zeros(restored.shape[0], dtype=torch.bool) + cursor = 0 + for compute_len, valid_len in zip( + plan.request_compute_rank_local_tokens, + plan.request_valid_rank_local_tokens, + ): + valid_mask[cursor : cursor + valid_len] = True + cursor += compute_len + self.assertTrue(torch.equal(restored[~valid_mask], torch.zeros_like(restored[~valid_mask]))) + def test_select_cp_current_valid_rows_accepts_global_rows_under_compute_padding( self, ):