[Feature] Xiaomi MiMo-V2-Flash day0 support (#15207)

Co-authored-by: 谢学扬 <xiexueyang@xiaomi.com>
Co-authored-by: tz <tangzhen3@xiaomi.com>
Co-authored-by: 李家乐 <lijiale10@xiaomi.com>
Co-authored-by: 张晨 <zhangchen50@xiaomi.com>
Co-authored-by: Shaohui Liu <liushaohui3@xiaomi.com>
Co-authored-by: 王晨 <wangchen77@xiaomi.com>
Co-authored-by: jiangzihan <jiangzihan@xiaomi.com>
Co-authored-by: xiexueyang <xyxie_wangyi@163.com>
Co-authored-by: Linghao Zhang <zhanglinghao@xiaomi.com>
Co-authored-by: ispobock <ispobaoke@gmail.com>
Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
Co-authored-by: JoyFuture <35593546+JoyFuture@users.noreply.github.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
Co-authored-by: Qiaolin Yu <liin1211@outlook.com>
Co-authored-by: root <root@bj9-ml-g8h20e-k8s-slave106-20251106.alicn.idc.xiaomi.com>
This commit is contained in:
Yingchun Lai
2025-12-19 11:40:07 +08:00
committed by GitHub
co-authored by 谢学扬 tz 李家乐 张晨 Shaohui Liu 王晨 jiangzihan xiexueyang Linghao Zhang ispobock Liangsheng Yin JoyFuture Liangsheng Yin Qiaolin Yu root
parent a0985dd5e5
commit 160a06cab2
38 changed files with 5396 additions and 169 deletions
@@ -240,6 +240,13 @@ class DecodePreallocQueue:
self.prefill_pp_size = prefill_pp_size
self.kv_manager = self._init_kv_manager()
if self.scheduler.tp_worker.is_hybrid_swa:
# FIXME: current SWA allocation allocate full kv cache size in prefill
self.max_total_num_tokens = min(
self.max_total_num_tokens,
self.scheduler.tp_worker.model_runner.swa_max_total_num_tokens,
)
def _init_kv_manager(self) -> BaseKVManager:
kv_args_class = get_kv_class(self.transfer_backend, KVClassType.KVARGS)
kv_args = kv_args_class()
@@ -132,13 +132,13 @@ class ScheduleBatchDisaggregationDecodeMixin:
# Simulate the eagle run.
if self.spec_algorithm.is_eagle():
b = len(self.reqs)
topk = server_args.speculative_eagle_topk
num_states = server_args.speculative_eagle_topk
if server_args.enable_mtp:
num_states *= server_args.speculative_num_steps
topk_p = torch.stack(
[
torch.as_tensor(
req.output_topk_p[:topk],
req.output_topk_p[:num_states],
device=self.device,
dtype=torch.float32,
)
@@ -149,7 +149,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
topk_index = torch.stack(
[
torch.as_tensor(
req.output_topk_index[:topk],
req.output_topk_index[:num_states],
device=self.device,
dtype=torch.int64,
)
@@ -300,18 +300,24 @@ class MooncakeKVManager(CommonKVManager):
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage = (
self.get_mha_kv_ptrs_with_pp(src_data_ptrs, dst_data_ptrs)
)
# item_lens structure: [k_layer0, k_layer1, ..., k_layerN, v_layer0, v_layer1, ..., v_layerN]
# Use correct item lengths for K and V separately
if layers_current_pp_stage > len(dst_k_ptrs):
logger.error(
f"layers_current_pp_stage is out of range: {layers_current_pp_stage=}, {len(dst_k_ptrs)}"
)
layers_params = [
(
src_k_ptrs[layer_id],
dst_k_ptrs[layer_id],
item_lens[layer_id],
item_lens[layer_id], # K item length
)
for layer_id in range(layers_current_pp_stage)
] + [
(
src_v_ptrs[layer_id],
dst_v_ptrs[layer_id],
item_lens[layer_id],
item_lens[layers_current_pp_stage + layer_id], # V item length
)
for layer_id in range(layers_current_pp_stage)
]
@@ -109,6 +109,13 @@ class PrefillBootstrapQueue:
self.transfer_backend = transfer_backend
self.kv_manager = self._init_kv_manager()
if self.scheduler.tp_worker.is_hybrid_swa:
# FIXME: current SWA allocation allocate full kv cache size in prefill
self.max_total_num_tokens = min(
self.max_total_num_tokens,
self.scheduler.tp_worker.model_runner.swa_max_total_num_tokens,
)
def _init_kv_manager(self) -> BaseKVManager:
kv_args_class = get_kv_class(self.transfer_backend, KVClassType.KVARGS)
kv_args = kv_args_class()