[Feature] Xiaomi MiMo-V2-Flash day0 support (#15207)
Co-authored-by: 谢学扬 <xiexueyang@xiaomi.com> Co-authored-by: tz <tangzhen3@xiaomi.com> Co-authored-by: 李家乐 <lijiale10@xiaomi.com> Co-authored-by: 张晨 <zhangchen50@xiaomi.com> Co-authored-by: Shaohui Liu <liushaohui3@xiaomi.com> Co-authored-by: 王晨 <wangchen77@xiaomi.com> Co-authored-by: jiangzihan <jiangzihan@xiaomi.com> Co-authored-by: xiexueyang <xyxie_wangyi@163.com> Co-authored-by: Linghao Zhang <zhanglinghao@xiaomi.com> Co-authored-by: ispobock <ispobaoke@gmail.com> Co-authored-by: Liangsheng Yin <lsyincs@gmail.com> Co-authored-by: JoyFuture <35593546+JoyFuture@users.noreply.github.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com> Co-authored-by: Qiaolin Yu <liin1211@outlook.com> Co-authored-by: root <root@bj9-ml-g8h20e-k8s-slave106-20251106.alicn.idc.xiaomi.com>
This commit is contained in:
co-authored by
谢学扬
tz
李家乐
张晨
Shaohui Liu
王晨
jiangzihan
xiexueyang
Linghao Zhang
ispobock
Liangsheng Yin
JoyFuture
Liangsheng Yin
Qiaolin Yu
root
parent
a0985dd5e5
commit
160a06cab2
@@ -240,6 +240,13 @@ class DecodePreallocQueue:
|
||||
self.prefill_pp_size = prefill_pp_size
|
||||
self.kv_manager = self._init_kv_manager()
|
||||
|
||||
if self.scheduler.tp_worker.is_hybrid_swa:
|
||||
# FIXME: current SWA allocation allocate full kv cache size in prefill
|
||||
self.max_total_num_tokens = min(
|
||||
self.max_total_num_tokens,
|
||||
self.scheduler.tp_worker.model_runner.swa_max_total_num_tokens,
|
||||
)
|
||||
|
||||
def _init_kv_manager(self) -> BaseKVManager:
|
||||
kv_args_class = get_kv_class(self.transfer_backend, KVClassType.KVARGS)
|
||||
kv_args = kv_args_class()
|
||||
|
||||
@@ -132,13 +132,13 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
|
||||
# Simulate the eagle run.
|
||||
if self.spec_algorithm.is_eagle():
|
||||
|
||||
b = len(self.reqs)
|
||||
topk = server_args.speculative_eagle_topk
|
||||
num_states = server_args.speculative_eagle_topk
|
||||
if server_args.enable_mtp:
|
||||
num_states *= server_args.speculative_num_steps
|
||||
topk_p = torch.stack(
|
||||
[
|
||||
torch.as_tensor(
|
||||
req.output_topk_p[:topk],
|
||||
req.output_topk_p[:num_states],
|
||||
device=self.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
@@ -149,7 +149,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
topk_index = torch.stack(
|
||||
[
|
||||
torch.as_tensor(
|
||||
req.output_topk_index[:topk],
|
||||
req.output_topk_index[:num_states],
|
||||
device=self.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
|
||||
@@ -300,18 +300,24 @@ class MooncakeKVManager(CommonKVManager):
|
||||
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage = (
|
||||
self.get_mha_kv_ptrs_with_pp(src_data_ptrs, dst_data_ptrs)
|
||||
)
|
||||
# item_lens structure: [k_layer0, k_layer1, ..., k_layerN, v_layer0, v_layer1, ..., v_layerN]
|
||||
# Use correct item lengths for K and V separately
|
||||
if layers_current_pp_stage > len(dst_k_ptrs):
|
||||
logger.error(
|
||||
f"layers_current_pp_stage is out of range: {layers_current_pp_stage=}, {len(dst_k_ptrs)}"
|
||||
)
|
||||
layers_params = [
|
||||
(
|
||||
src_k_ptrs[layer_id],
|
||||
dst_k_ptrs[layer_id],
|
||||
item_lens[layer_id],
|
||||
item_lens[layer_id], # K item length
|
||||
)
|
||||
for layer_id in range(layers_current_pp_stage)
|
||||
] + [
|
||||
(
|
||||
src_v_ptrs[layer_id],
|
||||
dst_v_ptrs[layer_id],
|
||||
item_lens[layer_id],
|
||||
item_lens[layers_current_pp_stage + layer_id], # V item length
|
||||
)
|
||||
for layer_id in range(layers_current_pp_stage)
|
||||
]
|
||||
|
||||
@@ -109,6 +109,13 @@ class PrefillBootstrapQueue:
|
||||
self.transfer_backend = transfer_backend
|
||||
self.kv_manager = self._init_kv_manager()
|
||||
|
||||
if self.scheduler.tp_worker.is_hybrid_swa:
|
||||
# FIXME: current SWA allocation allocate full kv cache size in prefill
|
||||
self.max_total_num_tokens = min(
|
||||
self.max_total_num_tokens,
|
||||
self.scheduler.tp_worker.model_runner.swa_max_total_num_tokens,
|
||||
)
|
||||
|
||||
def _init_kv_manager(self) -> BaseKVManager:
|
||||
kv_args_class = get_kv_class(self.transfer_backend, KVClassType.KVARGS)
|
||||
kv_args = kv_args_class()
|
||||
|
||||
Reference in New Issue
Block a user