[PD] use int32 for kv indices & get num_reserved_decode_tokens from server_args (#7214)

This commit is contained in:
Byron Hsu
2025-06-15 11:51:03 -07:00
committed by GitHub
parent fff10809bf
commit 88f9c347b2
8 changed files with 24 additions and 26 deletions
@@ -70,7 +70,7 @@ class BaseKVSender(ABC):
...
@abstractmethod
def send(self, kv_indices: npt.NDArray[np.int64]):
def send(self, kv_indices: npt.NDArray[np.int32]):
"""
Send the kv cache at the given kv indices to the decoder server
"""
@@ -102,7 +102,7 @@ class BaseKVReceiver(ABC):
): ...
@abstractmethod
def init(self, kv_indices: npt.NDArray[np.int64], aux_index: Optional[int] = None):
def init(self, kv_indices: npt.NDArray[np.int32], aux_index: Optional[int] = None):
"""
Notify the prefill server about the kv indices and aux index
"""