SWA Prefix Cache (#7367)
Co-authored-by: Ying Sheng <sqy1415@gmail.com>
This commit is contained in:
@@ -174,6 +174,20 @@ class TpModelWorker:
|
||||
self.model_runner.token_to_kv_pool.size,
|
||||
)
|
||||
|
||||
@property
|
||||
def sliding_window_size(self) -> Optional[int]:
|
||||
return self.model_runner.sliding_window_size
|
||||
|
||||
@property
|
||||
def is_hybrid(self) -> bool:
|
||||
return self.model_runner.is_hybrid is not None
|
||||
|
||||
def get_tokens_per_layer_info(self):
|
||||
return (
|
||||
self.model_runner.full_max_total_num_tokens,
|
||||
self.model_runner.swa_max_total_num_tokens,
|
||||
)
|
||||
|
||||
def get_pad_input_ids_func(self):
|
||||
return getattr(self.model_runner.model, "pad_input_ids", None)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user