Upstreaming hicache bug fixes (#7267)

This commit is contained in:
Zhiqiang Xie
2025-06-17 17:44:57 -07:00
committed by GitHub
parent c26d7349d3
commit e56685ac1b
7 changed files with 76 additions and 24 deletions
+9
View File
@@ -147,6 +147,15 @@ class TpModelWorker:
# A reference make this class has the same member as TpModelWorkerClient
self.worker = self
self.hicache_layer_transfer_counter = None
def register_hicache_layer_transfer_counter(self, counter):
self.hicache_layer_transfer_counter = counter
def set_hicache_consumer(self, consumer_index):
if self.hicache_layer_transfer_counter is not None:
self.hicache_layer_transfer_counter.set_consumer(consumer_index)
def get_worker_info(self):
return (
self.max_total_num_tokens,