[PD-HiCache]: Support Async Offloading KVCache In Decode Side (#10192)

Signed-off-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
2025-09-25 23:20:49 -07:00
parent 6088548216
commit 7ec5b4e89c
7 changed files with 523 additions and 8 deletions
@@ -250,7 +250,13 @@ class SchedulerOutputProcessorMixin:

            req.check_finished()
            if req.finished():
-                self.tree_cache.cache_finished_req(req)
+                if self.server_args.disaggregation_decode_enable_offload_kvcache:
+                    # Asynchronously offload KV cache; cache_finished_req will be called after Device->Host transfer completes
+                    if not self.decode_offload_manager.offload_kv_cache(req):
+                        self.tree_cache.cache_finished_req(req)
+                else:
+                    self.tree_cache.cache_finished_req(req)
+
                req.time_stats.completion_time = time.time()

            if req.return_logprob and batch.spec_algorithm.is_none():