Re-introduce get_cuda_graph_seq_len_fill_value (#1783)

2024-10-24 13:30:11 -07:00
parent 605972195b
commit 384d85ba35
5 changed files with 19 additions and 2 deletions
@@ -161,6 +161,9 @@ class DoubleSparseAttnBackend(AttentionBackend):
        self.cuda_graph_start_loc.zero_()
        self.cuda_graph_start_loc[1:bs] = torch.cumsum(seq_lens[: bs - 1], dim=0)

+    def get_cuda_graph_seq_len_fill_value(self):
+        return 1
+
    def forward_extend(
        self, q, k, v, layer: RadixAttention, forward_batch: ForwardBatch
    ):