Re-introduce get_cuda_graph_seq_len_fill_value (#1783)

2024-10-24 13:30:11 -07:00
parent 605972195b
commit 384d85ba35
5 changed files with 19 additions and 2 deletions
@@ -210,6 +210,9 @@ class FlashInferAttnBackend(AttentionBackend):
            encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
        )

+    def get_cuda_graph_seq_len_fill_value(self):
+        return 0
+
    def forward_extend(
        self, q, k, v, layer: RadixAttention, forward_batch: ForwardBatch
    ):