chenchenghao/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

Remove deprecated configs (#1431)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-09-15 08:52:18 -07:00
committed by GitHub
parent e79f6cd73d
commit 899cf5c438
5 changed files with 13 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/layers/attention_backend.py
View File

@@ -150,7 +150,7 @@ class FlashInferAttnBackend(AttentionBackend):
# Some heuristics to check whether to use ragged forward
use_ragged = False
if (
int(torch.sum(input_metadata.seq_lens)) > 4096
torch.sum(input_metadata.seq_lens).item() >= 4096
and self.model_runner.sliding_window_size is None
):
use_ragged = True
@@ -301,10 +301,6 @@ class FlashInferAttnBackend(AttentionBackend):
layer.layer_id, input_metadata.out_cache_loc, k, v
)
if total_num_tokens >= global_config.layer_sync_threshold:
# TODO: Revisit this. Why is this synchronize needed?
torch.cuda.synchronize()
return o.view(-1, layer.tp_q_head_num * layer.head_dim)
def forward_decode(self, q, k, v, layer: nn.Module, input_metadata: InputMetadata):