Fix incorrect KV indices creation when page_size=32 in TRTLLM MLA backend (#11985)
This commit is contained in:
@@ -9,19 +9,12 @@ and uses BatchMLAPaged wrapper for decoding.
|
||||
More details can be found in https://docs.flashinfer.ai/api/mla.html
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
if os.environ["SGLANG_ENABLE_TORCH_COMPILE"] == "1":
|
||||
import logging
|
||||
|
||||
torch._logging.set_logs(dynamo=logging.ERROR)
|
||||
torch._dynamo.config.suppress_errors = True
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
from sglang.srt.layers.attention.flashinfer_backend import (
|
||||
@@ -45,6 +38,12 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.speculative.spec_info import SpecInput
|
||||
|
||||
if envs.SGLANG_ENABLE_TORCH_COMPILE.get():
|
||||
import logging
|
||||
|
||||
torch._logging.set_logs(dynamo=logging.ERROR)
|
||||
torch._dynamo.config.suppress_errors = True
|
||||
|
||||
if is_flashinfer_available():
|
||||
from flashinfer import (
|
||||
BatchMLAPagedAttentionWrapper,
|
||||
|
||||
Reference in New Issue
Block a user