[Feature] Spec-Overlap supporting DP-ATTN; PD-Disaggregation; npugraph mode (#12443)
This commit is contained in:
@@ -930,7 +930,7 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
|
||||
# construct fake completed prefill
|
||||
new_batch.prepare_for_prebuilt()
|
||||
new_batch.process_prebuilt(self.server_args, self.model_config)
|
||||
new_batch.process_prebuilt(self.server_args, self.future_map)
|
||||
|
||||
return new_batch
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.managers.overlap_utils import FutureMap
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
@@ -102,7 +102,9 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
)
|
||||
|
||||
def process_prebuilt(
|
||||
self: ScheduleBatch, server_args: ServerArgs, model_config: ModelConfig
|
||||
self: ScheduleBatch,
|
||||
server_args: ServerArgs,
|
||||
future_map: FutureMap,
|
||||
):
|
||||
"""Assign the buffered last input id to schedule batch"""
|
||||
self.output_ids = []
|
||||
@@ -166,7 +168,16 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
topk_index=topk_index,
|
||||
hidden_states=hidden_states,
|
||||
verified_id=self.output_ids,
|
||||
new_seq_lens=self.seq_lens,
|
||||
allocate_lens=self.seq_lens,
|
||||
)
|
||||
spec_info.prepare_for_extend(self)
|
||||
spec_info.capture_hidden_mode = CaptureHiddenMode.LAST
|
||||
if self.enable_overlap:
|
||||
spec_info.future_indices = future_map.alloc_future_indices(
|
||||
len(self.seq_lens)
|
||||
)
|
||||
future_map.store_to_map_for_new_batch(
|
||||
spec_info.future_indices, spec_info
|
||||
)
|
||||
self.spec_info = spec_info
|
||||
|
||||
Reference in New Issue
Block a user