Beta spec-overlap for EAGLE (#11398)
Co-authored-by: Lianmin Zheng <15100009+merrymercy@users.noreply.github.com> Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
This commit is contained in:
co-authored by
Lianmin Zheng
Hanming Lu
parent
47c606d3dc
commit
20a6c0a63d
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
from copy import copy
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import ClassVar, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -10,6 +10,7 @@ from sglang.srt.constrained.base_grammar_backend import BaseGrammarObject
|
||||
from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.sampler import apply_custom_logit_processor
|
||||
from sglang.srt.managers.overlap_utils import FutureIndices
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch, global_server_args_dict
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.common import (
|
||||
@@ -18,16 +19,20 @@ from sglang.srt.mem_cache.common import (
|
||||
get_last_loc,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
|
||||
from sglang.srt.speculative.eagle_info_v2 import (
|
||||
EagleDraftInputV2Mixin,
|
||||
EagleVerifyInputV2Mixin,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
from sglang.srt.speculative.spec_utils import (
|
||||
SIMULATE_ACC_LEN,
|
||||
TREE_SPEC_KERNEL_AVAILABLE,
|
||||
_generate_simulated_accept_index,
|
||||
align_evict_mask_to_page_size,
|
||||
assign_req_to_token_pool,
|
||||
create_accept_length_filter,
|
||||
create_extend_after_decode_spec_info,
|
||||
filter_finished_cache_loc_kernel,
|
||||
generate_simulated_accept_index,
|
||||
get_src_tgt_cache_loc,
|
||||
get_target_cache_loc,
|
||||
)
|
||||
@@ -47,7 +52,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EagleVerifyInput(SpecInput):
|
||||
class EagleVerifyInput(SpecInput, EagleVerifyInputV2Mixin):
|
||||
draft_token: torch.Tensor
|
||||
custom_mask: torch.Tensor
|
||||
positions: torch.Tensor
|
||||
@@ -338,7 +343,7 @@ class EagleVerifyInput(SpecInput):
|
||||
|
||||
if SIMULATE_ACC_LEN > 0.0:
|
||||
# Do simulation
|
||||
accept_index = _generate_simulated_accept_index(
|
||||
accept_index = generate_simulated_accept_index(
|
||||
accept_index=accept_index,
|
||||
predict=predict, # mutable
|
||||
accept_length=accept_length, # mutable
|
||||
@@ -568,7 +573,7 @@ class EagleVerifyInput(SpecInput):
|
||||
|
||||
|
||||
@dataclass
|
||||
class EagleDraftInput(SpecInput):
|
||||
class EagleDraftInput(SpecInput, EagleDraftInputV2Mixin):
|
||||
# The inputs for decode
|
||||
# shape: (b, topk)
|
||||
topk_p: torch.Tensor = None
|
||||
@@ -598,6 +603,15 @@ class EagleDraftInput(SpecInput):
|
||||
seq_lens_for_draft_extend_cpu: torch.Tensor = None
|
||||
req_pool_indices_for_draft_extend: torch.Tensor = None
|
||||
|
||||
# Inputs for V2 overlap worker
|
||||
future_indices: Optional[FutureIndices] = None
|
||||
allocate_lens: Optional[torch.Tensor] = None
|
||||
new_seq_lens: Optional[torch.Tensor] = None
|
||||
verify_done: Optional[torch.cuda.Event] = None
|
||||
|
||||
# FIXME(lsyin): remove this hack
|
||||
ALLOC_LEN_PER_DECODE: ClassVar[int] = None
|
||||
|
||||
def __post_init__(self):
|
||||
super().__init__(SpecInputType.EAGLE_DRAFT)
|
||||
|
||||
@@ -703,6 +717,11 @@ class EagleDraftInput(SpecInput):
|
||||
return kv_indices, cum_kv_seq_len, qo_indptr, None
|
||||
|
||||
def filter_batch(self, new_indices: torch.Tensor, has_been_filtered: bool = True):
|
||||
if self.future_indices is not None:
|
||||
self.future_indices.indices = self.future_indices.indices[new_indices]
|
||||
self.allocate_lens = self.allocate_lens[new_indices]
|
||||
return
|
||||
|
||||
if has_been_filtered:
|
||||
# in eagle_utils.py:verify, we have already filtered the batch by `unfinished_index`
|
||||
# therefore, we don't need to filter the batch again in scheduler
|
||||
@@ -722,6 +741,18 @@ class EagleDraftInput(SpecInput):
|
||||
self.verified_id = self.verified_id[new_indices]
|
||||
|
||||
def merge_batch(self, spec_info: "EagleDraftInput"):
|
||||
if self.future_indices is not None:
|
||||
assert spec_info.future_indices is not None
|
||||
self.future_indices = FutureIndices(
|
||||
indices=torch.cat(
|
||||
[self.future_indices.indices, spec_info.future_indices.indices]
|
||||
)
|
||||
)
|
||||
self.allocate_lens = torch.cat(
|
||||
[self.allocate_lens, spec_info.allocate_lens]
|
||||
)
|
||||
return
|
||||
|
||||
if self.hidden_states is None:
|
||||
self.hidden_states = spec_info.hidden_states
|
||||
self.verified_id = spec_info.verified_id
|
||||
|
||||
Reference in New Issue
Block a user