support speculative execution for openai API (#48)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>

python/sglang/srt/managers/detokenizer_manager.py

@@ -60,7 +60,9 @@ class DetokenizerManager:
                         if first_token.startswith("▁"):
                             output_strs[i] = " " + output_strs[i]
 
-                    output_strs[i] = recv_obj.output_and_fast_forward_strs[i] + output_strs[i]
+                    output_strs[i] = (
+                        recv_obj.output_and_fast_forward_strs[i] + output_strs[i]
+                    )
 
                 self.send_to_tokenizer.send_pyobj(
                     BatchStrOut(

python/sglang/srt/managers/router/model_rpc.py

@@ -12,6 +12,7 @@ import rpyc
 import torch
 from rpyc.utils.classic import obtain
 from rpyc.utils.server import ThreadedServer
+from sglang.srt.constrained.fast_forward import FastForwardCache
 from sglang.srt.constrained.fsm_cache import FSMCache
 from sglang.srt.hf_transformers_utils import get_processor, get_tokenizer
 from sglang.srt.managers.io_struct import BatchTokenIDOut, TokenizedGenerateReqInput
@@ -21,7 +22,6 @@ from sglang.srt.managers.router.radix_cache import RadixCache
 from sglang.srt.managers.router.scheduler import Scheduler
 from sglang.srt.model_config import ModelConfig
 from sglang.srt.server_args import PortArgs, ServerArgs
-from sglang.srt.constrained.fast_forward import FastForwardCache
 from sglang.srt.utils import (
     get_exception_traceback,
     get_int_token_logit_bias,

python/sglang/srt/managers/tokenizer_manager.py

@@ -200,6 +200,7 @@ class TokenizerManager:
                     )
                 tokenized_obj = TokenizedGenerateReqInput(
                     rid=rid,
+                    input_text=obj.text[i],
                     input_ids=input_ids,
                     pixel_values=pixel_values,
                     image_hash=image_hash,