Simplify sampler and its error handling (#1441)

2024-09-16 21:23:31 -07:00
parent 27b557aea7
commit 2fa5cec775
4 changed files with 32 additions and 159 deletions
--- a/python/sglang/srt/managers/schedule_batch.py
+++ b/python/sglang/srt/managers/schedule_batch.py
@@ -360,6 +360,7 @@ class ScheduleBatch:
    tree_cache: BasePrefixCache

    forward_mode: ForwardMode = None
+    sampling_info: SamplingBatchInfo = None

    # Batched arguments to model runner
    input_ids: torch.Tensor = None