Clean up batch data structures: Introducing ModelWorkerBatch (#1544)

2024-09-30 06:41:49 -07:00
parent 36d5acfca5
commit 63ba2f8d7b
9 changed files with 274 additions and 155 deletions
--- a/python/sglang/srt/layers/logits_processor.py
+++ b/python/sglang/srt/layers/logits_processor.py
@@ -62,7 +62,11 @@ class LogitsMetadata:

    @classmethod
    def from_forward_batch(cls, forward_batch: ForwardBatch):
-        return_top_logprob = any(x > 0 for x in forward_batch.top_logprobs_nums)
+        if forward_batch.return_logprob:
+            return_top_logprob = any(x > 0 for x in forward_batch.top_logprobs_nums)
+        else:
+            return_top_logprob = False
+
        if forward_batch.forward_mode.is_extend():
            extend_logprob_pruned_lens_cpu = [
                extend_len - start_len