[Fix] Fix logprob and normalized_logprob (#1428)

2024-09-15 06:36:06 -07:00
parent 282681b8a1
commit 9ba1f09760
22 changed files with 314 additions and 215 deletions
@@ -239,9 +239,12 @@ class RuntimeEndpoint(BaseBackend):
        # Compute logprob
        data = {
            "text": [s.text_ + c for c in choices],
-            "sampling_params": {"max_new_tokens": 0},
+            "sampling_params": {
+                "max_new_tokens": 0,
+                "temperature": 0,
+            },
            "return_logprob": True,
-            "logprob_start_len": max(prompt_len - 2, 0),
+            "logprob_start_len": max(prompt_len - 2, 0),  # for token healing
        }
        obj = self._generate_http_request(s, data)