Fix /generate JSON serialization for non-finite top_logprobs (#20714)

python/sglang/srt/entrypoints/http_server.py

@@ -42,7 +42,6 @@ setattr(threading, "_register_atexit", lambda *args, **kwargs: None)
 
 
 import numpy as np
-import orjson
 import requests
 import uvicorn
 import uvloop
@@ -173,6 +172,11 @@ from sglang.srt.utils import (
     set_uvicorn_logging_configs,
 )
 from sglang.srt.utils.auth import AuthLevel, app_has_admin_force_endpoints, auth_level
+from sglang.srt.utils.json_response import (
+    SGLangORJSONResponse,
+    dumps_json,
+    orjson_response,
+)
 from sglang.utils import get_exception_traceback
 from sglang.version import __version__
 
@@ -666,7 +670,11 @@ if os.environ.get("DUMPER_SERVER_PORT") == "reuse":
 
 
 # fastapi implicitly converts json in the request to obj (dataclass)
-@app.api_route("/generate", methods=["POST", "PUT"])
+@app.api_route(
+    "/generate",
+    methods=["POST", "PUT"],
+    response_class=SGLangORJSONResponse,
+)
 async def generate_request(obj: GenerateReqInput, request: Request):
     """Handle a generate request."""
     if obj.stream:
@@ -676,15 +684,11 @@ async def generate_request(obj: GenerateReqInput, request: Request):
                 async for out in _global_state.tokenizer_manager.generate_request(
                     obj, request
                 ):
-                    yield b"data: " + orjson.dumps(
-                        out, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY
-                    ) + b"\n\n"
+                    yield b"data: " + dumps_json(out) + b"\n\n"
             except ValueError as e:
                 out = {"error": {"message": str(e)}}
                 logger.error(f"[http_server] Error: {e}")
-                yield b"data: " + orjson.dumps(
-                    out, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY
-                ) + b"\n\n"
+                yield b"data: " + dumps_json(out) + b"\n\n"
             yield b"data: [DONE]\n\n"
 
         return StreamingResponse(
@@ -697,12 +701,7 @@ async def generate_request(obj: GenerateReqInput, request: Request):
             ret = await _global_state.tokenizer_manager.generate_request(
                 obj, request
             ).__anext__()
-            return Response(
-                content=orjson.dumps(
-                    ret, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY
-                ),
-                media_type="application/json",
-            )
+            return orjson_response(ret)
         except ValueError as e:
             logger.error(f"[http_server] Error: {e}")
             return _create_error_response(e)

python/sglang/srt/utils/json_response.py

@@ -0,0 +1,32 @@
+"""Utilities for JSON serialization in HTTP responses."""
+
+from typing import Any
+
+import orjson
+from fastapi.responses import ORJSONResponse, Response
+
+# Keep response serialization behavior consistent across endpoints:
+# - Support non-string dictionary keys used in some metadata payloads.
+# - Support numpy scalars/arrays without pre-conversion.
+ORJSON_RESPONSE_OPTIONS = orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY
+
+
+def dumps_json(content: Any) -> bytes:
+    """Serialize content to JSON bytes using SGLang's ORJSON options."""
+    return orjson.dumps(content, option=ORJSON_RESPONSE_OPTIONS)
+
+
+class SGLangORJSONResponse(ORJSONResponse):
+    """ORJSON response with SGLang-specific serialization options."""
+
+    def render(self, content: Any) -> bytes:
+        return dumps_json(content)
+
+
+def orjson_response(content: Any, status_code: int = 200) -> Response:
+    """Create a JSON response with stable ORJSON serialization options."""
+    return Response(
+        content=dumps_json(content),
+        media_type="application/json",
+        status_code=status_code,
+    )