Fix /health regression from early prebound socket listen (#19805)

This commit is contained in:
Mohammad Miadh Angkad
2026-03-04 15:00:46 +08:00
committed by GitHub
parent 115f879958
commit 09fa012ba7
2 changed files with 12 additions and 9 deletions

View File

@@ -1978,6 +1978,7 @@ def launch_server(
# Reserve the HTTP port before launching subprocesses to fail fast if port is unavailable.
# This prevents wasting time loading models only to discover port conflicts later.
reserved_socket = _prebind_listening_socket(server_args.host, server_args.port)
multi_tokenizer_args_shm = None
try:
# Launch subprocesses
@@ -2052,6 +2053,10 @@ def launch_server(
# Update logging configs
set_uvicorn_logging_configs(server_args)
# Delay listen() until uvicorn startup to avoid accepting probe traffic
# while model/subprocess initialization is still in progress.
reserved_socket.listen(128)
# Listen for HTTP requests
if server_args.tokenizer_worker_num == 1:
# Default case, one tokenizer process
@@ -2089,5 +2094,7 @@ def launch_server(
reserved_socket.close()
if server_args.tokenizer_worker_num > 1:
multi_tokenizer_args_shm.unlink()
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()
if multi_tokenizer_args_shm is not None:
multi_tokenizer_args_shm.unlink()
if _global_state is not None:
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()

View File

@@ -416,7 +416,9 @@ def _prebind_listening_socket(host: str, port: int) -> socket.socket:
This prevents port conflicts that could occur if another process (e.g., a
subprocess spawned during model loading) binds to the same port before the
HTTP server starts. By binding early, we ensure the port is available and
reserved for the HTTP server.
reserved for the HTTP server. The socket is intentionally not put into
listening mode here to avoid accepting probe connections before uvicorn is
ready to serve requests.
Args:
host: The host address to bind to.
@@ -444,14 +446,8 @@ def _prebind_listening_socket(host: str, port: int) -> socket.socket:
# SO_REUSEADDR: Allows binding to a port in TIME_WAIT state
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# SO_REUSEPORT: Allows multiple processes to bind to the same port
# This is useful for multi-worker mode
if hasattr(socket, "SO_REUSEPORT"):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
try:
sock.bind((host or "", port))
sock.listen(128) # Put socket in listening state for uvicorn
sock.set_inheritable(True)
logger.info(
f"Successfully reserved port {port} on host '{host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}'"