Fix /health regression from early prebound socket listen (#19805)
This commit is contained in:
committed by
GitHub
parent
115f879958
commit
09fa012ba7
@@ -1978,6 +1978,7 @@ def launch_server(
|
||||
# Reserve the HTTP port before launching subprocesses to fail fast if port is unavailable.
|
||||
# This prevents wasting time loading models only to discover port conflicts later.
|
||||
reserved_socket = _prebind_listening_socket(server_args.host, server_args.port)
|
||||
multi_tokenizer_args_shm = None
|
||||
|
||||
try:
|
||||
# Launch subprocesses
|
||||
@@ -2052,6 +2053,10 @@ def launch_server(
|
||||
# Update logging configs
|
||||
set_uvicorn_logging_configs(server_args)
|
||||
|
||||
# Delay listen() until uvicorn startup to avoid accepting probe traffic
|
||||
# while model/subprocess initialization is still in progress.
|
||||
reserved_socket.listen(128)
|
||||
|
||||
# Listen for HTTP requests
|
||||
if server_args.tokenizer_worker_num == 1:
|
||||
# Default case, one tokenizer process
|
||||
@@ -2089,5 +2094,7 @@ def launch_server(
|
||||
reserved_socket.close()
|
||||
|
||||
if server_args.tokenizer_worker_num > 1:
|
||||
multi_tokenizer_args_shm.unlink()
|
||||
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()
|
||||
if multi_tokenizer_args_shm is not None:
|
||||
multi_tokenizer_args_shm.unlink()
|
||||
if _global_state is not None:
|
||||
_global_state.tokenizer_manager.socket_mapping.clear_all_sockets()
|
||||
|
||||
@@ -416,7 +416,9 @@ def _prebind_listening_socket(host: str, port: int) -> socket.socket:
|
||||
This prevents port conflicts that could occur if another process (e.g., a
|
||||
subprocess spawned during model loading) binds to the same port before the
|
||||
HTTP server starts. By binding early, we ensure the port is available and
|
||||
reserved for the HTTP server.
|
||||
reserved for the HTTP server. The socket is intentionally not put into
|
||||
listening mode here to avoid accepting probe connections before uvicorn is
|
||||
ready to serve requests.
|
||||
|
||||
Args:
|
||||
host: The host address to bind to.
|
||||
@@ -444,14 +446,8 @@ def _prebind_listening_socket(host: str, port: int) -> socket.socket:
|
||||
# SO_REUSEADDR: Allows binding to a port in TIME_WAIT state
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
|
||||
# SO_REUSEPORT: Allows multiple processes to bind to the same port
|
||||
# This is useful for multi-worker mode
|
||||
if hasattr(socket, "SO_REUSEPORT"):
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
|
||||
|
||||
try:
|
||||
sock.bind((host or "", port))
|
||||
sock.listen(128) # Put socket in listening state for uvicorn
|
||||
sock.set_inheritable(True)
|
||||
logger.info(
|
||||
f"Successfully reserved port {port} on host '{host or ('::' if family == socket.AF_INET6 else '0.0.0.0')}'"
|
||||
|
||||
Reference in New Issue
Block a user