[Bug fix] Add /model_info endpoint to mini_lb (#14535)

This commit is contained in:
Alison Shao
2025-12-06 11:34:03 -08:00
committed by GitHub
parent bc388471d2
commit cee93a6f26
@@ -341,8 +341,7 @@ async def get_server_info():
}
@app.get("/get_model_info")
async def get_model_info():
async def _get_model_info_impl():
if not lb or not lb.prefill_urls:
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
@@ -350,7 +349,7 @@ async def get_model_info():
)
target_server_url = lb.prefill_urls[0]
endpoint_url = f"{target_server_url}/get_model_info"
endpoint_url = f"{target_server_url}/model_info"
async with aiohttp.ClientSession() as session:
try:
@@ -375,6 +374,16 @@ async def get_model_info():
)
@app.get("/model_info")
async def model_info():
return await _get_model_info_impl()
@app.get("/get_model_info")
async def get_model_info():
return await _get_model_info_impl()
@app.post("/generate")
async def handle_generate_request(request_data: dict):
prefill_server, bootstrap_port, decode_server = lb.select_pair()