[Bug fix] Add /model_info endpoint to mini_lb (#14535)
This commit is contained in:
@@ -341,8 +341,7 @@ async def get_server_info():
|
||||
}
|
||||
|
||||
|
||||
@app.get("/get_model_info")
|
||||
async def get_model_info():
|
||||
async def _get_model_info_impl():
|
||||
if not lb or not lb.prefill_urls:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
@@ -350,7 +349,7 @@ async def get_model_info():
|
||||
)
|
||||
|
||||
target_server_url = lb.prefill_urls[0]
|
||||
endpoint_url = f"{target_server_url}/get_model_info"
|
||||
endpoint_url = f"{target_server_url}/model_info"
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
@@ -375,6 +374,16 @@ async def get_model_info():
|
||||
)
|
||||
|
||||
|
||||
@app.get("/model_info")
|
||||
async def model_info():
|
||||
return await _get_model_info_impl()
|
||||
|
||||
|
||||
@app.get("/get_model_info")
|
||||
async def get_model_info():
|
||||
return await _get_model_info_impl()
|
||||
|
||||
|
||||
@app.post("/generate")
|
||||
async def handle_generate_request(request_data: dict):
|
||||
prefill_server, bootstrap_port, decode_server = lb.select_pair()
|
||||
|
||||
Reference in New Issue
Block a user