Clean up model loader (#1440)

This commit is contained in:
Lianmin Zheng
2024-09-16 18:16:27 -07:00
committed by GitHub
parent 93dffd699b
commit 27b557aea7
5 changed files with 33 additions and 80 deletions

View File

@@ -415,7 +415,7 @@ class ModelTpServer:
# Truncate prompts that are too long
if len(req.origin_input_ids) >= self.max_req_input_len:
logger.warn(
logger.warning(
"Request length is longer than the KV cache pool size or "
"the max context length. Truncated!!!"
)
@@ -936,6 +936,8 @@ class ModelTpServer:
if success:
flash_cache_success = self.flush_cache()
assert flash_cache_success, "Cache flush failed after updating weights"
else:
logger.error(message)
return success, message