feat(token_limit): limit input token < 131072
4.max_tokens=131072,input<131072时不报错 5. 输入长度超过131072时,报错,且报错内容含有以下字符串,以便被网关识别 "is longer than the model's context length"
This commit is contained in:
@@ -797,7 +797,7 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
|
||||
|
||||
# Validate input length
|
||||
if input_token_num >= self.context_len:
|
||||
if self.server_args.allow_auto_truncate:
|
||||
if False:
|
||||
logger.warning(
|
||||
f"The input ({input_token_num} tokens) is longer than the "
|
||||
f"model's context length ({self.context_len} tokens). "
|
||||
@@ -818,7 +818,7 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
|
||||
and max_new_tokens is not None
|
||||
and (max_new_tokens + input_token_num) >= _max_req_len
|
||||
):
|
||||
if self.server_args.allow_auto_truncate:
|
||||
if True:
|
||||
logger.warning(
|
||||
f"Requested token count ({input_token_num} input + {max_new_tokens} new) "
|
||||
f"exceeds the model's context length ({self.context_len} tokens). "
|
||||
|
||||
Reference in New Issue
Block a user