[Feature] Support minicpmv v2.6 (#2785)

Co-authored-by: Chayenne <zhaochen20@outlook.com>
Co-authored-by: yizhang2077 <1109276519@qq.com>
This commit is contained in:
Mick
2025-01-19 06:14:19 +08:00
committed by GitHub
parent c2f212d672
commit 3d93f84a00
20 changed files with 1715 additions and 139 deletions

View File

@@ -56,6 +56,7 @@ class DataParallelController:
def __init__(self, server_args, port_args) -> None:
# Parse args
self.max_total_num_tokens = None
self.server_args = server_args
self.port_args = port_args
self.load_balance_method = LoadBalanceMethod.from_str(
@@ -96,6 +97,8 @@ class DataParallelController:
True,
)
self.max_req_input_len = None
def launch_dp_schedulers(self, server_args, port_args):
base_gpu_id = 0
@@ -189,6 +192,7 @@ class DataParallelController:
scheduler_info.append(scheduler_pipe_readers[i].recv())
self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"]
self.max_req_input_len = scheduler_info[0]["max_req_input_len"]
def round_robin_scheduler(self, req):
self.workers[self.round_robin_counter].send_pyobj(req)
@@ -231,7 +235,11 @@ def run_data_parallel_controller_process(
try:
controller = DataParallelController(server_args, port_args)
pipe_writer.send(
{"status": "ready", "max_total_num_tokens": controller.max_total_num_tokens}
{
"status": "ready",
"max_total_num_tokens": controller.max_total_num_tokens,
"max_req_input_len": controller.max_req_input_len,
}
)
if server_args.node_rank == 0:
controller.event_loop()