Clean up server_args.py (#7037)
This commit is contained in:
@@ -41,6 +41,9 @@ class EAGLEDraftCudaGraphRunner:
|
||||
self.tp_size = self.model_runner.tp_size
|
||||
self.topk = model_runner.server_args.speculative_eagle_topk
|
||||
self.speculative_num_steps = model_runner.server_args.speculative_num_steps
|
||||
self.enable_profile_cuda_graph = (
|
||||
model_runner.server_args.enable_profile_cuda_graph
|
||||
)
|
||||
server_args = model_runner.server_args
|
||||
|
||||
# Batch sizes to capture
|
||||
|
||||
@@ -39,6 +39,9 @@ class EAGLEDraftExtendCudaGraphRunner:
|
||||
self.dp_size = model_runner.server_args.dp_size
|
||||
self.speculative_num_steps = model_runner.server_args.speculative_num_steps
|
||||
self.topk = model_runner.server_args.speculative_eagle_topk
|
||||
self.enable_profile_cuda_graph = (
|
||||
model_runner.server_args.enable_profile_cuda_graph
|
||||
)
|
||||
self.capture_bs, self.compile_bs = get_batch_sizes_to_capture(model_runner)
|
||||
self.padded_static_len = -1
|
||||
|
||||
|
||||
Reference in New Issue
Block a user