Added the ability to Modify the Context Length (#210)

2024-02-20 16:22:56 -08:00
parent ce3b261053
commit 9de9a46815
4 changed files with 16 additions and 2 deletions
@@ -546,6 +546,7 @@ class Runtime:
        trust_remote_code: bool = True,
        mem_fraction_static: float = ServerArgs.mem_fraction_static,
        max_prefill_num_token: int = ServerArgs.max_prefill_num_token,
+        context_length: int = ServerArgs.context_length,
        tp_size: int = 1,
        model_mode: List[str] = (),
        schedule_heuristic: str = "lpm",
@@ -567,6 +568,7 @@ class Runtime:
            trust_remote_code=trust_remote_code,
            mem_fraction_static=mem_fraction_static,
            max_prefill_num_token=max_prefill_num_token,
+            context_length=context_length,
            tp_size=tp_size,
            model_mode=model_mode,
            schedule_heuristic=schedule_heuristic,