Added the ability to Modify the Context Length (#210)

2024-02-20 16:22:56 -08:00
parent ce3b261053
commit 9de9a46815
4 changed files with 16 additions and 2 deletions
@@ -57,7 +57,7 @@ class ModelRpcServer(rpyc.Service):

        # Init model and tokenizer
        self.model_config = ModelConfig(
-            server_args.model_path, server_args.trust_remote_code
+            server_args.model_path, server_args.trust_remote_code, context_length=server_args.context_length
        )
        self.model_runner = ModelRunner(
            self.model_config,