Improve docs & Rename Gemini -> VertexAI (#19)

2024-01-17 02:54:41 -08:00
parent fd7c479239
commit bf51ddc6e5
13 changed files with 56 additions and 583 deletions
@@ -108,7 +108,7 @@ class ModelRpcServer(rpyc.Service):
        self.running_batch: Batch = None
        self.out_pyobjs = []
        self.decode_forward_ct = 0
-        self.stream_interval = 2
+        self.stream_interval = server_args.stream_interval

        # Init the FSM cache for constrained generation
        self.regex_fsm_cache = FSMCache(self.tokenizer)