Use dtype to control generate (#1082)

Co-authored-by: zhyncs <me@zhyncs.com>
2024-08-14 15:58:07 +00:00
parent 67c0d832a6
commit a34dd86a7d
12 changed files with 110 additions and 88 deletions
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.linear import (
 from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
 from vllm.model_executor.layers.rotary_embedding import get_rope
 from vllm.model_executor.layers.vocab_parallel_embedding import (
+    DEFAULT_VOCAB_PADDING_SIZE,
    ParallelLMHead,
    VocabParallelEmbedding,
 )