Enable optional FP32 compute for LM Head (#10729)
Thanks to MiniMax Team and Chenyang Zhao's support.
This commit is contained in:
@@ -167,6 +167,7 @@ class ServerArgs:
|
||||
quantization: Optional[str] = None
|
||||
quantization_param_path: Optional[str] = None
|
||||
kv_cache_dtype: str = "auto"
|
||||
enable_fp32_lm_head: bool = False
|
||||
|
||||
# Memory and scheduling
|
||||
mem_fraction_static: Optional[float] = None
|
||||
@@ -1392,6 +1393,11 @@ class ServerArgs:
|
||||
choices=["auto", "fp8_e5m2", "fp8_e4m3"],
|
||||
help='Data type for kv cache storage. "auto" will use model data type. "fp8_e5m2" and "fp8_e4m3" is supported for CUDA 11.8+.',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-fp32-lm-head",
|
||||
action="store_true",
|
||||
help="If set, the LM head outputs (logits) are in FP32.",
|
||||
)
|
||||
|
||||
# Memory and scheduling
|
||||
parser.add_argument(
|
||||
|
||||
Reference in New Issue
Block a user