Fix kernel type annotations for fp8 quant and logging (#16994)
This commit is contained in:
@@ -174,5 +174,5 @@ class GuidanceBackend(BaseGrammarBackend):
|
||||
g = StructTag.to_grammar(tags)
|
||||
return self._from_serialized(g)
|
||||
except Exception as e:
|
||||
logging.error(f"Hit invalid structural_tag: {key_string=}, {e=}")
|
||||
logger.error(f"Hit invalid structural_tag: {key_string=}, {e=}")
|
||||
return INVALID_GRAMMAR_OBJ
|
||||
|
||||
@@ -230,7 +230,9 @@ class QuantizationConfig(ABC):
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def apply_sglang_mapper(self, hf_to_sglang_mapper: "WeightsMapper"): # noqa: B027
|
||||
def apply_weight_name_mapper(
|
||||
self, hf_to_sglang_mapper: "WeightsMapper"
|
||||
): # noqa: B027
|
||||
"""
|
||||
Interface for models to update module names referenced in
|
||||
quantization configs in order to reflect the sglang model structure
|
||||
|
||||
@@ -123,7 +123,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
def get_scaled_act_names(self) -> List[str]:
|
||||
return []
|
||||
|
||||
def apply_sglang_mapper(self, hf_to_sglang_mapper: "WeightsMapper"):
|
||||
def apply_weight_name_mapper(self, hf_to_sglang_mapper: "WeightsMapper"):
|
||||
self.target_scheme_map = hf_to_sglang_mapper.apply_dict(self.target_scheme_map)
|
||||
self.ignore = hf_to_sglang_mapper.apply_list(self.ignore)
|
||||
self.sparsity_scheme_map = hf_to_sglang_mapper.apply_dict(
|
||||
|
||||
@@ -261,7 +261,7 @@ def _initialize_model(
|
||||
hf_to_sglang_mapper = getattr(model_class, "hf_to_sglang_mapper", None)
|
||||
# pass mappings by reference to quant_config
|
||||
if hf_to_sglang_mapper is not None and quant_config is not None:
|
||||
quant_config.apply_sglang_mapper(hf_to_sglang_mapper)
|
||||
quant_config.apply_weight_name_mapper(hf_to_sglang_mapper)
|
||||
|
||||
# Build kwargs conditionally
|
||||
kwargs = {
|
||||
|
||||
Reference in New Issue
Block a user