Add Mistral Small 4 (Pixtral) support (#20708)
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com> Co-authored-by: Alex Nails <alexnails@radixark.ai> Co-authored-by: Dimitrios Bariamis <12195802+dbari@users.noreply.github.com> Co-authored-by: dbari <dbari@users.noreply.github.com>
This commit is contained in:
co-authored by
Alex Nails
Dimitrios Bariamis
dbari
parent
df1d046de2
commit
6b8a6545b2
@@ -25,18 +25,18 @@ class JetBlockConfig:
|
||||
class JetNemotronConfig(PretrainedConfig):
|
||||
model_type: str = "jet_nemotron"
|
||||
|
||||
efficient_attention_config: dict[str, dict[str, Any]]
|
||||
hidden_act: str
|
||||
hidden_size: int
|
||||
initializer_range: float
|
||||
intermediate_size: int
|
||||
layer_types: list[str]
|
||||
max_position_embeddings: int
|
||||
num_attention_heads: int
|
||||
num_key_value_heads: int
|
||||
rms_norm_eps: float
|
||||
rope_scaling: None
|
||||
rope_theta: float
|
||||
efficient_attention_config: dict[str, dict[str, Any]] = None
|
||||
hidden_act: str = None
|
||||
hidden_size: int = None
|
||||
initializer_range: float = None
|
||||
intermediate_size: int = None
|
||||
layer_types: list[str] = None
|
||||
max_position_embeddings: int = None
|
||||
num_attention_heads: int = None
|
||||
num_key_value_heads: int = None
|
||||
rms_norm_eps: float = None
|
||||
rope_scaling: None = None
|
||||
rope_theta: float = None
|
||||
|
||||
@property
|
||||
def full_attention_layer_ids(self) -> list[int]:
|
||||
|
||||
Reference in New Issue
Block a user