Add support for Matryoshka embeddings (#126) (#11142)

Co-authored-by: Satyam Kumar <satyamk@linkedin.com>
2025-10-28 02:49:36 +08:00
parent c11b34d599
commit 9fc3e8aac7
15 changed files with 314 additions and 12 deletions
@@ -205,6 +205,14 @@ class ModelConfig:
            self.hf_config, "image_token_id", None
        ) or getattr(self.hf_config, "image_token_index", None)

+        # matryoshka embeddings
+        self.matryoshka_dimensions = getattr(
+            self.hf_config, "matryoshka_dimensions", None
+        )
+        self.is_matryoshka = self.matryoshka_dimensions or getattr(
+            self.hf_config, "is_matryoshka", False
+        )
+
    @staticmethod
    def from_server_args(
        server_args: ServerArgs,