Add SarvamMLA model (sarvamai/sarvam-105b)

docs/source/en/_toctree.yml

@@ -778,6 +778,8 @@
         title: RoFormer
       - local: model_doc/rwkv
         title: RWKV
+      - local: model_doc/sarvam_mla
+        title: SarvamMLA
       - local: model_doc/seed_oss
         title: Seed-Oss
       - local: model_doc/solar_open

docs/source/en/model_doc/sarvam_mla.md

@@ -0,0 +1,48 @@
+<!--Copyright 2026 Sarvam AI and The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
+compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is
+distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+
+-->
+*This model was released on 2026-03-06 and added to Hugging Face Transformers on 2026-03-17.*
+
+# SarvamMLA
+
+## Overview
+
+SarvamMLA is a 105B parameter Mixture of Experts (MoE) language model developed by [Sarvam AI](https://www.sarvam.ai/). It uses Multi-head Latent Attention (MLA) combined with sparse MoE routing, architecturally identical to DeepSeek-V3.
+
+Key architectural features:
+
+- **Multi-head Latent Attention (MLA)**: Low-rank KV compression with decoupled RoPE, reducing KV cache memory while maintaining performance.
+- **Sparse Mixture of Experts**: 128 routed experts with 8 active per token, plus 1 shared expert. The first layer uses a dense MLP.
+- **DeepSeek YaRN RoPE**: Extended context support up to 131K tokens via YaRN rotary position embeddings.
+- **Sigmoid routing with group-based top-k**: Token-choice routing using sigmoid scores with expert bias correction and group-aware selection.
+
+This model uses the DeepSeek-V3 architecture with a custom configuration. See the [DeepSeek-V3 documentation](deepseek_v3) for model and forward reference.
+
+## Usage
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model = AutoModelForCausalLM.from_pretrained(
+    "sarvamai/sarvam-105b",
+    device_map="auto",
+)
+tokenizer = AutoTokenizer.from_pretrained("sarvamai/sarvam-105b")
+
+inputs = tokenizer("Hello, how are you?", return_tensors="pt").to(model.device)
+outputs = model.generate(**inputs, max_new_tokens=50)
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+```
+
+## SarvamMLAConfig
+
+[[autodoc]] SarvamMLAConfig

src/transformers/conversion_mapping.py

@@ -59,6 +59,7 @@
     "hunyuan_v1_moe": "qwen2_moe",
     "flex_olmo": "qwen2_moe",
     "olmoe": "qwen2_moe",
+    "sarvam_mla": "qwen2_moe",
     "exaone_moe": "qwen2_moe",
     "rt_detr_v2": "rt_detr",
     "pp_doclayout_v2": "rt_detr",

src/transformers/models/__init__.py

@@ -365,6 +365,7 @@
     from .sam3_tracker_video import *
     from .sam3_video import *
     from .sam_hq import *
+    from .sarvam_mla import *
     from .seamless_m4t import *
     from .seamless_m4t_v2 import *
     from .seed_oss import *

src/transformers/models/auto/configuration_auto.py

@@ -417,6 +417,7 @@
         ("sam_hq", "SamHQConfig"),
         ("sam_hq_vision_model", "SamHQVisionConfig"),
         ("sam_vision_model", "SamVisionConfig"),
+        ("sarvam_mla", "SarvamMLAConfig"),
         ("seamless_m4t", "SeamlessM4TConfig"),
         ("seamless_m4t_v2", "SeamlessM4Tv2Config"),
         ("seed_oss", "SeedOssConfig"),
@@ -934,6 +935,7 @@
         ("sam_hq", "SAM-HQ"),
         ("sam_hq_vision_model", "SamHQVisionModel"),
         ("sam_vision_model", "SamVisionModel"),
+        ("sarvam_mla", "SarvamMLA"),
         ("seamless_m4t", "SeamlessM4T"),
         ("seamless_m4t_v2", "SeamlessM4Tv2"),
         ("seed_oss", "SeedOss"),

src/transformers/models/auto/modeling_auto.py

@@ -397,6 +397,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("sam_hq", "SamHQModel"),
         ("sam_hq_vision_model", "SamHQVisionModel"),
         ("sam_vision_model", "SamVisionModel"),
+        ("sarvam_mla", "DeepseekV3Model"),
         ("seamless_m4t", "SeamlessM4TModel"),
         ("seamless_m4t_v2", "SeamlessM4Tv2Model"),
         ("seed_oss", "SeedOssModel"),
@@ -722,6 +723,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("roc_bert", "RoCBertForCausalLM"),
         ("roformer", "RoFormerForCausalLM"),
         ("rwkv", "RwkvForCausalLM"),
+        ("sarvam_mla", "DeepseekV3ForCausalLM"),
         ("seed_oss", "SeedOssForCausalLM"),
         ("smollm3", "SmolLM3ForCausalLM"),
         ("solar_open", "SolarOpenForCausalLM"),
@@ -1280,6 +1282,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("roberta-prelayernorm", "RobertaPreLayerNormForSequenceClassification"),
         ("roc_bert", "RoCBertForSequenceClassification"),
         ("roformer", "RoFormerForSequenceClassification"),
+        ("sarvam_mla", "DeepseekV3ForSequenceClassification"),
         ("seed_oss", "SeedOssForSequenceClassification"),
         ("smollm3", "SmolLM3ForSequenceClassification"),
         ("squeezebert", "SqueezeBertForSequenceClassification"),
@@ -1484,6 +1487,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("roberta-prelayernorm", "RobertaPreLayerNormForTokenClassification"),
         ("roc_bert", "RoCBertForTokenClassification"),
         ("roformer", "RoFormerForTokenClassification"),
+        ("sarvam_mla", "DeepseekV3ForTokenClassification"),
         ("seed_oss", "SeedOssForTokenClassification"),
         ("smollm3", "SmolLM3ForTokenClassification"),
         ("squeezebert", "SqueezeBertForTokenClassification"),

src/transformers/models/sarvam_mla/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2026 Sarvam AI and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_sarvam_mla import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

src/transformers/models/sarvam_mla/configuration_sarvam_mla.py

@@ -0,0 +1,118 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/sarvam_mla/modular_sarvam_mla.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_sarvam_mla.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 Sarvam AI and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...modeling_rope_utils import RopeParameters
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="sarvamai/sarvam-105b")
+@strict(accept_kwargs=True)
+class SarvamMLAConfig(PreTrainedConfig):
+    r"""
+    n_group (`int`, *optional*, defaults to 16):
+        Number of groups for routed experts.
+    rope_interleave (`bool`, *optional*, defaults to `True`):
+        Whether to interleave the rotary position embeddings.
+    first_k_dense_replace (`int`, *optional*, defaults to 1):
+        Number of dense layers in shallow layers(embed->dense->moe->moe...->lm_head).
+                                                        \--k dense layers--/
+
+    Example:
+
+    ```python
+    >>> from transformers import SarvamMLAConfig
+
+    >>> # Initializing a SarvamMLA style configuration
+    >>> configuration = SarvamMLAConfig()
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "sarvam_mla"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    base_model_tp_plan = {
+        "layers.*.mlp.experts.gate_up_proj": "packed_colwise",
+        "layers.*.mlp.experts.down_proj": "rowwise",
+        "layers.*.mlp.experts": "moe_tp_experts",
+        "layers.*.mlp.shared_experts.gate_proj": "colwise",
+        "layers.*.mlp.shared_experts.up_proj": "colwise",
+        "layers.*.mlp.shared_experts.down_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+    attribute_map = {
+        "num_local_experts": "n_routed_experts",
+    }
+
+    vocab_size: int = 262144
+    hidden_size: int = 4096
+    intermediate_size: int = 16384
+    moe_intermediate_size: int = 2048
+    num_hidden_layers: int = 32
+    num_attention_heads: int = 64
+    num_key_value_heads: int | None = None
+    n_shared_experts: int = 1
+    n_routed_experts: int = 128
+    routed_scaling_factor: float = 2.5
+    kv_lora_rank: int = 512
+    q_lora_rank: int | None = None
+    qk_rope_head_dim: int = 64
+    v_head_dim: int | None = 128
+    qk_nope_head_dim: int = 128
+    n_group: int | None = 16
+    topk_group: int | None = 2
+    num_experts_per_tok: int | None = 8
+    first_k_dense_replace: int | None = 1
+    norm_topk_prob: bool | None = True
+    hidden_act: str = "silu"
+    max_position_embeddings: int = 4096
+    initializer_range: float = 0.006
+    rms_norm_eps: float = 1e-6
+    use_cache: bool = True
+    pad_token_id: int | None = None
+    bos_token_id: int | None = 0
+    eos_token_id: int | list[int] | None = 1
+    pretraining_tp: int | None = 1
+    tie_word_embeddings: bool = False
+    rope_parameters: RopeParameters | dict | None = None
+    rope_interleave: bool | None = True
+    attention_bias: bool = False
+    attention_dropout: float | int | None = 0.0
+
+    def __post_init__(self, **kwargs):
+        if self.num_key_value_heads is None:
+            self.num_key_value_heads = self.num_attention_heads
+
+        self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
+        self.head_dim = self.qk_rope_head_dim
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["SarvamMLAConfig"]

src/transformers/models/sarvam_mla/modular_sarvam_mla.py

@@ -0,0 +1,64 @@
+# Copyright 2026 Sarvam AI and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from huggingface_hub.dataclasses import strict
+
+from ...utils import auto_docstring
+from ..deepseek_v3.configuration_deepseek_v3 import DeepseekV3Config
+
+
+@auto_docstring(checkpoint="sarvamai/sarvam-105b")
+@strict(accept_kwargs=True)
+class SarvamMLAConfig(DeepseekV3Config):
+    r"""
+    n_group (`int`, *optional*, defaults to 16):
+        Number of groups for routed experts.
+    rope_interleave (`bool`, *optional*, defaults to `True`):
+        Whether to interleave the rotary position embeddings.
+    first_k_dense_replace (`int`, *optional*, defaults to 1):
+        Number of dense layers in shallow layers(embed->dense->moe->moe...->lm_head).
+                                                        \--k dense layers--/
+
+    Example:
+
+    ```python
+    >>> from transformers import SarvamMLAConfig
+
+    >>> # Initializing a SarvamMLA style configuration
+    >>> configuration = SarvamMLAConfig()
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "sarvam_mla"
+
+    vocab_size: int = 262144
+    hidden_size: int = 4096
+    intermediate_size: int = 16384
+    num_hidden_layers: int = 32
+    num_attention_heads: int = 64
+    num_key_value_heads: int | None = None
+    n_routed_experts: int = 128
+    q_lora_rank: int | None = None
+    n_group: int | None = 16
+    topk_group: int | None = 2
+    first_k_dense_replace: int | None = 1
+    initializer_range: float = 0.006
+
+    def convert_rope_params_to_dict(self, **kwargs):
+        raise AttributeError("No BC behavior needed!")
+
+
+__all__ = ["SarvamMLAConfig"]

utils/check_config_attributes.py

@@ -138,6 +138,7 @@
     "GptOssConfig": True,
     "LwDetrConfig": True,
     "NemotronHConfig": True,
+    "SarvamMLAConfig": True,  # Uses DeepseekV3 under the hood
 }
 
 # Common and important attributes, even if they do not always appear in the modeling files (can be a regex pattern)