embeddings-benchmark
diff --git a/‎docs/tasks.md
Lines changed: 22 additions & 8 deletions b/‎docs/tasks.md
Lines changed: 22 additions & 8 deletions
diff --git a/‎mteb/leaderboard/app.py
Lines changed: 30 additions & 22 deletions b/‎mteb/leaderboard/app.py
Lines changed: 30 additions & 22 deletions
diff --git a/‎mteb/models/fa_models.py
Lines changed: 40 additions & 0 deletions b/‎mteb/models/fa_models.py
Lines changed: 40 additions & 0 deletions
diff --git a/‎mteb/models/hinvec_models.py
Lines changed: 60 additions & 0 deletions b/‎mteb/models/hinvec_models.py
Lines changed: 60 additions & 0 deletions
@@ -13,7 +13,6 @@
 import cachetools
 import gradio as gr
 import pandas as pd
-from gradio_rangeslider import RangeSlider
 
 import mteb
 from mteb.abstasks.TaskMetadata import TASK_DOMAIN, TASK_TYPE
@@ -158,10 +157,10 @@ def filter_models(
     availability: bool | None,
     compatibility: list[str],
     instructions: bool | None,
-    model_size: tuple[int | None, int | None],
+    max_model_size: int,
     zero_shot_setting: Literal["only_zero_shot", "allow_all", "remove_unknown"],
 ):
-    lower, upper = model_size
+    lower, upper = 0, max_model_size
     # Setting to None, when the user doesn't specify anything
     if (lower == MIN_MODEL_SIZE) or (lower is None):
         lower = None
@@ -179,6 +178,7 @@ def filter_models(
         frameworks=compatibility,
         n_parameters_range=(lower, upper),
     )
+
     models_to_keep = set()
     for model_meta in model_metas:
         is_model_zero_shot = model_meta.is_zero_shot_on(task_select)
@@ -217,7 +217,7 @@ def get_leaderboard_app() -> gr.Blocks:
         availability=None,
         compatibility=[],
         instructions=None,
-        model_size=(MIN_MODEL_SIZE, MAX_MODEL_SIZE),
+        max_model_size=MAX_MODEL_SIZE,
         zero_shot_setting="allow_all",
     )
 
@@ -378,11 +378,19 @@ def get_leaderboard_app() -> gr.Blocks:
                             label="Zero-shot",
                             interactive=True,
                         )
-                        model_size = RangeSlider(
-                            minimum=MIN_MODEL_SIZE,
-                            maximum=MAX_MODEL_SIZE,
-                            value=(MIN_MODEL_SIZE, MAX_MODEL_SIZE),
-                            label="Model Size (#M Parameters)",
+
+                        max_model_size = gr.Radio(
+                            [
+                                ("<100M", 100),
+                                ("<500M", 500),
+                                ("<1B", 1000),
+                                ("<5B", 5000),
+                                ("<10B", 10000),
+                                (">10B", MAX_MODEL_SIZE),
+                            ],
+                            value=MAX_MODEL_SIZE,
+                            label="Model Parameters",
+                            interactive=True,
                         )
 
         with gr.Tab("Summary"):
@@ -580,15 +588,15 @@ def update_task_list(
             availability,
             compatibility,
             instructions,
-            model_size,
+            max_model_size,
             zero_shot: hash(
                 (
                     id(scores),
                     hash(tuple(tasks)),
                     hash(availability),
                     hash(tuple(compatibility)),
                     hash(instructions),
-                    hash(model_size),
+                    hash(max_model_size),
                     hash(zero_shot),
                 )
             ),
@@ -599,7 +607,7 @@ def update_models(
             availability: bool | None,
             compatibility: list[str],
             instructions: bool | None,
-            model_size: tuple[int, int],
+            max_model_size: int,
             zero_shot: Literal["allow_all", "remove_unknown", "only_zero_shot"],
         ):
             start_time = time.time()
@@ -610,7 +618,7 @@ def update_models(
                 availability,
                 compatibility,
                 instructions,
-                model_size,
+                max_model_size,
                 zero_shot_setting=zero_shot,
             )
             elapsed = time.time() - start_time
@@ -628,7 +636,7 @@ def update_models(
                 availability,
                 compatibility,
                 instructions,
-                model_size,
+                max_model_size,
                 zero_shot,
             ],
             outputs=[models],
@@ -641,7 +649,7 @@ def update_models(
                 availability,
                 compatibility,
                 instructions,
-                model_size,
+                max_model_size,
                 zero_shot,
             ],
             outputs=[models],
@@ -654,7 +662,7 @@ def update_models(
                 availability,
                 compatibility,
                 instructions,
-                model_size,
+                max_model_size,
                 zero_shot,
             ],
             outputs=[models],
@@ -667,7 +675,7 @@ def update_models(
                 availability,
                 compatibility,
                 instructions,
-                model_size,
+                max_model_size,
                 zero_shot,
             ],
             outputs=[models],
@@ -680,20 +688,20 @@ def update_models(
                 availability,
                 compatibility,
                 instructions,
-                model_size,
+                max_model_size,
                 zero_shot,
             ],
             outputs=[models],
         )
-        model_size.change(
+        max_model_size.change(
             update_models,
             inputs=[
                 scores,
                 task_select,
                 availability,
                 compatibility,
                 instructions,
-                model_size,
+                max_model_size,
                 zero_shot,
             ],
             outputs=[models],
@@ -706,7 +714,7 @@ def update_models(
                 availability,
                 compatibility,
                 instructions,
-                model_size,
+                max_model_size,
                 zero_shot,
             ],
             outputs=[models],
@@ -784,7 +792,7 @@ def update_tables(
             availability=None,
             compatibility=[],
             instructions=None,
-            model_size=(MIN_MODEL_SIZE, MAX_MODEL_SIZE),
+            max_model_size=MAX_MODEL_SIZE,
             zero_shot="allow_all",
         )
         # We have to call this both on the filtered and unfiltered task because the callbacks
 
@@ -166,3 +166,43 @@
         # https://huggingface.co/datasets/sbunlp/hmblogs-v3
     },
 )
+
+tooka_sbert_v2_small = ModelMeta(
+    name="PartAI/Tooka-SBERT-V2-Small",
+    languages=["fas-Arab"],
+    open_weights=True,
+    revision="8bbed87e36669387f71437c061430ba56d1b496f",
+    release_date="2025-05-01",
+    n_parameters=122_905_344,
+    memory_usage_mb=496,
+    embed_dim=768,
+    license="not specified",
+    max_tokens=512,
+    reference="https://huggingface.co/PartAI/Tooka-SBERT-V2-Small",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=False,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets=None,
+)
+
+tooka_sbert_v2_large = ModelMeta(
+    name="PartAI/Tooka-SBERT-V2-Large",
+    languages=["fas-Arab"],
+    open_weights=True,
+    revision="b59682efa961122cc0e4408296d5852870c82eae",
+    release_date="2025-05-01",
+    n_parameters=353_039_360,
+    memory_usage_mb=1347,
+    embed_dim=1024,
+    license="not specified",
+    max_tokens=512,
+    reference="https://huggingface.co/PartAI/Tooka-SBERT-V2-Large",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=False,
+    public_training_code=None,
+    public_training_data=None,
+    training_datasets=None,
+)
@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import logging
+from functools import partial
+
+from mteb.encoder_interface import PromptType
+from mteb.model_meta import ModelMeta, sentence_transformers_loader
+
+logger = logging.getLogger(__name__)
+
+
+def instruction_template(
+    instruction: str, prompt_type: PromptType | None = None
+) -> str:
+    return f"Instruct: {instruction}\nQuery: " if instruction else ""
+
+
+hinvec_training_datasets = {
+    "MintakaRetrieval": ["train"],
+    "HindiDiscourseClassification": ["train"],
+    "SentimentAnalysisHindi": ["train"],
+    "MassiveScenarioClassification": ["train"],
+    "MTOPIntentClassification": ["train"],
+    "LinceMTBitextMining": ["train"],
+    "PhincBitextMining": ["train"],
+    "XNLI": ["train"],
+    "MLQARetrieval": ["validation"],
+    "FloresBitextMining": ["dev"],
+    "AmazonReviewsClassification": ["train"],
+}
+
+Hinvec_bidir = ModelMeta(
+    loader=partial(  # type: ignore
+        sentence_transformers_loader,
+        model_name="Sailesh97/Hinvec",
+        revision="d4fc678720cc1b8c5d18599ce2d9a4d6090c8b6b",
+        instruction_template=instruction_template,
+        trust_remote_code=True,
+        max_seq_length=2048,
+        padding_side="left",
+        add_eos_token=True,
+    ),
+    name="Sailesh97/Hinvec",
+    languages=["eng-Latn", "hin-Deva"],
+    open_weights=True,
+    revision="d4fc678720cc1b8c5d18599ce2d9a4d6090c8b6b",
+    release_date="2025-06-19",
+    n_parameters=939_591_680,
+    memory_usage_mb=3715,
+    embed_dim=2048,
+    license="cc-by-nc-4.0",
+    max_tokens=2048,
+    reference="https://huggingface.co/Sailesh97/Hinvec",
+    similarity_fn_name="cosine",
+    framework=["Sentence Transformers", "PyTorch"],
+    use_instructions=True,
+    training_datasets=hinvec_training_datasets,
+    public_training_code=None,
+    public_training_data=None,
+)