Merge branch 'main' of https://github.com/embeddings-benchmark/results

makefile

@@ -5,4 +5,8 @@ install-for-tests:
 
 test:
 	@echo "--- Running tests ---"
-	pytest
+	pytest
+
+pre-push:
+	@echo "--- Running pre-push commands ---"
+	python reduce_large_json_files.py

results.py

@@ -398,8 +398,8 @@
     "google-bert__bert-base-multilingual-cased",
     "google-bert__bert-base-multilingual-uncased",
     "google-bert__bert-base-uncased",
-    "google-gecko__text-embedding-preview-0409",
-    "google-gecko__text-embedding-preview-0409-256",
+    "google-gecko__text-embedding-preview-004",
+    "google-gecko__text-embedding-preview-004-256",
     "google__flan-t5-base",
     "google__flan-t5-large",
     "hkunlp__instructor-base",

results/Alibaba-NLP__gte-multilingual-base/7fc06782350c1a83f88b15dd4b38ef853d3b8503/AILAStatutes.json

@@ -0,0 +1,158 @@
+{
+  "dataset_revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e",
+  "evaluation_time": 2.1449167728424072,
+  "kg_co2_emissions": 0.00018021358730526757,
+  "mteb_version": "1.12.75",
+  "scores": {
+    "test": [
+      {
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "main_score": 0.33572,
+        "map_at_1": 0.098,
+        "map_at_10": 0.21049,
+        "map_at_100": 0.27684,
+        "map_at_1000": 0.27684,
+        "map_at_20": 0.23128,
+        "map_at_3": 0.15517,
+        "map_at_5": 0.17602,
+        "mrr_at_1": 0.42,
+        "mrr_at_10": 0.5489920634920634,
+        "mrr_at_100": 0.5579525811353396,
+        "mrr_at_1000": 0.5579525811353396,
+        "mrr_at_20": 0.5555538350538349,
+        "mrr_at_3": 0.5099999999999999,
+        "mrr_at_5": 0.5289999999999999,
+        "nauc_map_at_1000_diff1": 0.12334961747338444,
+        "nauc_map_at_1000_max": 0.30619209224293353,
+        "nauc_map_at_1000_std": -0.030718990518868516,
+        "nauc_map_at_100_diff1": 0.12334961747338444,
+        "nauc_map_at_100_max": 0.30619209224293353,
+        "nauc_map_at_100_std": -0.030718990518868516,
+        "nauc_map_at_10_diff1": 0.12326986123721569,
+        "nauc_map_at_10_max": 0.33448409936254264,
+        "nauc_map_at_10_std": -0.005586989019712461,
+        "nauc_map_at_1_diff1": 0.051085650745113044,
+        "nauc_map_at_1_max": 0.291641432443434,
+        "nauc_map_at_1_std": -0.039822086315935736,
+        "nauc_map_at_20_diff1": 0.14097198525881416,
+        "nauc_map_at_20_max": 0.32616722893992217,
+        "nauc_map_at_20_std": -0.02786122313893844,
+        "nauc_map_at_3_diff1": 0.14474626451994102,
+        "nauc_map_at_3_max": 0.3591146061343692,
+        "nauc_map_at_3_std": -0.000559546471639301,
+        "nauc_map_at_5_diff1": 0.10546517261748972,
+        "nauc_map_at_5_max": 0.35085876307049013,
+        "nauc_map_at_5_std": 0.0020084071107343703,
+        "nauc_mrr_at_1000_diff1": 0.035944630624480334,
+        "nauc_mrr_at_1000_max": 0.2340912362618136,
+        "nauc_mrr_at_1000_std": -0.054160371582013825,
+        "nauc_mrr_at_100_diff1": 0.035944630624480334,
+        "nauc_mrr_at_100_max": 0.2340912362618136,
+        "nauc_mrr_at_100_std": -0.054160371582013825,
+        "nauc_mrr_at_10_diff1": 0.035670988619914745,
+        "nauc_mrr_at_10_max": 0.2328703059621917,
+        "nauc_mrr_at_10_std": -0.04531014796575209,
+        "nauc_mrr_at_1_diff1": 0.016123596961920575,
+        "nauc_mrr_at_1_max": 0.18407341161832186,
+        "nauc_mrr_at_1_std": -0.11385022163465279,
+        "nauc_mrr_at_20_diff1": 0.035045783552563894,
+        "nauc_mrr_at_20_max": 0.23431242727526086,
+        "nauc_mrr_at_20_std": -0.05011543297767148,
+        "nauc_mrr_at_3_diff1": 0.0667203800686233,
+        "nauc_mrr_at_3_max": 0.2589217497603732,
+        "nauc_mrr_at_3_std": -0.07863919874421772,
+        "nauc_mrr_at_5_diff1": 0.016788634150608658,
+        "nauc_mrr_at_5_max": 0.2233374491085115,
+        "nauc_mrr_at_5_std": -0.0389593255291351,
+        "nauc_ndcg_at_1000_diff1": 0.08036072110815415,
+        "nauc_ndcg_at_1000_max": 0.25314596958217483,
+        "nauc_ndcg_at_1000_std": -0.04763471197741127,
+        "nauc_ndcg_at_100_diff1": 0.08036072110815415,
+        "nauc_ndcg_at_100_max": 0.25314596958217483,
+        "nauc_ndcg_at_100_std": -0.04763471197741127,
+        "nauc_ndcg_at_10_diff1": 0.10691068836439659,
+        "nauc_ndcg_at_10_max": 0.30903016453390264,
+        "nauc_ndcg_at_10_std": 0.005505461793332526,
+        "nauc_ndcg_at_1_diff1": 0.016123596961920575,
+        "nauc_ndcg_at_1_max": 0.18407341161832186,
+        "nauc_ndcg_at_1_std": -0.11385022163465279,
+        "nauc_ndcg_at_20_diff1": 0.13861577805806724,
+        "nauc_ndcg_at_20_max": 0.3057236603316381,
+        "nauc_ndcg_at_20_std": -0.023205629408024175,
+        "nauc_ndcg_at_3_diff1": 0.11908115635412221,
+        "nauc_ndcg_at_3_max": 0.2912550866483106,
+        "nauc_ndcg_at_3_std": -0.03811034731037659,
+        "nauc_ndcg_at_5_diff1": 0.08104210450975674,
+        "nauc_ndcg_at_5_max": 0.31527897011513895,
+        "nauc_ndcg_at_5_std": 0.006544003750270816,
+        "nauc_precision_at_1000_diff1": -0.35315855911893174,
+        "nauc_precision_at_1000_max": -0.5976107279413103,
+        "nauc_precision_at_1000_std": -0.38996725270302257,
+        "nauc_precision_at_100_diff1": -0.3531585591189289,
+        "nauc_precision_at_100_max": -0.5976107279413052,
+        "nauc_precision_at_100_std": -0.3899672527030205,
+        "nauc_precision_at_10_diff1": 0.05138605745990966,
+        "nauc_precision_at_10_max": 0.1751749647968724,
+        "nauc_precision_at_10_std": -0.006504697253105322,
+        "nauc_precision_at_1_diff1": 0.016123596961920575,
+        "nauc_precision_at_1_max": 0.18407341161832186,
+        "nauc_precision_at_1_std": -0.11385022163465279,
+        "nauc_precision_at_20_diff1": 0.09554317272068041,
+        "nauc_precision_at_20_max": 0.11491204545228778,
+        "nauc_precision_at_20_std": -0.09871172164446725,
+        "nauc_precision_at_3_diff1": 0.14812347324006228,
+        "nauc_precision_at_3_max": 0.291549223594527,
+        "nauc_precision_at_3_std": -0.027742188968038237,
+        "nauc_precision_at_5_diff1": 0.04836676217765031,
+        "nauc_precision_at_5_max": 0.23323782234957008,
+        "nauc_precision_at_5_std": 0.009212034383954164,
+        "nauc_recall_at_1000_diff1": NaN,
+        "nauc_recall_at_1000_max": NaN,
+        "nauc_recall_at_1000_std": NaN,
+        "nauc_recall_at_100_diff1": NaN,
+        "nauc_recall_at_100_max": NaN,
+        "nauc_recall_at_100_std": NaN,
+        "nauc_recall_at_10_diff1": 0.13946061260668566,
+        "nauc_recall_at_10_max": 0.3110333408713887,
+        "nauc_recall_at_10_std": 0.055293217173481755,
+        "nauc_recall_at_1_diff1": 0.051085650745113044,
+        "nauc_recall_at_1_max": 0.291641432443434,
+        "nauc_recall_at_1_std": -0.039822086315935736,
+        "nauc_recall_at_20_diff1": 0.22936946064692115,
+        "nauc_recall_at_20_max": 0.3165148473600486,
+        "nauc_recall_at_20_std": 0.012884914021665129,
+        "nauc_recall_at_3_diff1": 0.16820451669101075,
+        "nauc_recall_at_3_max": 0.38184856739462425,
+        "nauc_recall_at_3_std": 0.018151654250827248,
+        "nauc_recall_at_5_diff1": 0.07711443028168376,
+        "nauc_recall_at_5_max": 0.3253949638880427,
+        "nauc_recall_at_5_std": 0.05426185329620741,
+        "ndcg_at_1": 0.42,
+        "ndcg_at_10": 0.33572,
+        "ndcg_at_100": 0.54256,
+        "ndcg_at_1000": 0.54256,
+        "ndcg_at_20": 0.38576,
+        "ndcg_at_3": 0.30224,
+        "ndcg_at_5": 0.27254,
+        "precision_at_1": 0.42,
+        "precision_at_10": 0.154,
+        "precision_at_100": 0.0434,
+        "precision_at_1000": 0.00434,
+        "precision_at_20": 0.104,
+        "precision_at_3": 0.26,
+        "precision_at_5": 0.2,
+        "recall_at_1": 0.098,
+        "recall_at_10": 0.36067,
+        "recall_at_100": 1.0,
+        "recall_at_1000": 1.0,
+        "recall_at_20": 0.481,
+        "recall_at_3": 0.18667,
+        "recall_at_5": 0.23767
+      }
+    ]
+  },
+  "task_name": "AILAStatutes"
+}