Skip to content

Commit

Permalink
Added results on MTEB(Multilingual) and HardNegative retrieval tasks
Browse files Browse the repository at this point in the history
* Add

* Add res

* fix: reduce sizes of files and added makefile command

---------

Co-authored-by: Kenneth Enevoldsen <[email protected]>
  • Loading branch information
Muennighoff and KennethEnevoldsen authored Sep 29, 2024
1 parent 11a9906 commit 7a67ef5
Show file tree
Hide file tree
Showing 491 changed files with 280,022 additions and 3 deletions.
6 changes: 5 additions & 1 deletion makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,8 @@ install-for-tests:

test:
@echo "--- Running tests ---"
pytest
pytest

pre-push:
@echo "--- Running pre-push commands ---"
python reduce_large_json_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e",
"evaluation_time": 2.1449167728424072,
"kg_co2_emissions": 0.00018021358730526757,
"mteb_version": "1.12.75",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.33572,
"map_at_1": 0.098,
"map_at_10": 0.21049,
"map_at_100": 0.27684,
"map_at_1000": 0.27684,
"map_at_20": 0.23128,
"map_at_3": 0.15517,
"map_at_5": 0.17602,
"mrr_at_1": 0.42,
"mrr_at_10": 0.5489920634920634,
"mrr_at_100": 0.5579525811353396,
"mrr_at_1000": 0.5579525811353396,
"mrr_at_20": 0.5555538350538349,
"mrr_at_3": 0.5099999999999999,
"mrr_at_5": 0.5289999999999999,
"nauc_map_at_1000_diff1": 0.12334961747338444,
"nauc_map_at_1000_max": 0.30619209224293353,
"nauc_map_at_1000_std": -0.030718990518868516,
"nauc_map_at_100_diff1": 0.12334961747338444,
"nauc_map_at_100_max": 0.30619209224293353,
"nauc_map_at_100_std": -0.030718990518868516,
"nauc_map_at_10_diff1": 0.12326986123721569,
"nauc_map_at_10_max": 0.33448409936254264,
"nauc_map_at_10_std": -0.005586989019712461,
"nauc_map_at_1_diff1": 0.051085650745113044,
"nauc_map_at_1_max": 0.291641432443434,
"nauc_map_at_1_std": -0.039822086315935736,
"nauc_map_at_20_diff1": 0.14097198525881416,
"nauc_map_at_20_max": 0.32616722893992217,
"nauc_map_at_20_std": -0.02786122313893844,
"nauc_map_at_3_diff1": 0.14474626451994102,
"nauc_map_at_3_max": 0.3591146061343692,
"nauc_map_at_3_std": -0.000559546471639301,
"nauc_map_at_5_diff1": 0.10546517261748972,
"nauc_map_at_5_max": 0.35085876307049013,
"nauc_map_at_5_std": 0.0020084071107343703,
"nauc_mrr_at_1000_diff1": 0.035944630624480334,
"nauc_mrr_at_1000_max": 0.2340912362618136,
"nauc_mrr_at_1000_std": -0.054160371582013825,
"nauc_mrr_at_100_diff1": 0.035944630624480334,
"nauc_mrr_at_100_max": 0.2340912362618136,
"nauc_mrr_at_100_std": -0.054160371582013825,
"nauc_mrr_at_10_diff1": 0.035670988619914745,
"nauc_mrr_at_10_max": 0.2328703059621917,
"nauc_mrr_at_10_std": -0.04531014796575209,
"nauc_mrr_at_1_diff1": 0.016123596961920575,
"nauc_mrr_at_1_max": 0.18407341161832186,
"nauc_mrr_at_1_std": -0.11385022163465279,
"nauc_mrr_at_20_diff1": 0.035045783552563894,
"nauc_mrr_at_20_max": 0.23431242727526086,
"nauc_mrr_at_20_std": -0.05011543297767148,
"nauc_mrr_at_3_diff1": 0.0667203800686233,
"nauc_mrr_at_3_max": 0.2589217497603732,
"nauc_mrr_at_3_std": -0.07863919874421772,
"nauc_mrr_at_5_diff1": 0.016788634150608658,
"nauc_mrr_at_5_max": 0.2233374491085115,
"nauc_mrr_at_5_std": -0.0389593255291351,
"nauc_ndcg_at_1000_diff1": 0.08036072110815415,
"nauc_ndcg_at_1000_max": 0.25314596958217483,
"nauc_ndcg_at_1000_std": -0.04763471197741127,
"nauc_ndcg_at_100_diff1": 0.08036072110815415,
"nauc_ndcg_at_100_max": 0.25314596958217483,
"nauc_ndcg_at_100_std": -0.04763471197741127,
"nauc_ndcg_at_10_diff1": 0.10691068836439659,
"nauc_ndcg_at_10_max": 0.30903016453390264,
"nauc_ndcg_at_10_std": 0.005505461793332526,
"nauc_ndcg_at_1_diff1": 0.016123596961920575,
"nauc_ndcg_at_1_max": 0.18407341161832186,
"nauc_ndcg_at_1_std": -0.11385022163465279,
"nauc_ndcg_at_20_diff1": 0.13861577805806724,
"nauc_ndcg_at_20_max": 0.3057236603316381,
"nauc_ndcg_at_20_std": -0.023205629408024175,
"nauc_ndcg_at_3_diff1": 0.11908115635412221,
"nauc_ndcg_at_3_max": 0.2912550866483106,
"nauc_ndcg_at_3_std": -0.03811034731037659,
"nauc_ndcg_at_5_diff1": 0.08104210450975674,
"nauc_ndcg_at_5_max": 0.31527897011513895,
"nauc_ndcg_at_5_std": 0.006544003750270816,
"nauc_precision_at_1000_diff1": -0.35315855911893174,
"nauc_precision_at_1000_max": -0.5976107279413103,
"nauc_precision_at_1000_std": -0.38996725270302257,
"nauc_precision_at_100_diff1": -0.3531585591189289,
"nauc_precision_at_100_max": -0.5976107279413052,
"nauc_precision_at_100_std": -0.3899672527030205,
"nauc_precision_at_10_diff1": 0.05138605745990966,
"nauc_precision_at_10_max": 0.1751749647968724,
"nauc_precision_at_10_std": -0.006504697253105322,
"nauc_precision_at_1_diff1": 0.016123596961920575,
"nauc_precision_at_1_max": 0.18407341161832186,
"nauc_precision_at_1_std": -0.11385022163465279,
"nauc_precision_at_20_diff1": 0.09554317272068041,
"nauc_precision_at_20_max": 0.11491204545228778,
"nauc_precision_at_20_std": -0.09871172164446725,
"nauc_precision_at_3_diff1": 0.14812347324006228,
"nauc_precision_at_3_max": 0.291549223594527,
"nauc_precision_at_3_std": -0.027742188968038237,
"nauc_precision_at_5_diff1": 0.04836676217765031,
"nauc_precision_at_5_max": 0.23323782234957008,
"nauc_precision_at_5_std": 0.009212034383954164,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": NaN,
"nauc_recall_at_100_max": NaN,
"nauc_recall_at_100_std": NaN,
"nauc_recall_at_10_diff1": 0.13946061260668566,
"nauc_recall_at_10_max": 0.3110333408713887,
"nauc_recall_at_10_std": 0.055293217173481755,
"nauc_recall_at_1_diff1": 0.051085650745113044,
"nauc_recall_at_1_max": 0.291641432443434,
"nauc_recall_at_1_std": -0.039822086315935736,
"nauc_recall_at_20_diff1": 0.22936946064692115,
"nauc_recall_at_20_max": 0.3165148473600486,
"nauc_recall_at_20_std": 0.012884914021665129,
"nauc_recall_at_3_diff1": 0.16820451669101075,
"nauc_recall_at_3_max": 0.38184856739462425,
"nauc_recall_at_3_std": 0.018151654250827248,
"nauc_recall_at_5_diff1": 0.07711443028168376,
"nauc_recall_at_5_max": 0.3253949638880427,
"nauc_recall_at_5_std": 0.05426185329620741,
"ndcg_at_1": 0.42,
"ndcg_at_10": 0.33572,
"ndcg_at_100": 0.54256,
"ndcg_at_1000": 0.54256,
"ndcg_at_20": 0.38576,
"ndcg_at_3": 0.30224,
"ndcg_at_5": 0.27254,
"precision_at_1": 0.42,
"precision_at_10": 0.154,
"precision_at_100": 0.0434,
"precision_at_1000": 0.00434,
"precision_at_20": 0.104,
"precision_at_3": 0.26,
"precision_at_5": 0.2,
"recall_at_1": 0.098,
"recall_at_10": 0.36067,
"recall_at_100": 1.0,
"recall_at_1000": 1.0,
"recall_at_20": 0.481,
"recall_at_3": 0.18667,
"recall_at_5": 0.23767
}
]
},
"task_name": "AILAStatutes"
}
Loading

0 comments on commit 7a67ef5

Please sign in to comment.