Skip to content

Commit

Permalink
Standardize grit results
Browse files Browse the repository at this point in the history
  • Loading branch information
@Muennighoff
Muennighoff committed Sep 10, 2024
1 parent 16d7a28 commit bf3cf9b
Show file tree
Hide file tree
Showing 22 changed files with 17,761 additions and 15,372 deletions.
30,759 changes: 15,388 additions & 15,371 deletions paths.json
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
"Cohere-embed-multilingual-v3.0",
"DanskBERT",
"FollowIR-7B",
"GritLM-7B",
"GritLM__GritLM-7B",
"GritLM__GritLM-7B-noinstruct",
"LASER2",
"LLM2Vec-Llama-2-supervised",
"LLM2Vec-Llama-2-unsupervised",
Expand Down
158 changes: 158 additions & 0 deletions ...s/GritLM__GritLM-7B-noinstruct/13f00a0e36500c80ce12870ea513846a066004af/ARCChallenge.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "c481e0da3dcbbad8bce7721dea9085b74320a0a3",
"evaluation_time": 78.18728423118591,
"kg_co2_emissions": null,
"mteb_version": "1.12.27",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.16567,
"map_at_1": 0.08276,
"map_at_10": 0.1332,
"map_at_100": 0.14363,
"map_at_1000": 0.14482,
"map_at_20": 0.13866,
"map_at_3": 0.11277,
"map_at_5": 0.12433,
"mrr_at_1": 0.08276450511945392,
"mrr_at_10": 0.13319620510320165,
"mrr_at_100": 0.14362586972749813,
"mrr_at_1000": 0.1448178804221324,
"mrr_at_20": 0.1386578989402944,
"mrr_at_3": 0.11277019340159264,
"mrr_at_5": 0.12433162684869167,
"nauc_map_at_1000_diff1": 0.11606267607104377,
"nauc_map_at_1000_max": -0.10084652504177026,
"nauc_map_at_1000_std": -0.11828505538008827,
"nauc_map_at_100_diff1": 0.11617938969146996,
"nauc_map_at_100_max": -0.10103608095245299,
"nauc_map_at_100_std": -0.11836784322419454,
"nauc_map_at_10_diff1": 0.12171741485767183,
"nauc_map_at_10_max": -0.0951712196064573,
"nauc_map_at_10_std": -0.12040642685533308,
"nauc_map_at_1_diff1": 0.1637549340679917,
"nauc_map_at_1_max": -0.06866843294505832,
"nauc_map_at_1_std": -0.11623156565719857,
"nauc_map_at_20_diff1": 0.1189920257095774,
"nauc_map_at_20_max": -0.0991905913147805,
"nauc_map_at_20_std": -0.12012391309456927,
"nauc_map_at_3_diff1": 0.13195461509097473,
"nauc_map_at_3_max": -0.08656050010347069,
"nauc_map_at_3_std": -0.12814440057787524,
"nauc_map_at_5_diff1": 0.12381744141594893,
"nauc_map_at_5_max": -0.08188524850924572,
"nauc_map_at_5_std": -0.11540567655833621,
"nauc_mrr_at_1000_diff1": 0.11606267607104377,
"nauc_mrr_at_1000_max": -0.10084652504177026,
"nauc_mrr_at_1000_std": -0.11828505538008827,
"nauc_mrr_at_100_diff1": 0.11617938969146996,
"nauc_mrr_at_100_max": -0.10103608095245299,
"nauc_mrr_at_100_std": -0.11836784322419454,
"nauc_mrr_at_10_diff1": 0.12171741485767183,
"nauc_mrr_at_10_max": -0.0951712196064573,
"nauc_mrr_at_10_std": -0.12040642685533308,
"nauc_mrr_at_1_diff1": 0.1637549340679917,
"nauc_mrr_at_1_max": -0.06866843294505832,
"nauc_mrr_at_1_std": -0.11623156565719857,
"nauc_mrr_at_20_diff1": 0.1189920257095774,
"nauc_mrr_at_20_max": -0.0991905913147805,
"nauc_mrr_at_20_std": -0.12012391309456927,
"nauc_mrr_at_3_diff1": 0.13195461509097473,
"nauc_mrr_at_3_max": -0.08656050010347069,
"nauc_mrr_at_3_std": -0.12814440057787524,
"nauc_mrr_at_5_diff1": 0.12381744141594893,
"nauc_mrr_at_5_max": -0.08188524850924572,
"nauc_mrr_at_5_std": -0.11540567655833621,
"nauc_ndcg_at_1000_diff1": 0.07782564462113492,
"nauc_ndcg_at_1000_max": -0.1311850591294924,
"nauc_ndcg_at_1000_std": -0.1050073073458404,
"nauc_ndcg_at_100_diff1": 0.08039808605161892,
"nauc_ndcg_at_100_max": -0.130185279752856,
"nauc_ndcg_at_100_std": -0.10462409602995773,
"nauc_ndcg_at_10_diff1": 0.10518245547538639,
"nauc_ndcg_at_10_max": -0.11041301107009849,
"nauc_ndcg_at_10_std": -0.11849884556559163,
"nauc_ndcg_at_1_diff1": 0.1637549340679917,
"nauc_ndcg_at_1_max": -0.06866843294505832,
"nauc_ndcg_at_1_std": -0.11623156565719857,
"nauc_ndcg_at_20_diff1": 0.09856770854187791,
"nauc_ndcg_at_20_max": -0.12213906580274646,
"nauc_ndcg_at_20_std": -0.11819865442488073,
"nauc_ndcg_at_3_diff1": 0.12119705302223491,
"nauc_ndcg_at_3_max": -0.09227415998343752,
"nauc_ndcg_at_3_std": -0.12911306275481269,
"nauc_ndcg_at_5_diff1": 0.10981759290664823,
"nauc_ndcg_at_5_max": -0.08336011154639276,
"nauc_ndcg_at_5_std": -0.10750180809854237,
"nauc_precision_at_1000_diff1": -0.1695783827238421,
"nauc_precision_at_1000_max": -0.3245680277313244,
"nauc_precision_at_1000_std": -0.012669573395770703,
"nauc_precision_at_100_diff1": -0.013170183794179467,
"nauc_precision_at_100_max": -0.2004149608472424,
"nauc_precision_at_100_std": -0.05897637081303174,
"nauc_precision_at_10_diff1": 0.07275127233245807,
"nauc_precision_at_10_max": -0.14279613765432786,
"nauc_precision_at_10_std": -0.11534315061842612,
"nauc_precision_at_1_diff1": 0.1637549340679917,
"nauc_precision_at_1_max": -0.06866843294505832,
"nauc_precision_at_1_std": -0.11623156565719857,
"nauc_precision_at_20_diff1": 0.05928787026612005,
"nauc_precision_at_20_max": -0.17129862954688976,
"nauc_precision_at_20_std": -0.11542140579282543,
"nauc_precision_at_3_diff1": 0.09613544495372242,
"nauc_precision_at_3_max": -0.10552411799048211,
"nauc_precision_at_3_std": -0.13081876562122427,
"nauc_precision_at_5_diff1": 0.08113003762777546,
"nauc_precision_at_5_max": -0.08527591607021416,
"nauc_precision_at_5_std": -0.08915421346354885,
"nauc_recall_at_1000_diff1": -0.16957838272384387,
"nauc_recall_at_1000_max": -0.3245680277313267,
"nauc_recall_at_1000_std": -0.012669573395772498,
"nauc_recall_at_100_diff1": -0.013170183794179738,
"nauc_recall_at_100_max": -0.20041496084724292,
"nauc_recall_at_100_std": -0.05897637081303231,
"nauc_recall_at_10_diff1": 0.0727512723324577,
"nauc_recall_at_10_max": -0.1427961376543279,
"nauc_recall_at_10_std": -0.11534315061842598,
"nauc_recall_at_1_diff1": 0.1637549340679917,
"nauc_recall_at_1_max": -0.06866843294505832,
"nauc_recall_at_1_std": -0.11623156565719857,
"nauc_recall_at_20_diff1": 0.059287870266119605,
"nauc_recall_at_20_max": -0.17129862954689018,
"nauc_recall_at_20_std": -0.11542140579282577,
"nauc_recall_at_3_diff1": 0.09613544495372225,
"nauc_recall_at_3_max": -0.10552411799048216,
"nauc_recall_at_3_std": -0.13081876562122424,
"nauc_recall_at_5_diff1": 0.08113003762777533,
"nauc_recall_at_5_max": -0.08527591607021426,
"nauc_recall_at_5_std": -0.08915421346354895,
"ndcg_at_1": 0.08276,
"ndcg_at_10": 0.16567,
"ndcg_at_100": 0.2226,
"ndcg_at_1000": 0.25982,
"ndcg_at_20": 0.18539,
"ndcg_at_3": 0.1232,
"ndcg_at_5": 0.14417,
"precision_at_1": 0.08276,
"precision_at_10": 0.02713,
"precision_at_100": 0.00552,
"precision_at_1000": 0.00086,
"precision_at_20": 0.01745,
"precision_at_3": 0.05119,
"precision_at_5": 0.04096,
"recall_at_1": 0.08276,
"recall_at_10": 0.27133,
"recall_at_100": 0.55205,
"recall_at_1000": 0.85666,
"recall_at_20": 0.34898,
"recall_at_3": 0.15358,
"recall_at_5": 0.20478
}
]
},
"task_name": "ARCChallenge"
}
158 changes: 158 additions & 0 deletions results/GritLM__GritLM-7B-noinstruct/13f00a0e36500c80ce12870ea513846a066004af/AlphaNLI.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "303f40ef3d50918d3dc43577d33f2f7344ad72c1",
"evaluation_time": 1270.652179479599,
"kg_co2_emissions": null,
"mteb_version": "1.12.27",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.29556,
"map_at_1": 0.19582,
"map_at_10": 0.26009,
"map_at_100": 0.26884,
"map_at_1000": 0.26961,
"map_at_20": 0.26455,
"map_at_3": 0.24086,
"map_at_5": 0.25173,
"mrr_at_1": 0.195822454308094,
"mrr_at_10": 0.2600939222512328,
"mrr_at_100": 0.26883610136872893,
"mrr_at_1000": 0.26961346115632173,
"mrr_at_20": 0.2645490625655516,
"mrr_at_3": 0.24086161879895546,
"mrr_at_5": 0.2517297650130545,
"nauc_map_at_1000_diff1": 0.3581262993761119,
"nauc_map_at_1000_max": 0.18737882517084573,
"nauc_map_at_1000_std": -0.06452508289227181,
"nauc_map_at_100_diff1": 0.3580396187920006,
"nauc_map_at_100_max": 0.18715247309578215,
"nauc_map_at_100_std": -0.06467289761940416,
"nauc_map_at_10_diff1": 0.3602649101950382,
"nauc_map_at_10_max": 0.19084692411391246,
"nauc_map_at_10_std": -0.06424330205810602,
"nauc_map_at_1_diff1": 0.41684305133357324,
"nauc_map_at_1_max": 0.23999061803519373,
"nauc_map_at_1_std": -0.03497237739685838,
"nauc_map_at_20_diff1": 0.35923735124817535,
"nauc_map_at_20_max": 0.18692626854610142,
"nauc_map_at_20_std": -0.06671937446108235,
"nauc_map_at_3_diff1": 0.37685381270276136,
"nauc_map_at_3_max": 0.20715287494533172,
"nauc_map_at_3_std": -0.05590743263735323,
"nauc_map_at_5_diff1": 0.3654358413863913,
"nauc_map_at_5_max": 0.19607730137035934,
"nauc_map_at_5_std": -0.06337248072588127,
"nauc_mrr_at_1000_diff1": 0.3581262993761119,
"nauc_mrr_at_1000_max": 0.18737882517084573,
"nauc_mrr_at_1000_std": -0.06452508289227181,
"nauc_mrr_at_100_diff1": 0.3580396187920006,
"nauc_mrr_at_100_max": 0.18715247309578215,
"nauc_mrr_at_100_std": -0.06467289761940416,
"nauc_mrr_at_10_diff1": 0.3602649101950382,
"nauc_mrr_at_10_max": 0.19084692411391246,
"nauc_mrr_at_10_std": -0.06424330205810602,
"nauc_mrr_at_1_diff1": 0.41684305133357324,
"nauc_mrr_at_1_max": 0.23999061803519373,
"nauc_mrr_at_1_std": -0.03497237739685838,
"nauc_mrr_at_20_diff1": 0.35923735124817535,
"nauc_mrr_at_20_max": 0.18692626854610142,
"nauc_mrr_at_20_std": -0.06671937446108235,
"nauc_mrr_at_3_diff1": 0.37685381270276136,
"nauc_mrr_at_3_max": 0.20715287494533172,
"nauc_mrr_at_3_std": -0.05590743263735323,
"nauc_mrr_at_5_diff1": 0.3654358413863913,
"nauc_mrr_at_5_max": 0.19607730137035934,
"nauc_mrr_at_5_std": -0.06337248072588127,
"nauc_ndcg_at_1000_diff1": 0.3273834438022227,
"nauc_ndcg_at_1000_max": 0.1622409096338372,
"nauc_ndcg_at_1000_std": -0.0669388102417427,
"nauc_ndcg_at_100_diff1": 0.3223003368537524,
"nauc_ndcg_at_100_max": 0.1547146909189916,
"nauc_ndcg_at_100_std": -0.07012998864619853,
"nauc_ndcg_at_10_diff1": 0.33558897971035817,
"nauc_ndcg_at_10_max": 0.16674222031452857,
"nauc_ndcg_at_10_std": -0.07599286983639812,
"nauc_ndcg_at_1_diff1": 0.41684305133357324,
"nauc_ndcg_at_1_max": 0.23999061803519373,
"nauc_ndcg_at_1_std": -0.03497237739685838,
"nauc_ndcg_at_20_diff1": 0.3321776819933026,
"nauc_ndcg_at_20_max": 0.1526275128682169,
"nauc_ndcg_at_20_std": -0.08555865426380696,
"nauc_ndcg_at_3_diff1": 0.36622693404119916,
"nauc_ndcg_at_3_max": 0.1979519921767349,
"nauc_ndcg_at_3_std": -0.06086233490189131,
"nauc_ndcg_at_5_diff1": 0.346250107952567,
"nauc_ndcg_at_5_max": 0.17847240288665858,
"nauc_ndcg_at_5_std": -0.07384965773020805,
"nauc_precision_at_1000_diff1": 0.15722783022505635,
"nauc_precision_at_1000_max": 0.062389360128002475,
"nauc_precision_at_1000_std": -0.020194075968903987,
"nauc_precision_at_100_diff1": 0.17926769479182755,
"nauc_precision_at_100_max": 0.03764458108362797,
"nauc_precision_at_100_std": -0.06516684759990607,
"nauc_precision_at_10_diff1": 0.26861850851333074,
"nauc_precision_at_10_max": 0.09981912991436129,
"nauc_precision_at_10_std": -0.10775280987881723,
"nauc_precision_at_1_diff1": 0.41684305133357324,
"nauc_precision_at_1_max": 0.23999061803519373,
"nauc_precision_at_1_std": -0.03497237739685838,
"nauc_precision_at_20_diff1": 0.25543225738691927,
"nauc_precision_at_20_max": 0.04693182684250898,
"nauc_precision_at_20_std": -0.1445516047213592,
"nauc_precision_at_3_diff1": 0.33871394447027126,
"nauc_precision_at_3_max": 0.17398390212102777,
"nauc_precision_at_3_std": -0.07350206090918988,
"nauc_precision_at_5_diff1": 0.2946779502612536,
"nauc_precision_at_5_max": 0.1307848039211153,
"nauc_precision_at_5_std": -0.10206440369947867,
"nauc_recall_at_1000_diff1": 0.15722783022505793,
"nauc_recall_at_1000_max": 0.0623893601280026,
"nauc_recall_at_1000_std": -0.020194075968902568,
"nauc_recall_at_100_diff1": 0.17926769479182714,
"nauc_recall_at_100_max": 0.03764458108362699,
"nauc_recall_at_100_std": -0.06516684759990606,
"nauc_recall_at_10_diff1": 0.2686185085133306,
"nauc_recall_at_10_max": 0.09981912991436125,
"nauc_recall_at_10_std": -0.10775280987881715,
"nauc_recall_at_1_diff1": 0.41684305133357324,
"nauc_recall_at_1_max": 0.23999061803519373,
"nauc_recall_at_1_std": -0.03497237739685838,
"nauc_recall_at_20_diff1": 0.2554322573869195,
"nauc_recall_at_20_max": 0.046931826842508566,
"nauc_recall_at_20_std": -0.14455160472135906,
"nauc_recall_at_3_diff1": 0.3387139444702712,
"nauc_recall_at_3_max": 0.17398390212102802,
"nauc_recall_at_3_std": -0.07350206090918995,
"nauc_recall_at_5_diff1": 0.2946779502612537,
"nauc_recall_at_5_max": 0.13078480392111536,
"nauc_recall_at_5_std": -0.10206440369947854,
"ndcg_at_1": 0.19582,
"ndcg_at_10": 0.29556,
"ndcg_at_100": 0.34419,
"ndcg_at_1000": 0.36667,
"ndcg_at_20": 0.31208,
"ndcg_at_3": 0.25545,
"ndcg_at_5": 0.27506,
"precision_at_1": 0.19582,
"precision_at_10": 0.04093,
"precision_at_100": 0.00651,
"precision_at_1000": 0.00083,
"precision_at_20": 0.02376,
"precision_at_3": 0.09922,
"precision_at_5": 0.06906,
"recall_at_1": 0.19582,
"recall_at_10": 0.40927,
"recall_at_100": 0.65078,
"recall_at_1000": 0.83159,
"recall_at_20": 0.4752,
"recall_at_3": 0.29765,
"recall_at_5": 0.3453
}
]
},
"task_name": "AlphaNLI"
}
Loading

0 comments on commit bf3cf9b

Please sign in to comment.