Skip to content

Commit

Permalink
Merge main
Browse files Browse the repository at this point in the history
  • Loading branch information
@Muennighoff
Muennighoff committed Sep 11, 2024
2 parents 082fef7 + b7f3700 commit 8cf1d10
Show file tree
Hide file tree
Showing 17 changed files with 2,320 additions and 0 deletions.
16 changes: 16 additions & 0 deletions paths.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -814,9 +814,11 @@
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MLSUMClusteringS2S.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MLSUMClusteringP2P.v2.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClusteringP2P.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLReranking.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/VieMedEVBitextMining.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/FloresBitextMining.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSciBenchOECDClusteringP2P.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/CEDRClassification.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MLSUMClusteringP2P.json",
"results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/OpusparcusPC.json",
Expand Down Expand Up @@ -6558,8 +6560,10 @@
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MLSUMClusteringS2S.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MLSUMClusteringP2P.v2.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/GeoreviewClusteringP2P.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLReranking.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/FloresBitextMining.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/RuSciBenchOECDClusteringP2P.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLRetrieval.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/CEDRClassification.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MLSUMClusteringP2P.json",
"results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/OpusparcusPC.json",
Expand Down Expand Up @@ -8276,8 +8280,10 @@
"USER-bge-m3": [
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MLSUMClusteringP2P.v2.json",
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/GeoreviewClusteringP2P.json",
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLReranking.json",
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/FloresBitextMining.json",
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/RuSciBenchOECDClusteringP2P.json",
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLRetrieval.json",
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/CEDRClassification.json",
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/OpusparcusPC.json",
"results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MultilingualSentimentClassification.json",
Expand Down Expand Up @@ -9780,8 +9786,10 @@
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MLSUMClusteringS2S.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MLSUMClusteringP2P.v2.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/GeoreviewClusteringP2P.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLReranking.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/FloresBitextMining.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/RuSciBenchOECDClusteringP2P.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLRetrieval.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/CEDRClassification.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MLSUMClusteringP2P.json",
"results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/OpusparcusPC.json",
Expand Down Expand Up @@ -10846,8 +10854,10 @@
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MLSUMClusteringS2S.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MLSUMClusteringP2P.v2.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/GeoreviewClusteringP2P.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLReranking.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/FloresBitextMining.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/RuSciBenchOECDClusteringP2P.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLRetrieval.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/CEDRClassification.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MLSUMClusteringP2P.json",
"results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/OpusparcusPC.json",
Expand Down Expand Up @@ -11997,6 +12007,7 @@
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/LearnedHandsEstatesLegalBenchClassification.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/CUADExpirationDateLegalBenchClassification.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/GeoreviewClusteringP2P.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLReranking.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/CUADIPOwnershipAssignmentLegalBenchClassification.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/ContractNLISharingWithThirdPartiesLegalBenchClassification.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/Diversity2LegalBenchClassification.json",
Expand All @@ -12016,6 +12027,7 @@
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/SCDDTrainingLegalBenchClassification.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RuSciBenchOECDClusteringP2P.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/CUADWarrantyDurationLegalBenchClassification.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLRetrieval.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/StackExchangeClustering.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/CUADCapOnLiabilityLegalBenchClassification.json",
"results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/CUADMostFavoredNationLegalBenchClassification.json",
Expand Down Expand Up @@ -12302,8 +12314,10 @@
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MLSUMClusteringS2S.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MLSUMClusteringP2P.v2.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/GeoreviewClusteringP2P.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLReranking.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/FloresBitextMining.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/RuSciBenchOECDClusteringP2P.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLRetrieval.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/CEDRClassification.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MLSUMClusteringP2P.json",
"results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/OpusparcusPC.json",
Expand Down Expand Up @@ -14736,8 +14750,10 @@
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MLSUMClusteringS2S.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MLSUMClusteringP2P.v2.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/GeoreviewClusteringP2P.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLReranking.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/FloresBitextMining.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSciBenchOECDClusteringP2P.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLRetrieval.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/CEDRClassification.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MLSUMClusteringP2P.json",
"results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/OpusparcusPC.json",
Expand Down
130 changes: 130 additions & 0 deletions ...-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLReranking.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
{
"dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af",
"evaluation_time": 102.19886922836304,
"kg_co2_emissions": null,
"mteb_version": "1.12.94",
"scores": {
"dev": [
{
"MAP@1(MIRACL)": 0.03793,
"MAP@10(MIRACL)": 0.08492,
"MAP@100(MIRACL)": 0.11854,
"MAP@1000(MIRACL)": 0.11854,
"MAP@20(MIRACL)": 0.09699,
"MAP@3(MIRACL)": 0.0599,
"MAP@5(MIRACL)": 0.07087,
"NDCG@1(MIRACL)": 0.06576,
"NDCG@10(MIRACL)": 0.13094,
"NDCG@100(MIRACL)": 0.27866,
"NDCG@1000(MIRACL)": 0.27866,
"NDCG@20(MIRACL)": 0.16843,
"NDCG@3(MIRACL)": 0.08274,
"NDCG@5(MIRACL)": 0.09898,
"P@1(MIRACL)": 0.06576,
"P@10(MIRACL)": 0.04707,
"P@100(MIRACL)": 0.01953,
"P@1000(MIRACL)": 0.00195,
"P@20(MIRACL)": 0.03773,
"P@3(MIRACL)": 0.05801,
"P@5(MIRACL)": 0.05277,
"Recall@1(MIRACL)": 0.03793,
"Recall@10(MIRACL)": 0.20941,
"Recall@100(MIRACL)": 0.79952,
"Recall@1000(MIRACL)": 0.79952,
"Recall@20(MIRACL)": 0.32753,
"Recall@3(MIRACL)": 0.08533,
"Recall@5(MIRACL)": 0.12619,
"hf_subset": "ru",
"languages": [
"rus-Cyrl"
],
"main_score": 0.13094,
"nAUC_MAP@1000_diff1(MIRACL)": 0.11656879622209783,
"nAUC_MAP@1000_max(MIRACL)": -0.02184173429173281,
"nAUC_MAP@1000_std(MIRACL)": -0.0005910451991668005,
"nAUC_MAP@100_diff1(MIRACL)": 0.11656879622209783,
"nAUC_MAP@100_max(MIRACL)": -0.02184173429173281,
"nAUC_MAP@100_std(MIRACL)": -0.0005910451991668005,
"nAUC_MAP@10_diff1(MIRACL)": 0.12820268321122477,
"nAUC_MAP@10_max(MIRACL)": -0.006007170516302484,
"nAUC_MAP@10_std(MIRACL)": 0.004887023887105919,
"nAUC_MAP@1_diff1(MIRACL)": 0.19979024924352953,
"nAUC_MAP@1_max(MIRACL)": 0.0965067328936865,
"nAUC_MAP@1_std(MIRACL)": -0.01469295152548763,
"nAUC_MAP@20_diff1(MIRACL)": 0.11943138095364866,
"nAUC_MAP@20_max(MIRACL)": -0.014742898115901397,
"nAUC_MAP@20_std(MIRACL)": 0.007627922730200203,
"nAUC_MAP@3_diff1(MIRACL)": 0.16247267806292065,
"nAUC_MAP@3_max(MIRACL)": 0.01916771582088975,
"nAUC_MAP@3_std(MIRACL)": -0.024768207978978725,
"nAUC_MAP@5_diff1(MIRACL)": 0.13153704499454139,
"nAUC_MAP@5_max(MIRACL)": 0.007096539434435867,
"nAUC_MAP@5_std(MIRACL)": -0.013659018365901245,
"nAUC_NDCG@1000_diff1(MIRACL)": 0.09046593862073116,
"nAUC_NDCG@1000_max(MIRACL)": -0.049656945308565785,
"nAUC_NDCG@1000_std(MIRACL)": -0.02852729085649775,
"nAUC_NDCG@100_diff1(MIRACL)": 0.09046593862073116,
"nAUC_NDCG@100_max(MIRACL)": -0.049656945308565785,
"nAUC_NDCG@100_std(MIRACL)": -0.02852729085649775,
"nAUC_NDCG@10_diff1(MIRACL)": 0.10077472820623631,
"nAUC_NDCG@10_max(MIRACL)": -0.033319860687037434,
"nAUC_NDCG@10_std(MIRACL)": 0.008223662807163275,
"nAUC_NDCG@1_diff1(MIRACL)": 0.151186819507912,
"nAUC_NDCG@1_max(MIRACL)": 0.11537026630871994,
"nAUC_NDCG@1_std(MIRACL)": -0.045578082017163145,
"nAUC_NDCG@20_diff1(MIRACL)": 0.08640831332839452,
"nAUC_NDCG@20_max(MIRACL)": -0.05269863130012071,
"nAUC_NDCG@20_std(MIRACL)": 0.007783868824801839,
"nAUC_NDCG@3_diff1(MIRACL)": 0.13178318008309772,
"nAUC_NDCG@3_max(MIRACL)": 0.0035297506848497,
"nAUC_NDCG@3_std(MIRACL)": -0.042422835970525714,
"nAUC_NDCG@5_diff1(MIRACL)": 0.10348566598107344,
"nAUC_NDCG@5_max(MIRACL)": -0.015154481875360658,
"nAUC_NDCG@5_std(MIRACL)": -0.02018522684348813,
"nAUC_P@1000_diff1(MIRACL)": 0.006436408505254566,
"nAUC_P@1000_max(MIRACL)": -0.062365784452251265,
"nAUC_P@1000_std(MIRACL)": -0.06679081786133326,
"nAUC_P@100_diff1(MIRACL)": 0.006436408505254932,
"nAUC_P@100_max(MIRACL)": -0.06236578445225068,
"nAUC_P@100_std(MIRACL)": -0.06679081786133294,
"nAUC_P@10_diff1(MIRACL)": 0.035924513017525805,
"nAUC_P@10_max(MIRACL)": -0.07407327390595522,
"nAUC_P@10_std(MIRACL)": 0.011824346308663887,
"nAUC_P@1_diff1(MIRACL)": 0.151186819507912,
"nAUC_P@1_max(MIRACL)": 0.11537026630871994,
"nAUC_P@1_std(MIRACL)": -0.045578082017163145,
"nAUC_P@20_diff1(MIRACL)": 0.020405546456346138,
"nAUC_P@20_max(MIRACL)": -0.09600408910657676,
"nAUC_P@20_std(MIRACL)": -0.0001021731695539199,
"nAUC_P@3_diff1(MIRACL)": 0.08187396234458519,
"nAUC_P@3_max(MIRACL)": -0.02981991700564444,
"nAUC_P@3_std(MIRACL)": -0.04315477063398759,
"nAUC_P@5_diff1(MIRACL)": 0.029326475776068565,
"nAUC_P@5_max(MIRACL)": -0.05538591644280579,
"nAUC_P@5_std(MIRACL)": -0.013627413052371293,
"nAUC_Recall@1000_diff1(MIRACL)": 0.0626568052254435,
"nAUC_Recall@1000_max(MIRACL)": -0.056351555553082765,
"nAUC_Recall@1000_std(MIRACL)": -0.07720527712702241,
"nAUC_Recall@100_diff1(MIRACL)": 0.0626568052254435,
"nAUC_Recall@100_max(MIRACL)": -0.056351555553082765,
"nAUC_Recall@100_std(MIRACL)": -0.07720527712702241,
"nAUC_Recall@10_diff1(MIRACL)": 0.0699596207061801,
"nAUC_Recall@10_max(MIRACL)": -0.06623123931077808,
"nAUC_Recall@10_std(MIRACL)": 0.04314390563870134,
"nAUC_Recall@1_diff1(MIRACL)": 0.19979024924352953,
"nAUC_Recall@1_max(MIRACL)": 0.0965067328936865,
"nAUC_Recall@1_std(MIRACL)": -0.01469295152548763,
"nAUC_Recall@20_diff1(MIRACL)": 0.04151985357457182,
"nAUC_Recall@20_max(MIRACL)": -0.09346206581673573,
"nAUC_Recall@20_std(MIRACL)": 0.03867752916801929,
"nAUC_Recall@3_diff1(MIRACL)": 0.15002227895926992,
"nAUC_Recall@3_max(MIRACL)": -0.03085604524638985,
"nAUC_Recall@3_std(MIRACL)": -0.04078077585553242,
"nAUC_Recall@5_diff1(MIRACL)": 0.08035497411159716,
"nAUC_Recall@5_max(MIRACL)": -0.053504090706605265,
"nAUC_Recall@5_std(MIRACL)": -0.009017940713406399
}
]
},
"task_name": "MIRACLReranking"
}
Loading

0 comments on commit 8cf1d10

Please sign in to comment.