new ml mmlu templates

prepare/templates/qa/multiple_choice/templates.py

@@ -1,3 +1,5 @@
+import pandas as pd
+
 from src.unitxt.catalog import add_to_catalog
 from src.unitxt.templates import MultipleChoiceTemplate, TemplatesList
 
@@ -6,11 +8,10 @@
         "mmlu": {
             "en": "The following are multiple choice questions (with answers) about {topic}.\n{question}.\nAnswers: \n{choices}.\nAnswer:",
             "ja": "次は {topic}に関する選択式の問題です。\n{question}.\n選択肢: {choices}.\n答え:",
-            "pt": "A seguir estão perguntas de múltipla escolha (com respostas) sobre {topic}.\n\{question}.\nRespostas: \n{choices}.\nResposta:",
+            "pt": "A seguir estão perguntas de múltipla escolha (com respostas) sobre {topic}.\n\\{question}.\nRespostas: \n{choices}.\nResposta:",
             "es": "Las siguientes son preguntas de opción múltiple (con respuestas) sobre {topic}.\n{question}.\nRespuestas: \n{choices}.\nRespuesta:",
             "fr": "Ce qui suit sont des questions à choix multiples (avec réponses) concernant {topic}.\n{question}.\nRéponses \n{choices}.\nRéponse:",
             "de": "Das folgende sind mehrfache auswahlfragen (mit antworten) bezueglich {topic}.\n\n{question}.\nAatworten: {choices}.\nAatwort:",
-
         },
         "helm": {
             "en": "The following are multiple choice questions (with answers) about {topic}.\n\nQuestion: {question}.\nAnswers: \n{choices}.\nAnswer:",
@@ -27,18 +28,17 @@
             "es": "Las siguientes son preguntas de opción múltiple (con respuestas) sobre {topic}.\n\n{question}.\n{choices}.\nRespuesta:",
             "fr": "Ce qui suit sont des questions à choix multiples (avec réponses) concernant {topic}.\n\n{question}.\n{choices}.\nRéponse:",
             "de": "Das folgende sind mehrfache auswahlfragen (mit antworten) bezueglich {topic}.\n\n{question}.\n{choices}.\nAatwort:",
-
         },
     },
-    "no_intro":{
+    "no_intro": {
         "helm": {
             "en": "Question: {question}.\nAnswers: \n{choices}.\nAnswer:",
             "ja": "質問: {question}.\n選択肢: \n{choices}.\n答え:",
             "pt": "Pergunta: {question}.\nRespostas: \n{choices}.\nResposta:",
             "es": "Pregunta: {question}.\nRespuestas: \n{choices}.\nRespuesta:",
             "fr": "Question: {question}.\nRéponses \n{choices}.\nRéponse:",
             "de": "Frage: {question}.\nAatworten: {choices}.\nAatwort:",
-        },  
+        },
         "lm_eval_harness": {
             "en": "{question}\n{choices}\nAnswer:",
             "ja": "{question}\n{choices}\n答え:",
@@ -49,13 +49,12 @@
         },
         "mmlu": {
             "en": "{question}.\nAnswers: \n{choices}.\nAnswer:",
-            "ja":"{question}.\n選択肢: \n{choices}.\n答え:",
+            "ja": "{question}.\n選択肢: \n{choices}.\n答え:",
             "pt": "{question}.\nResposta: \n{choices}.\nResposta:",
             "es": "{question}.\nRespuestas: \n{choices}.\nRespuesta:",
             "fr": "{question}.\nRéponses \n{choices}.\nRéponse:",
             "de": "{question}.\nAatworten: {choices}.\nAatwort:",
         },
-
     },
     "context_no_intro": {
         "helm": {
@@ -65,8 +64,7 @@
             "es": "Contexto: {context}\nPregunta: {question}.\nRespuestas: \n{choices}.\nRespuesta:",
             "fr": "Contexte: {context}\nQuestion: {question}.\nRéponses \n{choices}.\nRéponse:",
             "de": "Zusammenhang: {context}\nFrage: {question}.\nAatworten: {choices}.\nAatwort:",
-
-        },  
+        },
         "mmlu": {
             "en": "{context}\n{question}.\nAnswers: \n{choices}.\nAnswer:",
             "ja": "{context}\n{question}.\n選択肢: \n{choices}.\n答え:",
@@ -111,6 +109,7 @@
         },
     },
 }
+template_handels = []
 
 for template_type, template_type_groups in templates.items():
     for bechmark_name, template_groups in template_type_groups.items():
@@ -124,12 +123,36 @@
                 else "{choice_numeral}",
                 postprocessors=["processors.first_character"],
             )
-            template_handle = f"templates.qa.multiple_choice.{template_type}.{bechmark_name}.{language}"
-            # .replace(
-            #     ".en", ""
-            # )
+            template_handle = f"templates.qa.multiple_choice.{template_type}.{language}.{bechmark_name}".replace(
+                ".en", ""
+            )
             add_to_catalog(
                 template,
                 template_handle,
                 overwrite=True,
             )
+
+            template_handels.append(
+                {
+                    "handle": template_handle,
+                    "template_type": template_type,
+                    "language": language,
+                }
+            )
+
+template_handels = pd.DataFrame(template_handels)
+for template_type in template_handels.template_type.unique():
+    for lang in template_handels.language.unique():
+        template_handle_list = template_handels.query(
+            f'language=="{lang}" and template_type=="{template_type}"'
+        ).handle.tolist()
+
+        cur_handle = ".".join(template_handle_list[0].split(".")[:-1]) + ".all"
+
+        add_to_catalog(
+            artifact=TemplatesList(
+                template_handle_list,
+            ),
+            name=cur_handle,
+            overwrite=True,
+        )

src/unitxt/catalog/templates/qa/multiple_choice/context/all.json

@@ -2,6 +2,7 @@
     "type": "templates_list",
     "items": [
         "templates.qa.multiple_choice.context.mmlu",
-        "templates.qa.multiple_choice.context.helm"
+        "templates.qa.multiple_choice.context.helm",
+        "templates.qa.multiple_choice.context.lm_eval_harness"
     ]
 }

src/unitxt/catalog/templates/qa/multiple_choice/context/de/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context.de.mmlu",
+        "templates.qa.multiple_choice.context.de.helm",
+        "templates.qa.multiple_choice.context.de.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context/es/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context.es.mmlu",
+        "templates.qa.multiple_choice.context.es.helm",
+        "templates.qa.multiple_choice.context.es.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context/fr/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context.fr.mmlu",
+        "templates.qa.multiple_choice.context.fr.helm",
+        "templates.qa.multiple_choice.context.fr.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context/helm.json

@@ -3,6 +3,7 @@
     "input_format": "The following are multiple choice questions (with answers) about {topic}.\n\nContext: {context}\nQuestion: {question}.\nAnswers: \n{choices}.\nAnswer:",
     "target_field": "answer",
     "choices_seperator": "\n",
+    "target_choice_format": "{choice_numeral}",
     "postprocessors": [
         "processors.first_character"
     ]

src/unitxt/catalog/templates/qa/multiple_choice/context/ja/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context.ja.mmlu",
+        "templates.qa.multiple_choice.context.ja.helm",
+        "templates.qa.multiple_choice.context.ja.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context/lm_eval_harness.json

@@ -3,6 +3,7 @@
     "input_format": "The following are multiple choice questions (with answers) about {topic}.\n\n{context}\n{question}\n{choices}\nAnswer:",
     "target_field": "answer",
     "choices_seperator": "\n",
+    "target_choice_format": " {choice_numeral}",
     "postprocessors": [
         "processors.first_character"
     ]

src/unitxt/catalog/templates/qa/multiple_choice/context/mmlu.json

@@ -3,6 +3,7 @@
     "input_format": "The following are multiple choice questions (with answers) about {topic}.\n{context}\n{question}.\nAnswers: \n{choices}.\nAnswer:",
     "target_field": "answer",
     "choices_seperator": "\n",
+    "target_choice_format": "{choice_numeral}",
     "postprocessors": [
         "processors.first_character"
     ]

src/unitxt/catalog/templates/qa/multiple_choice/context/pt/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context.pt.mmlu",
+        "templates.qa.multiple_choice.context.pt.helm",
+        "templates.qa.multiple_choice.context.pt.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/all.json

@@ -1,7 +1,8 @@
 {
     "type": "templates_list",
     "items": [
+        "templates.qa.multiple_choice.context_no_intro.helm",
         "templates.qa.multiple_choice.context_no_intro.mmlu",
-        "templates.qa.multiple_choice.context_no_intro.helm"
+        "templates.qa.multiple_choice.context_no_intro.lm_eval_harness"
     ]
 }

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/de/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context_no_intro.de.helm",
+        "templates.qa.multiple_choice.context_no_intro.de.mmlu",
+        "templates.qa.multiple_choice.context_no_intro.de.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/es/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context_no_intro.es.helm",
+        "templates.qa.multiple_choice.context_no_intro.es.mmlu",
+        "templates.qa.multiple_choice.context_no_intro.es.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/fr/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context_no_intro.fr.helm",
+        "templates.qa.multiple_choice.context_no_intro.fr.mmlu",
+        "templates.qa.multiple_choice.context_no_intro.fr.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/helm.json

@@ -3,6 +3,7 @@
     "input_format": "Context: {context}\nQuestion: {question}.\nAnswers: \n{choices}.\nAnswer:",
     "target_field": "answer",
     "choices_seperator": "\n",
+    "target_choice_format": "{choice_numeral}",
     "postprocessors": [
         "processors.first_character"
     ]

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/ja/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context_no_intro.ja.helm",
+        "templates.qa.multiple_choice.context_no_intro.ja.mmlu",
+        "templates.qa.multiple_choice.context_no_intro.ja.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/lm_eval_harness.json

@@ -3,6 +3,7 @@
     "input_format": "{context}\n{question}\n{choices}\nAnswer:",
     "target_field": "answer",
     "choices_seperator": "\n",
+    "target_choice_format": " {choice_numeral}",
     "postprocessors": [
         "processors.first_character"
     ]

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/mmlu.json

@@ -3,6 +3,7 @@
     "input_format": "{context}\n{question}.\nAnswers: \n{choices}.\nAnswer:",
     "target_field": "answer",
     "choices_seperator": "\n",
+    "target_choice_format": "{choice_numeral}",
     "postprocessors": [
         "processors.first_character"
     ]

src/unitxt/catalog/templates/qa/multiple_choice/context_no_intro/pt/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.context_no_intro.pt.helm",
+        "templates.qa.multiple_choice.context_no_intro.pt.mmlu",
+        "templates.qa.multiple_choice.context_no_intro.pt.lm_eval_harness"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/no_intro/all.json

@@ -1,7 +1,8 @@
 {
     "type": "templates_list",
     "items": [
-        "templates.qa.multiple_choice.no_intro.mmlu",
-        "templates.qa.multiple_choice.no_intro.helm"
+        "templates.qa.multiple_choice.no_intro.helm",
+        "templates.qa.multiple_choice.no_intro.lm_eval_harness",
+        "templates.qa.multiple_choice.no_intro.mmlu"
     ]
 }

src/unitxt/catalog/templates/qa/multiple_choice/no_intro/de/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.no_intro.de.helm",
+        "templates.qa.multiple_choice.no_intro.de.lm_eval_harness",
+        "templates.qa.multiple_choice.no_intro.de.mmlu"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/no_intro/es/all.json

@@ -0,0 +1,8 @@
+{
+    "type": "templates_list",
+    "items": [
+        "templates.qa.multiple_choice.no_intro.es.helm",
+        "templates.qa.multiple_choice.no_intro.es.lm_eval_harness",
+        "templates.qa.multiple_choice.no_intro.es.mmlu"
+    ]
+}