diff --git "a/evaluation/ar/openaimmlu_0_shot.json" "b/evaluation/ar/openaimmlu_0_shot.json" new file mode 100644--- /dev/null +++ "b/evaluation/ar/openaimmlu_0_shot.json" @@ -0,0 +1,2707 @@ +{ + "results": { + "openaimmlu": { + " ": " ", + "alias": "openaimmlu" + }, + "openaimmlu_STEM": { + "acc,none": 0.4900662251655629, + "acc_stderr,none": 0.00883192107765626, + "alias": " - STEM" + }, + "openaimmlu_abstract_algebra": { + "alias": " - abstract_algebra", + "acc,none": 0.25, + "acc_stderr,none": 0.04351941398892446 + }, + "openaimmlu_astronomy": { + "alias": " - astronomy", + "acc,none": 0.6842105263157895, + "acc_stderr,none": 0.037827289808654685 + }, + "openaimmlu_college_biology": { + "alias": " - college_biology", + "acc,none": 0.6597222222222222, + "acc_stderr,none": 0.039621355734862175 + }, + "openaimmlu_college_chemistry": { + "alias": " - college_chemistry", + "acc,none": 0.35, + "acc_stderr,none": 0.047937248544110196 + }, + "openaimmlu_college_computer_science": { + "alias": " - college_computer_science", + "acc,none": 0.44, + "acc_stderr,none": 0.04988876515698589 + }, + "openaimmlu_college_mathematics": { + "alias": " - college_mathematics", + "acc,none": 0.3, + "acc_stderr,none": 0.046056618647183814 + }, + "openaimmlu_college_physics": { + "alias": " - college_physics", + "acc,none": 0.37254901960784315, + "acc_stderr,none": 0.04810840148082633 + }, + "openaimmlu_computer_security": { + "alias": " - computer_security", + "acc,none": 0.71, + "acc_stderr,none": 0.045604802157206845 + }, + "openaimmlu_conceptual_physics": { + "alias": " - conceptual_physics", + "acc,none": 0.548936170212766, + "acc_stderr,none": 0.032529096196131965 + }, + "openaimmlu_econometrics": { + "alias": " - econometrics", + "acc,none": 0.3684210526315789, + "acc_stderr,none": 0.04537815354939391 + }, + "openaimmlu_electrical_engineering": { + "alias": " - electrical_engineering", + "acc,none": 0.5103448275862069, + "acc_stderr,none": 0.04165774775728763 + }, + "openaimmlu_elementary_mathematics": { + "alias": " - elementary_mathematics", + "acc,none": 0.48677248677248675, + "acc_stderr,none": 0.025742297289575142 + }, + "openaimmlu_high_school_biology": { + "alias": " - high_school_biology", + "acc,none": 0.6645161290322581, + "acc_stderr,none": 0.026860206444724352 + }, + "openaimmlu_high_school_chemistry": { + "alias": " - high_school_chemistry", + "acc,none": 0.4630541871921182, + "acc_stderr,none": 0.035083705204426656 + }, + "openaimmlu_high_school_computer_science": { + "alias": " - high_school_computer_science", + "acc,none": 0.56, + "acc_stderr,none": 0.04988876515698589 + }, + "openaimmlu_high_school_mathematics": { + "alias": " - high_school_mathematics", + "acc,none": 0.35185185185185186, + "acc_stderr,none": 0.02911661760608301 + }, + "openaimmlu_high_school_physics": { + "alias": " - high_school_physics", + "acc,none": 0.37748344370860926, + "acc_stderr,none": 0.039580272311215706 + }, + "openaimmlu_high_school_statistics": { + "alias": " - high_school_statistics", + "acc,none": 0.4675925925925926, + "acc_stderr,none": 0.03402801581358966 + }, + "openaimmlu_humanities": { + "acc,none": 0.6834811529933481, + "acc_stderr,none": 0.01087157296938379, + "alias": " - Humanities" + }, + "openaimmlu_high_school_european_history": { + "alias": " - high_school_european_history", + "acc,none": 0.7333333333333333, + "acc_stderr,none": 0.03453131801885417 + }, + "openaimmlu_high_school_us_history": { + "alias": " - high_school_us_history", + "acc,none": 0.7254901960784313, + "acc_stderr,none": 0.03132179803083291 + }, + "openaimmlu_high_school_world_history": { + "alias": " - high_school_world_history", + "acc,none": 0.7721518987341772, + "acc_stderr,none": 0.027303484599069415 + }, + "openaimmlu_international_law": { + "alias": " - international_law", + "acc,none": 0.7355371900826446, + "acc_stderr,none": 0.04026187527591205 + }, + "openaimmlu_jurisprudence": { + "alias": " - jurisprudence", + "acc,none": 0.6851851851851852, + "acc_stderr,none": 0.04489931073591311 + }, + "openaimmlu_logical_fallacies": { + "alias": " - logical_fallacies", + "acc,none": 0.6871165644171779, + "acc_stderr,none": 0.03642914578292404 + }, + "openaimmlu_philosophy": { + "alias": " - philosophy", + "acc,none": 0.6077170418006431, + "acc_stderr,none": 0.027731258647011987 + }, + "openaimmlu_prehistory": { + "alias": " - prehistory", + "acc,none": 0.595679012345679, + "acc_stderr,none": 0.027306625297327698 + }, + "openaimmlu_world_religions": { + "alias": " - world_religions", + "acc,none": 0.7251461988304093, + "acc_stderr,none": 0.034240429246915824 + }, + "openaimmlu_other": { + "acc,none": 0.5571476736345247, + "acc_stderr,none": 0.0062200183711956835, + "alias": " - Other" + }, + "openaimmlu_anatomy": { + "alias": " - anatomy", + "acc,none": 0.4740740740740741, + "acc_stderr,none": 0.04313531696750575 + }, + "openaimmlu_clinical_knowledge": { + "alias": " - clinical_knowledge", + "acc,none": 0.5773584905660377, + "acc_stderr,none": 0.030402331445769537 + }, + "openaimmlu_college_medicine": { + "alias": " - college_medicine", + "acc,none": 0.5086705202312138, + "acc_stderr,none": 0.0381189098894041 + }, + "openaimmlu_formal_logic": { + "alias": " - formal_logic", + "acc,none": 0.3888888888888889, + "acc_stderr,none": 0.04360314860077459 + }, + "openaimmlu_global_facts": { + "alias": " - global_facts", + "acc,none": 0.4, + "acc_stderr,none": 0.049236596391733084 + }, + "openaimmlu_high_school_geography": { + "alias": " - high_school_geography", + "acc,none": 0.7121212121212122, + "acc_stderr,none": 0.03225883512300992 + }, + "openaimmlu_high_school_psychology": { + "alias": " - high_school_psychology", + "acc,none": 0.7302752293577982, + "acc_stderr,none": 0.01902848671111545 + }, + "openaimmlu_human_aging": { + "alias": " - human_aging", + "acc,none": 0.6278026905829597, + "acc_stderr,none": 0.0324430528300873 + }, + "openaimmlu_machine_learning": { + "alias": " - machine_learning", + "acc,none": 0.41964285714285715, + "acc_stderr,none": 0.04684099321077106 + }, + "openaimmlu_medical_genetics": { + "alias": " - medical_genetics", + "acc,none": 0.66, + "acc_stderr,none": 0.04760952285695237 + }, + "openaimmlu_miscellaneous": { + "alias": " - miscellaneous", + "acc,none": 0.7573435504469987, + "acc_stderr,none": 0.015329888940899873 + }, + "openaimmlu_nutrition": { + "alias": " - nutrition", + "acc,none": 0.6601307189542484, + "acc_stderr,none": 0.027121956071388856 + }, + "openaimmlu_professional_accounting": { + "alias": " - professional_accounting", + "acc,none": 0.41843971631205673, + "acc_stderr,none": 0.029427994039419994 + }, + "openaimmlu_professional_law": { + "alias": " - professional_law", + "acc,none": 0.41264667535853977, + "acc_stderr,none": 0.012573836633799016 + }, + "openaimmlu_professional_medicine": { + "alias": " - professional_medicine", + "acc,none": 0.5735294117647058, + "acc_stderr,none": 0.030042615832714857 + }, + "openaimmlu_professional_psychology": { + "alias": " - professional_psychology", + "acc,none": 0.5522875816993464, + "acc_stderr,none": 0.020116925347422425 + }, + "openaimmlu_virology": { + "alias": " - virology", + "acc,none": 0.4759036144578313, + "acc_stderr,none": 0.03887971849597264 + }, + "openaimmlu_social_science": { + "acc,none": 0.5578210590383444, + "acc_stderr,none": 0.008094265116110859, + "alias": " - Social Science" + }, + "openaimmlu_business_ethics": { + "alias": " - business_ethics", + "acc,none": 0.67, + "acc_stderr,none": 0.04725815626252609 + }, + "openaimmlu_high_school_government_and_politics": { + "alias": " - high_school_government_and_politics", + "acc,none": 0.772020725388601, + "acc_stderr,none": 0.03027690994517826 + }, + "openaimmlu_high_school_macroeconomics": { + "alias": " - high_school_macroeconomics", + "acc,none": 0.5692307692307692, + "acc_stderr,none": 0.025106820660539753 + }, + "openaimmlu_high_school_microeconomics": { + "alias": " - high_school_microeconomics", + "acc,none": 0.5756302521008403, + "acc_stderr,none": 0.03210479051015776 + }, + "openaimmlu_human_sexuality": { + "alias": " - human_sexuality", + "acc,none": 0.6641221374045801, + "acc_stderr,none": 0.04142313771996664 + }, + "openaimmlu_management": { + "alias": " - management", + "acc,none": 0.7281553398058253, + "acc_stderr,none": 0.044052680241409216 + }, + "openaimmlu_marketing": { + "alias": " - marketing", + "acc,none": 0.8076923076923077, + "acc_stderr,none": 0.025819233256483727 + }, + "openaimmlu_moral_disputes": { + "alias": " - moral_disputes", + "acc,none": 0.5751445086705202, + "acc_stderr,none": 0.026613350840261746 + }, + "openaimmlu_moral_scenarios": { + "alias": " - moral_scenarios", + "acc,none": 0.2916201117318436, + "acc_stderr,none": 0.015201032512520442 + }, + "openaimmlu_public_relations": { + "alias": " - public_relations", + "acc,none": 0.5727272727272728, + "acc_stderr,none": 0.047381987035454834 + }, + "openaimmlu_security_studies": { + "alias": " - security_studies", + "acc,none": 0.6693877551020408, + "acc_stderr,none": 0.030116426296540603 + }, + "openaimmlu_sociology": { + "alias": " - sociology", + "acc,none": 0.6915422885572139, + "acc_stderr,none": 0.032658195885126966 + }, + "openaimmlu_us_foreign_policy": { + "alias": " - us_foreign_policy", + "acc,none": 0.81, + "acc_stderr,none": 0.039427724440366234 + } + }, + "groups": { + "openaimmlu_STEM": { + "acc,none": 0.4900662251655629, + "acc_stderr,none": 0.00883192107765626, + "alias": " - STEM" + }, + "openaimmlu_humanities": { + "acc,none": 0.6834811529933481, + "acc_stderr,none": 0.01087157296938379, + "alias": " - Humanities" + }, + "openaimmlu_other": { + "acc,none": 0.5571476736345247, + "acc_stderr,none": 0.0062200183711956835, + "alias": " - Other" + }, + "openaimmlu_social_science": { + "acc,none": 0.5578210590383444, + "acc_stderr,none": 0.008094265116110859, + "alias": " - Social Science" + } + }, + "group_subtasks": { + "openaimmlu_humanities": [ + "openaimmlu_jurisprudence", + "openaimmlu_logical_fallacies", + "openaimmlu_philosophy", + "openaimmlu_high_school_world_history", + "openaimmlu_high_school_european_history", + "openaimmlu_prehistory", + "openaimmlu_high_school_us_history", + "openaimmlu_international_law", + "openaimmlu_world_religions" + ], + "openaimmlu_social_science": [ + "openaimmlu_high_school_microeconomics", + "openaimmlu_high_school_government_and_politics", + "openaimmlu_management", + "openaimmlu_security_studies", + "openaimmlu_business_ethics", + "openaimmlu_sociology", + "openaimmlu_high_school_macroeconomics", + "openaimmlu_moral_scenarios", + "openaimmlu_public_relations", + "openaimmlu_us_foreign_policy", + "openaimmlu_moral_disputes", + "openaimmlu_human_sexuality", + "openaimmlu_marketing" + ], + "openaimmlu_other": [ + "openaimmlu_nutrition", + "openaimmlu_miscellaneous", + "openaimmlu_anatomy", + "openaimmlu_virology", + "openaimmlu_professional_medicine", + "openaimmlu_human_aging", + "openaimmlu_clinical_knowledge", + "openaimmlu_professional_accounting", + "openaimmlu_high_school_geography", + "openaimmlu_professional_psychology", + "openaimmlu_high_school_psychology", + "openaimmlu_machine_learning", + "openaimmlu_medical_genetics", + "openaimmlu_professional_law", + "openaimmlu_college_medicine", + "openaimmlu_formal_logic", + "openaimmlu_global_facts" + ], + "openaimmlu_STEM": [ + "openaimmlu_college_physics", + "openaimmlu_astronomy", + "openaimmlu_computer_security", + "openaimmlu_elementary_mathematics", + "openaimmlu_high_school_chemistry", + "openaimmlu_college_mathematics", + "openaimmlu_college_chemistry", + "openaimmlu_college_biology", + "openaimmlu_conceptual_physics", + "openaimmlu_high_school_statistics", + "openaimmlu_electrical_engineering", + "openaimmlu_high_school_computer_science", + "openaimmlu_high_school_mathematics", + "openaimmlu_abstract_algebra", + "openaimmlu_high_school_physics", + "openaimmlu_college_computer_science", + "openaimmlu_econometrics", + "openaimmlu_high_school_biology" + ], + "openaimmlu": [ + "openaimmlu_STEM", + "openaimmlu_other", + "openaimmlu_social_science", + "openaimmlu_humanities" + ] + }, + "configs": { + "openaimmlu_abstract_algebra": { + "task": "openaimmlu_abstract_algebra", + "task_alias": "abstract_algebra", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "abstract_algebra", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_anatomy": { + "task": "openaimmlu_anatomy", + "task_alias": "anatomy", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "anatomy", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_astronomy": { + "task": "openaimmlu_astronomy", + "task_alias": "astronomy", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "astronomy", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_business_ethics": { + "task": "openaimmlu_business_ethics", + "task_alias": "business_ethics", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "business_ethics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_clinical_knowledge": { + "task": "openaimmlu_clinical_knowledge", + "task_alias": "clinical_knowledge", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "clinical_knowledge", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_college_biology": { + "task": "openaimmlu_college_biology", + "task_alias": "college_biology", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "college_biology", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_college_chemistry": { + "task": "openaimmlu_college_chemistry", + "task_alias": "college_chemistry", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "college_chemistry", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_college_computer_science": { + "task": "openaimmlu_college_computer_science", + "task_alias": "college_computer_science", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "college_computer_science", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_college_mathematics": { + "task": "openaimmlu_college_mathematics", + "task_alias": "college_mathematics", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "college_mathematics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_college_medicine": { + "task": "openaimmlu_college_medicine", + "task_alias": "college_medicine", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "college_medicine", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_college_physics": { + "task": "openaimmlu_college_physics", + "task_alias": "college_physics", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "college_physics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_computer_security": { + "task": "openaimmlu_computer_security", + "task_alias": "computer_security", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "computer_security", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_conceptual_physics": { + "task": "openaimmlu_conceptual_physics", + "task_alias": "conceptual_physics", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "conceptual_physics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_econometrics": { + "task": "openaimmlu_econometrics", + "task_alias": "econometrics", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "econometrics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_electrical_engineering": { + "task": "openaimmlu_electrical_engineering", + "task_alias": "electrical_engineering", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "electrical_engineering", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_elementary_mathematics": { + "task": "openaimmlu_elementary_mathematics", + "task_alias": "elementary_mathematics", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "elementary_mathematics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_formal_logic": { + "task": "openaimmlu_formal_logic", + "task_alias": "formal_logic", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "formal_logic", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_global_facts": { + "task": "openaimmlu_global_facts", + "task_alias": "global_facts", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "global_facts", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_biology": { + "task": "openaimmlu_high_school_biology", + "task_alias": "high_school_biology", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_biology", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_chemistry": { + "task": "openaimmlu_high_school_chemistry", + "task_alias": "high_school_chemistry", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_chemistry", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_computer_science": { + "task": "openaimmlu_high_school_computer_science", + "task_alias": "high_school_computer_science", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_computer_science", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_european_history": { + "task": "openaimmlu_high_school_european_history", + "task_alias": "high_school_european_history", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_european_history", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_geography": { + "task": "openaimmlu_high_school_geography", + "task_alias": "high_school_geography", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_geography", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_government_and_politics": { + "task": "openaimmlu_high_school_government_and_politics", + "task_alias": "high_school_government_and_politics", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_government_and_politics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_macroeconomics": { + "task": "openaimmlu_high_school_macroeconomics", + "task_alias": "high_school_macroeconomics", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_macroeconomics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_mathematics": { + "task": "openaimmlu_high_school_mathematics", + "task_alias": "high_school_mathematics", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_mathematics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_microeconomics": { + "task": "openaimmlu_high_school_microeconomics", + "task_alias": "high_school_microeconomics", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_microeconomics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_physics": { + "task": "openaimmlu_high_school_physics", + "task_alias": "high_school_physics", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_physics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_psychology": { + "task": "openaimmlu_high_school_psychology", + "task_alias": "high_school_psychology", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_psychology", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_statistics": { + "task": "openaimmlu_high_school_statistics", + "task_alias": "high_school_statistics", + "tag": "openaimmlu_STEM_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_statistics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_us_history": { + "task": "openaimmlu_high_school_us_history", + "task_alias": "high_school_us_history", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_us_history", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_high_school_world_history": { + "task": "openaimmlu_high_school_world_history", + "task_alias": "high_school_world_history", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "high_school_world_history", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_human_aging": { + "task": "openaimmlu_human_aging", + "task_alias": "human_aging", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "human_aging", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_human_sexuality": { + "task": "openaimmlu_human_sexuality", + "task_alias": "human_sexuality", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "human_sexuality", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_international_law": { + "task": "openaimmlu_international_law", + "task_alias": "international_law", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "international_law", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_jurisprudence": { + "task": "openaimmlu_jurisprudence", + "task_alias": "jurisprudence", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "jurisprudence", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_logical_fallacies": { + "task": "openaimmlu_logical_fallacies", + "task_alias": "logical_fallacies", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "logical_fallacies", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_machine_learning": { + "task": "openaimmlu_machine_learning", + "task_alias": "machine_learning", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "machine_learning", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_management": { + "task": "openaimmlu_management", + "task_alias": "management", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "management", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_marketing": { + "task": "openaimmlu_marketing", + "task_alias": "marketing", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "marketing", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_medical_genetics": { + "task": "openaimmlu_medical_genetics", + "task_alias": "medical_genetics", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "medical_genetics", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_miscellaneous": { + "task": "openaimmlu_miscellaneous", + "task_alias": "miscellaneous", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "miscellaneous", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_moral_disputes": { + "task": "openaimmlu_moral_disputes", + "task_alias": "moral_disputes", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "moral_disputes", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_moral_scenarios": { + "task": "openaimmlu_moral_scenarios", + "task_alias": "moral_scenarios", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "moral_scenarios", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_nutrition": { + "task": "openaimmlu_nutrition", + "task_alias": "nutrition", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "nutrition", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_philosophy": { + "task": "openaimmlu_philosophy", + "task_alias": "philosophy", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "philosophy", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_prehistory": { + "task": "openaimmlu_prehistory", + "task_alias": "prehistory", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "prehistory", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_professional_accounting": { + "task": "openaimmlu_professional_accounting", + "task_alias": "professional_accounting", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "professional_accounting", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_professional_law": { + "task": "openaimmlu_professional_law", + "task_alias": "professional_law", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "professional_law", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_professional_medicine": { + "task": "openaimmlu_professional_medicine", + "task_alias": "professional_medicine", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "professional_medicine", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_professional_psychology": { + "task": "openaimmlu_professional_psychology", + "task_alias": "professional_psychology", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "professional_psychology", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_public_relations": { + "task": "openaimmlu_public_relations", + "task_alias": "public_relations", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "public_relations", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_security_studies": { + "task": "openaimmlu_security_studies", + "task_alias": "security_studies", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "security_studies", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_sociology": { + "task": "openaimmlu_sociology", + "task_alias": "sociology", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "sociology", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_us_foreign_policy": { + "task": "openaimmlu_us_foreign_policy", + "task_alias": "us_foreign_policy", + "tag": "openaimmlu_social_science_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "us_foreign_policy", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_virology": { + "task": "openaimmlu_virology", + "task_alias": "virology", + "tag": "openaimmlu_other_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "virology", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "openaimmlu_world_religions": { + "task": "openaimmlu_world_religions", + "task_alias": "world_religions", + "tag": "openaimmlu_humanities_tasks", + "dataset_path": "khalidalt/openai_mmlu_arabic", + "dataset_name": "world_religions", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_docs(doc):\n\n def format_example(doc, choices):\n options = []\n for _, choice in enumerate(choices):\n options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n ar_subject = SUBJECTS[doc['Subject']]\n query = PROMPT.format(ar_subject, #doc['Subject'],\n doc['Question'],\n \"\\n\".join(options))\n return query\n\n keys_en = [\"A\", \"B\", \"C\", \"D\"]\n keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n ar_label = en2ar[doc['Answer']]\n\n out_doc = {\n \"query\": format_example(doc, keys_en),\n \"choices\": keys_ar,\n \"gold\": keys_ar.index(ar_label)\n }\n\n return out_doc\n\n return dataset.map(_process_docs) \n", + "doc_to_text": "query", + "doc_to_target": "gold", + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + } + }, + "versions": { + "openaimmlu_STEM": 0, + "openaimmlu_abstract_algebra": 0.0, + "openaimmlu_anatomy": 0.0, + "openaimmlu_astronomy": 0.0, + "openaimmlu_business_ethics": 0.0, + "openaimmlu_clinical_knowledge": 0.0, + "openaimmlu_college_biology": 0.0, + "openaimmlu_college_chemistry": 0.0, + "openaimmlu_college_computer_science": 0.0, + "openaimmlu_college_mathematics": 0.0, + "openaimmlu_college_medicine": 0.0, + "openaimmlu_college_physics": 0.0, + "openaimmlu_computer_security": 0.0, + "openaimmlu_conceptual_physics": 0.0, + "openaimmlu_econometrics": 0.0, + "openaimmlu_electrical_engineering": 0.0, + "openaimmlu_elementary_mathematics": 0.0, + "openaimmlu_formal_logic": 0.0, + "openaimmlu_global_facts": 0.0, + "openaimmlu_high_school_biology": 0.0, + "openaimmlu_high_school_chemistry": 0.0, + "openaimmlu_high_school_computer_science": 0.0, + "openaimmlu_high_school_european_history": 0.0, + "openaimmlu_high_school_geography": 0.0, + "openaimmlu_high_school_government_and_politics": 0.0, + "openaimmlu_high_school_macroeconomics": 0.0, + "openaimmlu_high_school_mathematics": 0.0, + "openaimmlu_high_school_microeconomics": 0.0, + "openaimmlu_high_school_physics": 0.0, + "openaimmlu_high_school_psychology": 0.0, + "openaimmlu_high_school_statistics": 0.0, + "openaimmlu_high_school_us_history": 0.0, + "openaimmlu_high_school_world_history": 0.0, + "openaimmlu_human_aging": 0.0, + "openaimmlu_human_sexuality": 0.0, + "openaimmlu_humanities": 0, + "openaimmlu_international_law": 0.0, + "openaimmlu_jurisprudence": 0.0, + "openaimmlu_logical_fallacies": 0.0, + "openaimmlu_machine_learning": 0.0, + "openaimmlu_management": 0.0, + "openaimmlu_marketing": 0.0, + "openaimmlu_medical_genetics": 0.0, + "openaimmlu_miscellaneous": 0.0, + "openaimmlu_moral_disputes": 0.0, + "openaimmlu_moral_scenarios": 0.0, + "openaimmlu_nutrition": 0.0, + "openaimmlu_other": 0, + "openaimmlu_philosophy": 0.0, + "openaimmlu_prehistory": 0.0, + "openaimmlu_professional_accounting": 0.0, + "openaimmlu_professional_law": 0.0, + "openaimmlu_professional_medicine": 0.0, + "openaimmlu_professional_psychology": 0.0, + "openaimmlu_public_relations": 0.0, + "openaimmlu_security_studies": 0.0, + "openaimmlu_social_science": 0, + "openaimmlu_sociology": 0.0, + "openaimmlu_us_foreign_policy": 0.0, + "openaimmlu_virology": 0.0, + "openaimmlu_world_religions": 0.0 + }, + "n-shot": { + "openaimmlu_abstract_algebra": 0, + "openaimmlu_anatomy": 0, + "openaimmlu_astronomy": 0, + "openaimmlu_business_ethics": 0, + "openaimmlu_clinical_knowledge": 0, + "openaimmlu_college_biology": 0, + "openaimmlu_college_chemistry": 0, + "openaimmlu_college_computer_science": 0, + "openaimmlu_college_mathematics": 0, + "openaimmlu_college_medicine": 0, + "openaimmlu_college_physics": 0, + "openaimmlu_computer_security": 0, + "openaimmlu_conceptual_physics": 0, + "openaimmlu_econometrics": 0, + "openaimmlu_electrical_engineering": 0, + "openaimmlu_elementary_mathematics": 0, + "openaimmlu_formal_logic": 0, + "openaimmlu_global_facts": 0, + "openaimmlu_high_school_biology": 0, + "openaimmlu_high_school_chemistry": 0, + "openaimmlu_high_school_computer_science": 0, + "openaimmlu_high_school_european_history": 0, + "openaimmlu_high_school_geography": 0, + "openaimmlu_high_school_government_and_politics": 0, + "openaimmlu_high_school_macroeconomics": 0, + "openaimmlu_high_school_mathematics": 0, + "openaimmlu_high_school_microeconomics": 0, + "openaimmlu_high_school_physics": 0, + "openaimmlu_high_school_psychology": 0, + "openaimmlu_high_school_statistics": 0, + "openaimmlu_high_school_us_history": 0, + "openaimmlu_high_school_world_history": 0, + "openaimmlu_human_aging": 0, + "openaimmlu_human_sexuality": 0, + "openaimmlu_international_law": 0, + "openaimmlu_jurisprudence": 0, + "openaimmlu_logical_fallacies": 0, + "openaimmlu_machine_learning": 0, + "openaimmlu_management": 0, + "openaimmlu_marketing": 0, + "openaimmlu_medical_genetics": 0, + "openaimmlu_miscellaneous": 0, + "openaimmlu_moral_disputes": 0, + "openaimmlu_moral_scenarios": 0, + "openaimmlu_nutrition": 0, + "openaimmlu_philosophy": 0, + "openaimmlu_prehistory": 0, + "openaimmlu_professional_accounting": 0, + "openaimmlu_professional_law": 0, + "openaimmlu_professional_medicine": 0, + "openaimmlu_professional_psychology": 0, + "openaimmlu_public_relations": 0, + "openaimmlu_security_studies": 0, + "openaimmlu_sociology": 0, + "openaimmlu_us_foreign_policy": 0, + "openaimmlu_virology": 0, + "openaimmlu_world_religions": 0 + }, + "higher_is_better": { + "openaimmlu": { + "acc": true + }, + "openaimmlu_STEM": { + "acc": true + }, + "openaimmlu_abstract_algebra": { + "acc": true + }, + "openaimmlu_anatomy": { + "acc": true + }, + "openaimmlu_astronomy": { + "acc": true + }, + "openaimmlu_business_ethics": { + "acc": true + }, + "openaimmlu_clinical_knowledge": { + "acc": true + }, + "openaimmlu_college_biology": { + "acc": true + }, + "openaimmlu_college_chemistry": { + "acc": true + }, + "openaimmlu_college_computer_science": { + "acc": true + }, + "openaimmlu_college_mathematics": { + "acc": true + }, + "openaimmlu_college_medicine": { + "acc": true + }, + "openaimmlu_college_physics": { + "acc": true + }, + "openaimmlu_computer_security": { + "acc": true + }, + "openaimmlu_conceptual_physics": { + "acc": true + }, + "openaimmlu_econometrics": { + "acc": true + }, + "openaimmlu_electrical_engineering": { + "acc": true + }, + "openaimmlu_elementary_mathematics": { + "acc": true + }, + "openaimmlu_formal_logic": { + "acc": true + }, + "openaimmlu_global_facts": { + "acc": true + }, + "openaimmlu_high_school_biology": { + "acc": true + }, + "openaimmlu_high_school_chemistry": { + "acc": true + }, + "openaimmlu_high_school_computer_science": { + "acc": true + }, + "openaimmlu_high_school_european_history": { + "acc": true + }, + "openaimmlu_high_school_geography": { + "acc": true + }, + "openaimmlu_high_school_government_and_politics": { + "acc": true + }, + "openaimmlu_high_school_macroeconomics": { + "acc": true + }, + "openaimmlu_high_school_mathematics": { + "acc": true + }, + "openaimmlu_high_school_microeconomics": { + "acc": true + }, + "openaimmlu_high_school_physics": { + "acc": true + }, + "openaimmlu_high_school_psychology": { + "acc": true + }, + "openaimmlu_high_school_statistics": { + "acc": true + }, + "openaimmlu_high_school_us_history": { + "acc": true + }, + "openaimmlu_high_school_world_history": { + "acc": true + }, + "openaimmlu_human_aging": { + "acc": true + }, + "openaimmlu_human_sexuality": { + "acc": true + }, + "openaimmlu_humanities": { + "acc": true + }, + "openaimmlu_international_law": { + "acc": true + }, + "openaimmlu_jurisprudence": { + "acc": true + }, + "openaimmlu_logical_fallacies": { + "acc": true + }, + "openaimmlu_machine_learning": { + "acc": true + }, + "openaimmlu_management": { + "acc": true + }, + "openaimmlu_marketing": { + "acc": true + }, + "openaimmlu_medical_genetics": { + "acc": true + }, + "openaimmlu_miscellaneous": { + "acc": true + }, + "openaimmlu_moral_disputes": { + "acc": true + }, + "openaimmlu_moral_scenarios": { + "acc": true + }, + "openaimmlu_nutrition": { + "acc": true + }, + "openaimmlu_other": { + "acc": true + }, + "openaimmlu_philosophy": { + "acc": true + }, + "openaimmlu_prehistory": { + "acc": true + }, + "openaimmlu_professional_accounting": { + "acc": true + }, + "openaimmlu_professional_law": { + "acc": true + }, + "openaimmlu_professional_medicine": { + "acc": true + }, + "openaimmlu_professional_psychology": { + "acc": true + }, + "openaimmlu_public_relations": { + "acc": true + }, + "openaimmlu_security_studies": { + "acc": true + }, + "openaimmlu_social_science": { + "acc": true + }, + "openaimmlu_sociology": { + "acc": true + }, + "openaimmlu_us_foreign_policy": { + "acc": true + }, + "openaimmlu_virology": { + "acc": true + }, + "openaimmlu_world_religions": { + "acc": true + } + }, + "n-samples": { + "openaimmlu_college_physics": { + "original": 102, + "effective": 102 + }, + "openaimmlu_astronomy": { + "original": 152, + "effective": 152 + }, + "openaimmlu_computer_security": { + "original": 100, + "effective": 100 + }, + "openaimmlu_elementary_mathematics": { + "original": 378, + "effective": 378 + }, + "openaimmlu_high_school_chemistry": { + "original": 203, + "effective": 203 + }, + "openaimmlu_college_mathematics": { + "original": 100, + "effective": 100 + }, + "openaimmlu_college_chemistry": { + "original": 100, + "effective": 100 + }, + "openaimmlu_college_biology": { + "original": 144, + "effective": 144 + }, + "openaimmlu_conceptual_physics": { + "original": 235, + "effective": 235 + }, + "openaimmlu_high_school_statistics": { + "original": 216, + "effective": 216 + }, + "openaimmlu_electrical_engineering": { + "original": 145, + "effective": 145 + }, + "openaimmlu_high_school_computer_science": { + "original": 100, + "effective": 100 + }, + "openaimmlu_high_school_mathematics": { + "original": 270, + "effective": 270 + }, + "openaimmlu_abstract_algebra": { + "original": 100, + "effective": 100 + }, + "openaimmlu_high_school_physics": { + "original": 151, + "effective": 151 + }, + "openaimmlu_college_computer_science": { + "original": 100, + "effective": 100 + }, + "openaimmlu_econometrics": { + "original": 114, + "effective": 114 + }, + "openaimmlu_high_school_biology": { + "original": 310, + "effective": 310 + }, + "openaimmlu_nutrition": { + "original": 306, + "effective": 306 + }, + "openaimmlu_miscellaneous": { + "original": 783, + "effective": 783 + }, + "openaimmlu_anatomy": { + "original": 135, + "effective": 135 + }, + "openaimmlu_virology": { + "original": 166, + "effective": 166 + }, + "openaimmlu_professional_medicine": { + "original": 272, + "effective": 272 + }, + "openaimmlu_human_aging": { + "original": 223, + "effective": 223 + }, + "openaimmlu_clinical_knowledge": { + "original": 265, + "effective": 265 + }, + "openaimmlu_professional_accounting": { + "original": 282, + "effective": 282 + }, + "openaimmlu_high_school_geography": { + "original": 198, + "effective": 198 + }, + "openaimmlu_professional_psychology": { + "original": 612, + "effective": 612 + }, + "openaimmlu_high_school_psychology": { + "original": 545, + "effective": 545 + }, + "openaimmlu_machine_learning": { + "original": 112, + "effective": 112 + }, + "openaimmlu_medical_genetics": { + "original": 100, + "effective": 100 + }, + "openaimmlu_professional_law": { + "original": 1534, + "effective": 1534 + }, + "openaimmlu_college_medicine": { + "original": 173, + "effective": 173 + }, + "openaimmlu_formal_logic": { + "original": 126, + "effective": 126 + }, + "openaimmlu_global_facts": { + "original": 100, + "effective": 100 + }, + "openaimmlu_high_school_microeconomics": { + "original": 238, + "effective": 238 + }, + "openaimmlu_high_school_government_and_politics": { + "original": 193, + "effective": 193 + }, + "openaimmlu_management": { + "original": 103, + "effective": 103 + }, + "openaimmlu_security_studies": { + "original": 245, + "effective": 245 + }, + "openaimmlu_business_ethics": { + "original": 100, + "effective": 100 + }, + "openaimmlu_sociology": { + "original": 201, + "effective": 201 + }, + "openaimmlu_high_school_macroeconomics": { + "original": 390, + "effective": 390 + }, + "openaimmlu_moral_scenarios": { + "original": 895, + "effective": 895 + }, + "openaimmlu_public_relations": { + "original": 110, + "effective": 110 + }, + "openaimmlu_us_foreign_policy": { + "original": 100, + "effective": 100 + }, + "openaimmlu_moral_disputes": { + "original": 346, + "effective": 346 + }, + "openaimmlu_human_sexuality": { + "original": 131, + "effective": 131 + }, + "openaimmlu_marketing": { + "original": 234, + "effective": 234 + }, + "openaimmlu_jurisprudence": { + "original": 108, + "effective": 108 + }, + "openaimmlu_logical_fallacies": { + "original": 163, + "effective": 163 + }, + "openaimmlu_philosophy": { + "original": 311, + "effective": 311 + }, + "openaimmlu_high_school_world_history": { + "original": 237, + "effective": 237 + }, + "openaimmlu_high_school_european_history": { + "original": 165, + "effective": 165 + }, + "openaimmlu_prehistory": { + "original": 324, + "effective": 324 + }, + "openaimmlu_high_school_us_history": { + "original": 204, + "effective": 204 + }, + "openaimmlu_international_law": { + "original": 121, + "effective": 121 + }, + "openaimmlu_world_religions": { + "original": 171, + "effective": 171 + } + }, + "config": { + "model": "vllm", + "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8", + "batch_size": 1, + "batch_sizes": [], + "device": null, + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "8e1bd48d", + "date": 1735663577.7452598, + "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 48 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 48\nOn-line CPU(s) list: 0-47\nVendor ID: AuthenticAMD\nModel name: AMD EPYC 7V13 64-Core Processor\nCPU family: 25\nModel: 1\nThread(s) per core: 1\nCore(s) per socket: 48\nSocket(s): 1\nStepping: 1\nBogoMIPS: 4890.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor: Microsoft\nVirtualization type: full\nL1d cache: 1.5 MiB (48 instances)\nL1i cache: 1.5 MiB (48 instances)\nL2 cache: 24 MiB (48 instances)\nL3 cache: 192 MiB (6 instances)\nNUMA node(s): 2\nNUMA node0 CPU(s): 0-23\nNUMA node1 CPU(s): 24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Retbleed: Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect", + "transformers_version": "4.47.1", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "0" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 4096, + "task_hashes": { + "openaimmlu_college_physics": "61aa73bd44d8ef4ba6cb739692c6eb8cabf49e8896a7f725463819ef0dbd0132", + "openaimmlu_astronomy": "4c75961650ca77c7fb21671a45e42b30b2d6737dd89a9dd0f98b5a102a6fd21c", + "openaimmlu_computer_security": "b3b48aa3be2791a75a9678e21c3f7000c0994934e0892e21be48b61eee9022b1", + "openaimmlu_elementary_mathematics": "7ea44fa3e77564b6d8928cb20f739393b20c6df004e164290d5d90ef3d0a8b79", + "openaimmlu_high_school_chemistry": "6d6a118469563d3ce711f5e5ca944e10ed6ea4e52c813217124fc043b7423db6", + "openaimmlu_college_mathematics": "005761dd03c5fd7ac82e8a629717d9fa02e750f7f4913433240ae0886e421bc6", + "openaimmlu_college_chemistry": "0539b9d96465df48c1370ac576a07d6d92f0829fe05cc79bd260ff763a74263d", + "openaimmlu_college_biology": "ac595a195f3fe505c334d8ed12697594cafcbaca1d3247eb8d70a3562f41443e", + "openaimmlu_conceptual_physics": "7e7cb338548eaf777c9eb7cca310fdf726660871c640842032735cc891816586", + "openaimmlu_high_school_statistics": "94f1ab74a8bdbd75041a62e1855c3d15b6ade91a7cc96d274bf57c420c5e7a91", + "openaimmlu_electrical_engineering": "599ce9a4e0938a3911259b3556952c70a5d9ac08df41625179cc73cb45a9797a", + "openaimmlu_high_school_computer_science": "f89803071a28b442ab2f85f9dc6f5c3acc87118b662626c1e164aae4304f45ea", + "openaimmlu_high_school_mathematics": "571309ff8e58fb1d5741d2d95ef005ff09d7e1fc61e75a59fd9bf77d1e4ec25e", + "openaimmlu_abstract_algebra": "18718e53c9eb375b294dc89fddd44e9fec66166611545df741957cb9c3056597", + "openaimmlu_high_school_physics": "b6a0c08c931f22af3809aba7b65315bf82834cf089961e2cada1bc6dab063306", + "openaimmlu_college_computer_science": "614eb77451f839f693631aef6269e65c82e88ad3aa7105f665f4e6187723f986", + "openaimmlu_econometrics": "5b060aa4148ab3c9e801d0591d391b90a21259c436d082120f19a16ce63f7c15", + "openaimmlu_high_school_biology": "fbd661d888bdfd56e1256684914f1b2d2e90f128b26503e0e0d10af6af678e01", + "openaimmlu_nutrition": "c44694a990c0a1187712f3e7d83ee10b5682fde624260c4b78bbd33641647f01", + "openaimmlu_miscellaneous": "a42b3d1263bfa01552f44579362d25e558662731a595c2dec558d8c9ca4d727b", + "openaimmlu_anatomy": "a2158278024b1e9f8867e30434721221f91cbcebaa34c3ca065cc039f6d9ce56", + "openaimmlu_virology": "3fd44c94e0170284d5232b194c5604d338d0ace9cd0ff686d2349b0e7c2e19d6", + "openaimmlu_professional_medicine": "c45c22a09efc77881a194f39b9622414eea01fdd59a4ce6fec12ca0bd542f73b", + "openaimmlu_human_aging": "4216c0274bb171bdb7c8cc2640cd812401e292271f4ee2b95b73b73a48b061c1", + "openaimmlu_clinical_knowledge": "3de23a26358560ceb58b2bd43bf1ca0873f1bde03f92f16048a7fe73ef086f3d", + "openaimmlu_professional_accounting": "b08f816170cc1f742b5d62eca448427a3d57369d56d0db3349d79f0d9da3aec3", + "openaimmlu_high_school_geography": "5f781f776d42c8b641139ad51da36b50da36c450bbec8a01fa33ae25a684133c", + "openaimmlu_professional_psychology": "ee9b7102b9f931cb3c7fab155f7b1828160df589ae8b16844039de7ae3c8d064", + "openaimmlu_high_school_psychology": "f62b828c33fd2ad378bb7aaf081c290210b378f9309aeef1fff01ddc83dd34c6", + "openaimmlu_machine_learning": "4f9232ed92776579ea24eb5fe1ef1275bb2bf8a290f5f004f749cb64b6feee40", + "openaimmlu_medical_genetics": "aece3b6adba4255559594b80438a0e80181e24366ff39b1c91456df945b01f4f", + "openaimmlu_professional_law": "a86f6179dbad6d30286d1b71dafd2799ae6fa219cd7b3a079bb067483bc64b2a", + "openaimmlu_college_medicine": "36232a0c51f93f761adf0529e7004f2f21479517d238b0c9bc538a1138e7482f", + "openaimmlu_formal_logic": "5d7a76ba9f40981d143f6fe780a8269bd525cc54aea3fac3dde297b4d4491413", + "openaimmlu_global_facts": "14d0542f6b985287da88bf4956de680e20754b9d74a009ca463beb66ae081e92", + "openaimmlu_high_school_microeconomics": "51a598169dbd7a3de2e64558fd952a17cd39b49f0c6cb4de95fdb0e5520245c7", + "openaimmlu_high_school_government_and_politics": "7c79983103e230916bf4d730743e92feb7f17e308a2897de764d24f49de93f37", + "openaimmlu_management": "b85920acbd491f4bafeda9de9dc3af4408e63e0d53bfbeade834107dff6c3e3b", + "openaimmlu_security_studies": "de9eccb24ee7d56897728b9bd30c0159f42a6cac86f3d2090504439efcbb2348", + "openaimmlu_business_ethics": "5536730d841c70c256991081bd4d8a9c28aaebcab9ac3cdd36ceb1aad896cfae", + "openaimmlu_sociology": "1fa4ebb60178be200e3b8167e10fec0843964725a6be034e6893ae42d5dd1a3a", + "openaimmlu_high_school_macroeconomics": "938c0435e322f454a5b1f26f1b53870141a9e311bbc95512cf307ecd007e66db", + "openaimmlu_moral_scenarios": "c0158287c824e917d6d76a9d4a4e68a53af20dee7bb7c1d372a759546edef562", + "openaimmlu_public_relations": "545ea7d7b3dbbe04d2c367cd70142c35e1ce585a3c3e4b9d9fa2290d1d25272f", + "openaimmlu_us_foreign_policy": "cddc366ef735093ff1ab6d3660a19d52ae146b9ca18668d8a878be81466cb626", + "openaimmlu_moral_disputes": "2b9fd83448202cb343ad8473f9d34194776e73f9fac0fad093610033039e0152", + "openaimmlu_human_sexuality": "cd4281e8629dd63b57e11fff680c2813bfc156d0807d9a3424670422bb8a8f02", + "openaimmlu_marketing": "85aaada41a32346c0dce6f252b7e5e50a1bce1641cab3ecf6e1590deb8927db4", + "openaimmlu_jurisprudence": "ab9de498411479a47a892895a70b20948854fe8c8177f9851da339a984c534f0", + "openaimmlu_logical_fallacies": "1c809bb030ca1d7256a741cd2f7b3719053d6387df5f89762fca7aa430374461", + "openaimmlu_philosophy": "e6367d4cab84d33e5ba62f20bc52f72d630c4324c0d34628b546cf72a83eb94f", + "openaimmlu_high_school_world_history": "f7e18a11fcc4e11b8c758d3227d7e7fc59157c9bd465ade0c8e4707cb3c76c2a", + "openaimmlu_high_school_european_history": "f6ee95e6dc273ad3d18c110a69772161a9eb250ef81c3202a46228689c5e2071", + "openaimmlu_prehistory": "88d6574515b52d900aab10f1f22d026fa33c8d910a6528acf3dac384d1e82b0a", + "openaimmlu_high_school_us_history": "541952c75bfb8c256d813fdfc4d7707ad25448980fd70d39142ec03a15af9d0d", + "openaimmlu_international_law": "c1acfc8203c4d2f4d5d9245685804c2b7406601dfd8106cc4fac985915559f52", + "openaimmlu_world_religions": "9b68b777a6bc2b05efee57f75e87792c6b14f39464621c16e4c24b024aeb2630" + }, + "model_source": "vllm", + "model_name": "/ALLaM-7B-Instruct", + "model_name_sanitized": "/ALLaM-7B-Instruct", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 4238.00553465, + "end_time": 4722.113520369, + "total_evaluation_time_seconds": "484.10798571899977" +} \ No newline at end of file