evaluations/ar/Allam-7b-instruct-preview/openaimmlu_0_shot.json

{
  "results": {
    "openaimmlu": {
      " ": " ",
      "alias": "openaimmlu"
    },
    "openaimmlu_STEM": {
      "acc,none": 0.4900662251655629,
      "acc_stderr,none": 0.00883192107765626,
      "alias": " - STEM"
    },
    "openaimmlu_abstract_algebra": {
      "alias": "  - abstract_algebra",
      "acc,none": 0.25,
      "acc_stderr,none": 0.04351941398892446
    },
    "openaimmlu_astronomy": {
      "alias": "  - astronomy",
      "acc,none": 0.6842105263157895,
      "acc_stderr,none": 0.037827289808654685
    },
    "openaimmlu_college_biology": {
      "alias": "  - college_biology",
      "acc,none": 0.6597222222222222,
      "acc_stderr,none": 0.039621355734862175
    },
    "openaimmlu_college_chemistry": {
      "alias": "  - college_chemistry",
      "acc,none": 0.35,
      "acc_stderr,none": 0.047937248544110196
    },
    "openaimmlu_college_computer_science": {
      "alias": "  - college_computer_science",
      "acc,none": 0.44,
      "acc_stderr,none": 0.04988876515698589
    },
    "openaimmlu_college_mathematics": {
      "alias": "  - college_mathematics",
      "acc,none": 0.3,
      "acc_stderr,none": 0.046056618647183814
    },
    "openaimmlu_college_physics": {
      "alias": "  - college_physics",
      "acc,none": 0.37254901960784315,
      "acc_stderr,none": 0.04810840148082633
    },
    "openaimmlu_computer_security": {
      "alias": "  - computer_security",
      "acc,none": 0.71,
      "acc_stderr,none": 0.045604802157206845
    },
    "openaimmlu_conceptual_physics": {
      "alias": "  - conceptual_physics",
      "acc,none": 0.548936170212766,
      "acc_stderr,none": 0.032529096196131965
    },
    "openaimmlu_econometrics": {
      "alias": "  - econometrics",
      "acc,none": 0.3684210526315789,
      "acc_stderr,none": 0.04537815354939391
    },
    "openaimmlu_electrical_engineering": {
      "alias": "  - electrical_engineering",
      "acc,none": 0.5103448275862069,
      "acc_stderr,none": 0.04165774775728763
    },
    "openaimmlu_elementary_mathematics": {
      "alias": "  - elementary_mathematics",
      "acc,none": 0.48677248677248675,
      "acc_stderr,none": 0.025742297289575142
    },
    "openaimmlu_high_school_biology": {
      "alias": "  - high_school_biology",
      "acc,none": 0.6645161290322581,
      "acc_stderr,none": 0.026860206444724352
    },
    "openaimmlu_high_school_chemistry": {
      "alias": "  - high_school_chemistry",
      "acc,none": 0.4630541871921182,
      "acc_stderr,none": 0.035083705204426656
    },
    "openaimmlu_high_school_computer_science": {
      "alias": "  - high_school_computer_science",
      "acc,none": 0.56,
      "acc_stderr,none": 0.04988876515698589
    },
    "openaimmlu_high_school_mathematics": {
      "alias": "  - high_school_mathematics",
      "acc,none": 0.35185185185185186,
      "acc_stderr,none": 0.02911661760608301
    },
    "openaimmlu_high_school_physics": {
      "alias": "  - high_school_physics",
      "acc,none": 0.37748344370860926,
      "acc_stderr,none": 0.039580272311215706
    },
    "openaimmlu_high_school_statistics": {
      "alias": "  - high_school_statistics",
      "acc,none": 0.4675925925925926,
      "acc_stderr,none": 0.03402801581358966
    },
    "openaimmlu_humanities": {
      "acc,none": 0.6834811529933481,
      "acc_stderr,none": 0.01087157296938379,
      "alias": " - Humanities"
    },
    "openaimmlu_high_school_european_history": {
      "alias": "  - high_school_european_history",
      "acc,none": 0.7333333333333333,
      "acc_stderr,none": 0.03453131801885417
    },
    "openaimmlu_high_school_us_history": {
      "alias": "  - high_school_us_history",
      "acc,none": 0.7254901960784313,
      "acc_stderr,none": 0.03132179803083291
    },
    "openaimmlu_high_school_world_history": {
      "alias": "  - high_school_world_history",
      "acc,none": 0.7721518987341772,
      "acc_stderr,none": 0.027303484599069415
    },
    "openaimmlu_international_law": {
      "alias": "  - international_law",
      "acc,none": 0.7355371900826446,
      "acc_stderr,none": 0.04026187527591205
    },
    "openaimmlu_jurisprudence": {
      "alias": "  - jurisprudence",
      "acc,none": 0.6851851851851852,
      "acc_stderr,none": 0.04489931073591311
    },
    "openaimmlu_logical_fallacies": {
      "alias": "  - logical_fallacies",
      "acc,none": 0.6871165644171779,
      "acc_stderr,none": 0.03642914578292404
    },
    "openaimmlu_philosophy": {
      "alias": "  - philosophy",
      "acc,none": 0.6077170418006431,
      "acc_stderr,none": 0.027731258647011987
    },
    "openaimmlu_prehistory": {
      "alias": "  - prehistory",
      "acc,none": 0.595679012345679,
      "acc_stderr,none": 0.027306625297327698
    },
    "openaimmlu_world_religions": {
      "alias": "  - world_religions",
      "acc,none": 0.7251461988304093,
      "acc_stderr,none": 0.034240429246915824
    },
    "openaimmlu_other": {
      "acc,none": 0.5571476736345247,
      "acc_stderr,none": 0.0062200183711956835,
      "alias": " - Other"
    },
    "openaimmlu_anatomy": {
      "alias": "  - anatomy",
      "acc,none": 0.4740740740740741,
      "acc_stderr,none": 0.04313531696750575
    },
    "openaimmlu_clinical_knowledge": {
      "alias": "  - clinical_knowledge",
      "acc,none": 0.5773584905660377,
      "acc_stderr,none": 0.030402331445769537
    },
    "openaimmlu_college_medicine": {
      "alias": "  - college_medicine",
      "acc,none": 0.5086705202312138,
      "acc_stderr,none": 0.0381189098894041
    },
    "openaimmlu_formal_logic": {
      "alias": "  - formal_logic",
      "acc,none": 0.3888888888888889,
      "acc_stderr,none": 0.04360314860077459
    },
    "openaimmlu_global_facts": {
      "alias": "  - global_facts",
      "acc,none": 0.4,
      "acc_stderr,none": 0.049236596391733084
    },
    "openaimmlu_high_school_geography": {
      "alias": "  - high_school_geography",
      "acc,none": 0.7121212121212122,
      "acc_stderr,none": 0.03225883512300992
    },
    "openaimmlu_high_school_psychology": {
      "alias": "  - high_school_psychology",
      "acc,none": 0.7302752293577982,
      "acc_stderr,none": 0.01902848671111545
    },
    "openaimmlu_human_aging": {
      "alias": "  - human_aging",
      "acc,none": 0.6278026905829597,
      "acc_stderr,none": 0.0324430528300873
    },
    "openaimmlu_machine_learning": {
      "alias": "  - machine_learning",
      "acc,none": 0.41964285714285715,
      "acc_stderr,none": 0.04684099321077106
    },
    "openaimmlu_medical_genetics": {
      "alias": "  - medical_genetics",
      "acc,none": 0.66,
      "acc_stderr,none": 0.04760952285695237
    },
    "openaimmlu_miscellaneous": {
      "alias": "  - miscellaneous",
      "acc,none": 0.7573435504469987,
      "acc_stderr,none": 0.015329888940899873
    },
    "openaimmlu_nutrition": {
      "alias": "  - nutrition",
      "acc,none": 0.6601307189542484,
      "acc_stderr,none": 0.027121956071388856
    },
    "openaimmlu_professional_accounting": {
      "alias": "  - professional_accounting",
      "acc,none": 0.41843971631205673,
      "acc_stderr,none": 0.029427994039419994
    },
    "openaimmlu_professional_law": {
      "alias": "  - professional_law",
      "acc,none": 0.41264667535853977,
      "acc_stderr,none": 0.012573836633799016
    },
    "openaimmlu_professional_medicine": {
      "alias": "  - professional_medicine",
      "acc,none": 0.5735294117647058,
      "acc_stderr,none": 0.030042615832714857
    },
    "openaimmlu_professional_psychology": {
      "alias": "  - professional_psychology",
      "acc,none": 0.5522875816993464,
      "acc_stderr,none": 0.020116925347422425
    },
    "openaimmlu_virology": {
      "alias": "  - virology",
      "acc,none": 0.4759036144578313,
      "acc_stderr,none": 0.03887971849597264
    },
    "openaimmlu_social_science": {
      "acc,none": 0.5578210590383444,
      "acc_stderr,none": 0.008094265116110859,
      "alias": " - Social Science"
    },
    "openaimmlu_business_ethics": {
      "alias": "  - business_ethics",
      "acc,none": 0.67,
      "acc_stderr,none": 0.04725815626252609
    },
    "openaimmlu_high_school_government_and_politics": {
      "alias": "  - high_school_government_and_politics",
      "acc,none": 0.772020725388601,
      "acc_stderr,none": 0.03027690994517826
    },
    "openaimmlu_high_school_macroeconomics": {
      "alias": "  - high_school_macroeconomics",
      "acc,none": 0.5692307692307692,
      "acc_stderr,none": 0.025106820660539753
    },
    "openaimmlu_high_school_microeconomics": {
      "alias": "  - high_school_microeconomics",
      "acc,none": 0.5756302521008403,
      "acc_stderr,none": 0.03210479051015776
    },
    "openaimmlu_human_sexuality": {
      "alias": "  - human_sexuality",
      "acc,none": 0.6641221374045801,
      "acc_stderr,none": 0.04142313771996664
    },
    "openaimmlu_management": {
      "alias": "  - management",
      "acc,none": 0.7281553398058253,
      "acc_stderr,none": 0.044052680241409216
    },
    "openaimmlu_marketing": {
      "alias": "  - marketing",
      "acc,none": 0.8076923076923077,
      "acc_stderr,none": 0.025819233256483727
    },
    "openaimmlu_moral_disputes": {
      "alias": "  - moral_disputes",
      "acc,none": 0.5751445086705202,
      "acc_stderr,none": 0.026613350840261746
    },
    "openaimmlu_moral_scenarios": {
      "alias": "  - moral_scenarios",
      "acc,none": 0.2916201117318436,
      "acc_stderr,none": 0.015201032512520442
    },
    "openaimmlu_public_relations": {
      "alias": "  - public_relations",
      "acc,none": 0.5727272727272728,
      "acc_stderr,none": 0.047381987035454834
    },
    "openaimmlu_security_studies": {
      "alias": "  - security_studies",
      "acc,none": 0.6693877551020408,
      "acc_stderr,none": 0.030116426296540603
    },
    "openaimmlu_sociology": {
      "alias": "  - sociology",
      "acc,none": 0.6915422885572139,
      "acc_stderr,none": 0.032658195885126966
    },
    "openaimmlu_us_foreign_policy": {
      "alias": "  - us_foreign_policy",
      "acc,none": 0.81,
      "acc_stderr,none": 0.039427724440366234
    }
  },
  "groups": {
    "openaimmlu_STEM": {
      "acc,none": 0.4900662251655629,
      "acc_stderr,none": 0.00883192107765626,
      "alias": " - STEM"
    },
    "openaimmlu_humanities": {
      "acc,none": 0.6834811529933481,
      "acc_stderr,none": 0.01087157296938379,
      "alias": " - Humanities"
    },
    "openaimmlu_other": {
      "acc,none": 0.5571476736345247,
      "acc_stderr,none": 0.0062200183711956835,
      "alias": " - Other"
    },
    "openaimmlu_social_science": {
      "acc,none": 0.5578210590383444,
      "acc_stderr,none": 0.008094265116110859,
      "alias": " - Social Science"
    }
  },
  "group_subtasks": {
    "openaimmlu_humanities": [
      "openaimmlu_jurisprudence",
      "openaimmlu_logical_fallacies",
      "openaimmlu_philosophy",
      "openaimmlu_high_school_world_history",
      "openaimmlu_high_school_european_history",
      "openaimmlu_prehistory",
      "openaimmlu_high_school_us_history",
      "openaimmlu_international_law",
      "openaimmlu_world_religions"
    ],
    "openaimmlu_social_science": [
      "openaimmlu_high_school_microeconomics",
      "openaimmlu_high_school_government_and_politics",
      "openaimmlu_management",
      "openaimmlu_security_studies",
      "openaimmlu_business_ethics",
      "openaimmlu_sociology",
      "openaimmlu_high_school_macroeconomics",
      "openaimmlu_moral_scenarios",
      "openaimmlu_public_relations",
      "openaimmlu_us_foreign_policy",
      "openaimmlu_moral_disputes",
      "openaimmlu_human_sexuality",
      "openaimmlu_marketing"
    ],
    "openaimmlu_other": [
      "openaimmlu_nutrition",
      "openaimmlu_miscellaneous",
      "openaimmlu_anatomy",
      "openaimmlu_virology",
      "openaimmlu_professional_medicine",
      "openaimmlu_human_aging",
      "openaimmlu_clinical_knowledge",
      "openaimmlu_professional_accounting",
      "openaimmlu_high_school_geography",
      "openaimmlu_professional_psychology",
      "openaimmlu_high_school_psychology",
      "openaimmlu_machine_learning",
      "openaimmlu_medical_genetics",
      "openaimmlu_professional_law",
      "openaimmlu_college_medicine",
      "openaimmlu_formal_logic",
      "openaimmlu_global_facts"
    ],
    "openaimmlu_STEM": [
      "openaimmlu_college_physics",
      "openaimmlu_astronomy",
      "openaimmlu_computer_security",
      "openaimmlu_elementary_mathematics",
      "openaimmlu_high_school_chemistry",
      "openaimmlu_college_mathematics",
      "openaimmlu_college_chemistry",
      "openaimmlu_college_biology",
      "openaimmlu_conceptual_physics",
      "openaimmlu_high_school_statistics",
      "openaimmlu_electrical_engineering",
      "openaimmlu_high_school_computer_science",
      "openaimmlu_high_school_mathematics",
      "openaimmlu_abstract_algebra",
      "openaimmlu_high_school_physics",
      "openaimmlu_college_computer_science",
      "openaimmlu_econometrics",
      "openaimmlu_high_school_biology"
    ],
    "openaimmlu": [
      "openaimmlu_STEM",
      "openaimmlu_other",
      "openaimmlu_social_science",
      "openaimmlu_humanities"
    ]
  },
  "configs": {
    "openaimmlu_abstract_algebra": {
      "task": "openaimmlu_abstract_algebra",
      "task_alias": "abstract_algebra",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "abstract_algebra",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_anatomy": {
      "task": "openaimmlu_anatomy",
      "task_alias": "anatomy",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "anatomy",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_astronomy": {
      "task": "openaimmlu_astronomy",
      "task_alias": "astronomy",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "astronomy",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_business_ethics": {
      "task": "openaimmlu_business_ethics",
      "task_alias": "business_ethics",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "business_ethics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_clinical_knowledge": {
      "task": "openaimmlu_clinical_knowledge",
      "task_alias": "clinical_knowledge",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "clinical_knowledge",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_college_biology": {
      "task": "openaimmlu_college_biology",
      "task_alias": "college_biology",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "college_biology",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_college_chemistry": {
      "task": "openaimmlu_college_chemistry",
      "task_alias": "college_chemistry",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "college_chemistry",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_college_computer_science": {
      "task": "openaimmlu_college_computer_science",
      "task_alias": "college_computer_science",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "college_computer_science",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_college_mathematics": {
      "task": "openaimmlu_college_mathematics",
      "task_alias": "college_mathematics",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "college_mathematics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_college_medicine": {
      "task": "openaimmlu_college_medicine",
      "task_alias": "college_medicine",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "college_medicine",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_college_physics": {
      "task": "openaimmlu_college_physics",
      "task_alias": "college_physics",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "college_physics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_computer_security": {
      "task": "openaimmlu_computer_security",
      "task_alias": "computer_security",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "computer_security",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_conceptual_physics": {
      "task": "openaimmlu_conceptual_physics",
      "task_alias": "conceptual_physics",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "conceptual_physics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_econometrics": {
      "task": "openaimmlu_econometrics",
      "task_alias": "econometrics",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "econometrics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_electrical_engineering": {
      "task": "openaimmlu_electrical_engineering",
      "task_alias": "electrical_engineering",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "electrical_engineering",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_elementary_mathematics": {
      "task": "openaimmlu_elementary_mathematics",
      "task_alias": "elementary_mathematics",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "elementary_mathematics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_formal_logic": {
      "task": "openaimmlu_formal_logic",
      "task_alias": "formal_logic",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "formal_logic",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_global_facts": {
      "task": "openaimmlu_global_facts",
      "task_alias": "global_facts",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "global_facts",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_biology": {
      "task": "openaimmlu_high_school_biology",
      "task_alias": "high_school_biology",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_biology",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_chemistry": {
      "task": "openaimmlu_high_school_chemistry",
      "task_alias": "high_school_chemistry",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_chemistry",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_computer_science": {
      "task": "openaimmlu_high_school_computer_science",
      "task_alias": "high_school_computer_science",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_computer_science",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_european_history": {
      "task": "openaimmlu_high_school_european_history",
      "task_alias": "high_school_european_history",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_european_history",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_geography": {
      "task": "openaimmlu_high_school_geography",
      "task_alias": "high_school_geography",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_geography",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_government_and_politics": {
      "task": "openaimmlu_high_school_government_and_politics",
      "task_alias": "high_school_government_and_politics",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_government_and_politics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_macroeconomics": {
      "task": "openaimmlu_high_school_macroeconomics",
      "task_alias": "high_school_macroeconomics",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_macroeconomics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_mathematics": {
      "task": "openaimmlu_high_school_mathematics",
      "task_alias": "high_school_mathematics",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_mathematics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_microeconomics": {
      "task": "openaimmlu_high_school_microeconomics",
      "task_alias": "high_school_microeconomics",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_microeconomics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_physics": {
      "task": "openaimmlu_high_school_physics",
      "task_alias": "high_school_physics",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_physics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_psychology": {
      "task": "openaimmlu_high_school_psychology",
      "task_alias": "high_school_psychology",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_psychology",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_statistics": {
      "task": "openaimmlu_high_school_statistics",
      "task_alias": "high_school_statistics",
      "tag": "openaimmlu_STEM_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_statistics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_us_history": {
      "task": "openaimmlu_high_school_us_history",
      "task_alias": "high_school_us_history",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_us_history",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_high_school_world_history": {
      "task": "openaimmlu_high_school_world_history",
      "task_alias": "high_school_world_history",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "high_school_world_history",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_human_aging": {
      "task": "openaimmlu_human_aging",
      "task_alias": "human_aging",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "human_aging",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_human_sexuality": {
      "task": "openaimmlu_human_sexuality",
      "task_alias": "human_sexuality",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "human_sexuality",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_international_law": {
      "task": "openaimmlu_international_law",
      "task_alias": "international_law",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "international_law",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_jurisprudence": {
      "task": "openaimmlu_jurisprudence",
      "task_alias": "jurisprudence",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "jurisprudence",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_logical_fallacies": {
      "task": "openaimmlu_logical_fallacies",
      "task_alias": "logical_fallacies",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "logical_fallacies",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_machine_learning": {
      "task": "openaimmlu_machine_learning",
      "task_alias": "machine_learning",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "machine_learning",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_management": {
      "task": "openaimmlu_management",
      "task_alias": "management",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "management",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_marketing": {
      "task": "openaimmlu_marketing",
      "task_alias": "marketing",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "marketing",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_medical_genetics": {
      "task": "openaimmlu_medical_genetics",
      "task_alias": "medical_genetics",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "medical_genetics",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_miscellaneous": {
      "task": "openaimmlu_miscellaneous",
      "task_alias": "miscellaneous",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "miscellaneous",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_moral_disputes": {
      "task": "openaimmlu_moral_disputes",
      "task_alias": "moral_disputes",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "moral_disputes",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_moral_scenarios": {
      "task": "openaimmlu_moral_scenarios",
      "task_alias": "moral_scenarios",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "moral_scenarios",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_nutrition": {
      "task": "openaimmlu_nutrition",
      "task_alias": "nutrition",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "nutrition",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_philosophy": {
      "task": "openaimmlu_philosophy",
      "task_alias": "philosophy",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "philosophy",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_prehistory": {
      "task": "openaimmlu_prehistory",
      "task_alias": "prehistory",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "prehistory",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_professional_accounting": {
      "task": "openaimmlu_professional_accounting",
      "task_alias": "professional_accounting",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "professional_accounting",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_professional_law": {
      "task": "openaimmlu_professional_law",
      "task_alias": "professional_law",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "professional_law",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_professional_medicine": {
      "task": "openaimmlu_professional_medicine",
      "task_alias": "professional_medicine",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "professional_medicine",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_professional_psychology": {
      "task": "openaimmlu_professional_psychology",
      "task_alias": "professional_psychology",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "professional_psychology",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_public_relations": {
      "task": "openaimmlu_public_relations",
      "task_alias": "public_relations",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "public_relations",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_security_studies": {
      "task": "openaimmlu_security_studies",
      "task_alias": "security_studies",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "security_studies",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_sociology": {
      "task": "openaimmlu_sociology",
      "task_alias": "sociology",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "sociology",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_us_foreign_policy": {
      "task": "openaimmlu_us_foreign_policy",
      "task_alias": "us_foreign_policy",
      "tag": "openaimmlu_social_science_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "us_foreign_policy",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_virology": {
      "task": "openaimmlu_virology",
      "task_alias": "virology",
      "tag": "openaimmlu_other_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "virology",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    },
    "openaimmlu_world_religions": {
      "task": "openaimmlu_world_religions",
      "task_alias": "world_religions",
      "tag": "openaimmlu_humanities_tasks",
      "dataset_path": "khalidalt/openai_mmlu_arabic",
      "dataset_name": "world_religions",
      "test_split": "test",
      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n\n        def format_example(doc, choices):\n            options = []\n            for _, choice in enumerate(choices):\n                options.append(f'{en2ar[choice]}. {doc[choice]}')\n\n            ar_subject = SUBJECTS[doc['Subject']]\n            query = PROMPT.format(ar_subject, #doc['Subject'],\n                                    doc['Question'],\n                                    \"\\n\".join(options))\n            return query\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        keys_ar = ['\u0623', '\u0628', '\u062c', '\u062f']\n        ar_label = en2ar[doc['Answer']]\n\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_ar,\n            \"gold\": keys_ar.index(ar_label)\n                }\n\n        return out_doc\n\n    return dataset.map(_process_docs)             \n",
      "doc_to_text": "query",
      "doc_to_target": "gold",
      "doc_to_choice": "choices",
      "description": "",
      "target_delimiter": " ",
      "fewshot_delimiter": "\n\n",
      "num_fewshot": 0,
      "metric_list": [
        {
          "metric": "acc",
          "aggregation": "mean",
          "higher_is_better": true
        }
      ],
      "output_type": "multiple_choice",
      "repeats": 1,
      "should_decontaminate": false,
      "metadata": {
        "version": 0.0
      }
    }
  },
  "versions": {
    "openaimmlu_STEM": 0,
    "openaimmlu_abstract_algebra": 0.0,
    "openaimmlu_anatomy": 0.0,
    "openaimmlu_astronomy": 0.0,
    "openaimmlu_business_ethics": 0.0,
    "openaimmlu_clinical_knowledge": 0.0,
    "openaimmlu_college_biology": 0.0,
    "openaimmlu_college_chemistry": 0.0,
    "openaimmlu_college_computer_science": 0.0,
    "openaimmlu_college_mathematics": 0.0,
    "openaimmlu_college_medicine": 0.0,
    "openaimmlu_college_physics": 0.0,
    "openaimmlu_computer_security": 0.0,
    "openaimmlu_conceptual_physics": 0.0,
    "openaimmlu_econometrics": 0.0,
    "openaimmlu_electrical_engineering": 0.0,
    "openaimmlu_elementary_mathematics": 0.0,
    "openaimmlu_formal_logic": 0.0,
    "openaimmlu_global_facts": 0.0,
    "openaimmlu_high_school_biology": 0.0,
    "openaimmlu_high_school_chemistry": 0.0,
    "openaimmlu_high_school_computer_science": 0.0,
    "openaimmlu_high_school_european_history": 0.0,
    "openaimmlu_high_school_geography": 0.0,
    "openaimmlu_high_school_government_and_politics": 0.0,
    "openaimmlu_high_school_macroeconomics": 0.0,
    "openaimmlu_high_school_mathematics": 0.0,
    "openaimmlu_high_school_microeconomics": 0.0,
    "openaimmlu_high_school_physics": 0.0,
    "openaimmlu_high_school_psychology": 0.0,
    "openaimmlu_high_school_statistics": 0.0,
    "openaimmlu_high_school_us_history": 0.0,
    "openaimmlu_high_school_world_history": 0.0,
    "openaimmlu_human_aging": 0.0,
    "openaimmlu_human_sexuality": 0.0,
    "openaimmlu_humanities": 0,
    "openaimmlu_international_law": 0.0,
    "openaimmlu_jurisprudence": 0.0,
    "openaimmlu_logical_fallacies": 0.0,
    "openaimmlu_machine_learning": 0.0,
    "openaimmlu_management": 0.0,
    "openaimmlu_marketing": 0.0,
    "openaimmlu_medical_genetics": 0.0,
    "openaimmlu_miscellaneous": 0.0,
    "openaimmlu_moral_disputes": 0.0,
    "openaimmlu_moral_scenarios": 0.0,
    "openaimmlu_nutrition": 0.0,
    "openaimmlu_other": 0,
    "openaimmlu_philosophy": 0.0,
    "openaimmlu_prehistory": 0.0,
    "openaimmlu_professional_accounting": 0.0,
    "openaimmlu_professional_law": 0.0,
    "openaimmlu_professional_medicine": 0.0,
    "openaimmlu_professional_psychology": 0.0,
    "openaimmlu_public_relations": 0.0,
    "openaimmlu_security_studies": 0.0,
    "openaimmlu_social_science": 0,
    "openaimmlu_sociology": 0.0,
    "openaimmlu_us_foreign_policy": 0.0,
    "openaimmlu_virology": 0.0,
    "openaimmlu_world_religions": 0.0
  },
  "n-shot": {
    "openaimmlu_abstract_algebra": 0,
    "openaimmlu_anatomy": 0,
    "openaimmlu_astronomy": 0,
    "openaimmlu_business_ethics": 0,
    "openaimmlu_clinical_knowledge": 0,
    "openaimmlu_college_biology": 0,
    "openaimmlu_college_chemistry": 0,
    "openaimmlu_college_computer_science": 0,
    "openaimmlu_college_mathematics": 0,
    "openaimmlu_college_medicine": 0,
    "openaimmlu_college_physics": 0,
    "openaimmlu_computer_security": 0,
    "openaimmlu_conceptual_physics": 0,
    "openaimmlu_econometrics": 0,
    "openaimmlu_electrical_engineering": 0,
    "openaimmlu_elementary_mathematics": 0,
    "openaimmlu_formal_logic": 0,
    "openaimmlu_global_facts": 0,
    "openaimmlu_high_school_biology": 0,
    "openaimmlu_high_school_chemistry": 0,
    "openaimmlu_high_school_computer_science": 0,
    "openaimmlu_high_school_european_history": 0,
    "openaimmlu_high_school_geography": 0,
    "openaimmlu_high_school_government_and_politics": 0,
    "openaimmlu_high_school_macroeconomics": 0,
    "openaimmlu_high_school_mathematics": 0,
    "openaimmlu_high_school_microeconomics": 0,
    "openaimmlu_high_school_physics": 0,
    "openaimmlu_high_school_psychology": 0,
    "openaimmlu_high_school_statistics": 0,
    "openaimmlu_high_school_us_history": 0,
    "openaimmlu_high_school_world_history": 0,
    "openaimmlu_human_aging": 0,
    "openaimmlu_human_sexuality": 0,
    "openaimmlu_international_law": 0,
    "openaimmlu_jurisprudence": 0,
    "openaimmlu_logical_fallacies": 0,
    "openaimmlu_machine_learning": 0,
    "openaimmlu_management": 0,
    "openaimmlu_marketing": 0,
    "openaimmlu_medical_genetics": 0,
    "openaimmlu_miscellaneous": 0,
    "openaimmlu_moral_disputes": 0,
    "openaimmlu_moral_scenarios": 0,
    "openaimmlu_nutrition": 0,
    "openaimmlu_philosophy": 0,
    "openaimmlu_prehistory": 0,
    "openaimmlu_professional_accounting": 0,
    "openaimmlu_professional_law": 0,
    "openaimmlu_professional_medicine": 0,
    "openaimmlu_professional_psychology": 0,
    "openaimmlu_public_relations": 0,
    "openaimmlu_security_studies": 0,
    "openaimmlu_sociology": 0,
    "openaimmlu_us_foreign_policy": 0,
    "openaimmlu_virology": 0,
    "openaimmlu_world_religions": 0
  },
  "higher_is_better": {
    "openaimmlu": {
      "acc": true
    },
    "openaimmlu_STEM": {
      "acc": true
    },
    "openaimmlu_abstract_algebra": {
      "acc": true
    },
    "openaimmlu_anatomy": {
      "acc": true
    },
    "openaimmlu_astronomy": {
      "acc": true
    },
    "openaimmlu_business_ethics": {
      "acc": true
    },
    "openaimmlu_clinical_knowledge": {
      "acc": true
    },
    "openaimmlu_college_biology": {
      "acc": true
    },
    "openaimmlu_college_chemistry": {
      "acc": true
    },
    "openaimmlu_college_computer_science": {
      "acc": true
    },
    "openaimmlu_college_mathematics": {
      "acc": true
    },
    "openaimmlu_college_medicine": {
      "acc": true
    },
    "openaimmlu_college_physics": {
      "acc": true
    },
    "openaimmlu_computer_security": {
      "acc": true
    },
    "openaimmlu_conceptual_physics": {
      "acc": true
    },
    "openaimmlu_econometrics": {
      "acc": true
    },
    "openaimmlu_electrical_engineering": {
      "acc": true
    },
    "openaimmlu_elementary_mathematics": {
      "acc": true
    },
    "openaimmlu_formal_logic": {
      "acc": true
    },
    "openaimmlu_global_facts": {
      "acc": true
    },
    "openaimmlu_high_school_biology": {
      "acc": true
    },
    "openaimmlu_high_school_chemistry": {
      "acc": true
    },
    "openaimmlu_high_school_computer_science": {
      "acc": true
    },
    "openaimmlu_high_school_european_history": {
      "acc": true
    },
    "openaimmlu_high_school_geography": {
      "acc": true
    },
    "openaimmlu_high_school_government_and_politics": {
      "acc": true
    },
    "openaimmlu_high_school_macroeconomics": {
      "acc": true
    },
    "openaimmlu_high_school_mathematics": {
      "acc": true
    },
    "openaimmlu_high_school_microeconomics": {
      "acc": true
    },
    "openaimmlu_high_school_physics": {
      "acc": true
    },
    "openaimmlu_high_school_psychology": {
      "acc": true
    },
    "openaimmlu_high_school_statistics": {
      "acc": true
    },
    "openaimmlu_high_school_us_history": {
      "acc": true
    },
    "openaimmlu_high_school_world_history": {
      "acc": true
    },
    "openaimmlu_human_aging": {
      "acc": true
    },
    "openaimmlu_human_sexuality": {
      "acc": true
    },
    "openaimmlu_humanities": {
      "acc": true
    },
    "openaimmlu_international_law": {
      "acc": true
    },
    "openaimmlu_jurisprudence": {
      "acc": true
    },
    "openaimmlu_logical_fallacies": {
      "acc": true
    },
    "openaimmlu_machine_learning": {
      "acc": true
    },
    "openaimmlu_management": {
      "acc": true
    },
    "openaimmlu_marketing": {
      "acc": true
    },
    "openaimmlu_medical_genetics": {
      "acc": true
    },
    "openaimmlu_miscellaneous": {
      "acc": true
    },
    "openaimmlu_moral_disputes": {
      "acc": true
    },
    "openaimmlu_moral_scenarios": {
      "acc": true
    },
    "openaimmlu_nutrition": {
      "acc": true
    },
    "openaimmlu_other": {
      "acc": true
    },
    "openaimmlu_philosophy": {
      "acc": true
    },
    "openaimmlu_prehistory": {
      "acc": true
    },
    "openaimmlu_professional_accounting": {
      "acc": true
    },
    "openaimmlu_professional_law": {
      "acc": true
    },
    "openaimmlu_professional_medicine": {
      "acc": true
    },
    "openaimmlu_professional_psychology": {
      "acc": true
    },
    "openaimmlu_public_relations": {
      "acc": true
    },
    "openaimmlu_security_studies": {
      "acc": true
    },
    "openaimmlu_social_science": {
      "acc": true
    },
    "openaimmlu_sociology": {
      "acc": true
    },
    "openaimmlu_us_foreign_policy": {
      "acc": true
    },
    "openaimmlu_virology": {
      "acc": true
    },
    "openaimmlu_world_religions": {
      "acc": true
    }
  },
  "n-samples": {
    "openaimmlu_college_physics": {
      "original": 102,
      "effective": 102
    },
    "openaimmlu_astronomy": {
      "original": 152,
      "effective": 152
    },
    "openaimmlu_computer_security": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_elementary_mathematics": {
      "original": 378,
      "effective": 378
    },
    "openaimmlu_high_school_chemistry": {
      "original": 203,
      "effective": 203
    },
    "openaimmlu_college_mathematics": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_college_chemistry": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_college_biology": {
      "original": 144,
      "effective": 144
    },
    "openaimmlu_conceptual_physics": {
      "original": 235,
      "effective": 235
    },
    "openaimmlu_high_school_statistics": {
      "original": 216,
      "effective": 216
    },
    "openaimmlu_electrical_engineering": {
      "original": 145,
      "effective": 145
    },
    "openaimmlu_high_school_computer_science": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_high_school_mathematics": {
      "original": 270,
      "effective": 270
    },
    "openaimmlu_abstract_algebra": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_high_school_physics": {
      "original": 151,
      "effective": 151
    },
    "openaimmlu_college_computer_science": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_econometrics": {
      "original": 114,
      "effective": 114
    },
    "openaimmlu_high_school_biology": {
      "original": 310,
      "effective": 310
    },
    "openaimmlu_nutrition": {
      "original": 306,
      "effective": 306
    },
    "openaimmlu_miscellaneous": {
      "original": 783,
      "effective": 783
    },
    "openaimmlu_anatomy": {
      "original": 135,
      "effective": 135
    },
    "openaimmlu_virology": {
      "original": 166,
      "effective": 166
    },
    "openaimmlu_professional_medicine": {
      "original": 272,
      "effective": 272
    },
    "openaimmlu_human_aging": {
      "original": 223,
      "effective": 223
    },
    "openaimmlu_clinical_knowledge": {
      "original": 265,
      "effective": 265
    },
    "openaimmlu_professional_accounting": {
      "original": 282,
      "effective": 282
    },
    "openaimmlu_high_school_geography": {
      "original": 198,
      "effective": 198
    },
    "openaimmlu_professional_psychology": {
      "original": 612,
      "effective": 612
    },
    "openaimmlu_high_school_psychology": {
      "original": 545,
      "effective": 545
    },
    "openaimmlu_machine_learning": {
      "original": 112,
      "effective": 112
    },
    "openaimmlu_medical_genetics": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_professional_law": {
      "original": 1534,
      "effective": 1534
    },
    "openaimmlu_college_medicine": {
      "original": 173,
      "effective": 173
    },
    "openaimmlu_formal_logic": {
      "original": 126,
      "effective": 126
    },
    "openaimmlu_global_facts": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_high_school_microeconomics": {
      "original": 238,
      "effective": 238
    },
    "openaimmlu_high_school_government_and_politics": {
      "original": 193,
      "effective": 193
    },
    "openaimmlu_management": {
      "original": 103,
      "effective": 103
    },
    "openaimmlu_security_studies": {
      "original": 245,
      "effective": 245
    },
    "openaimmlu_business_ethics": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_sociology": {
      "original": 201,
      "effective": 201
    },
    "openaimmlu_high_school_macroeconomics": {
      "original": 390,
      "effective": 390
    },
    "openaimmlu_moral_scenarios": {
      "original": 895,
      "effective": 895
    },
    "openaimmlu_public_relations": {
      "original": 110,
      "effective": 110
    },
    "openaimmlu_us_foreign_policy": {
      "original": 100,
      "effective": 100
    },
    "openaimmlu_moral_disputes": {
      "original": 346,
      "effective": 346
    },
    "openaimmlu_human_sexuality": {
      "original": 131,
      "effective": 131
    },
    "openaimmlu_marketing": {
      "original": 234,
      "effective": 234
    },
    "openaimmlu_jurisprudence": {
      "original": 108,
      "effective": 108
    },
    "openaimmlu_logical_fallacies": {
      "original": 163,
      "effective": 163
    },
    "openaimmlu_philosophy": {
      "original": 311,
      "effective": 311
    },
    "openaimmlu_high_school_world_history": {
      "original": 237,
      "effective": 237
    },
    "openaimmlu_high_school_european_history": {
      "original": 165,
      "effective": 165
    },
    "openaimmlu_prehistory": {
      "original": 324,
      "effective": 324
    },
    "openaimmlu_high_school_us_history": {
      "original": 204,
      "effective": 204
    },
    "openaimmlu_international_law": {
      "original": 121,
      "effective": 121
    },
    "openaimmlu_world_religions": {
      "original": 171,
      "effective": 171
    }
  },
  "config": {
    "model": "vllm",
    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
    "batch_size": 1,
    "batch_sizes": [],
    "device": null,
    "use_cache": null,
    "limit": null,
    "bootstrap_iters": 100000,
    "gen_kwargs": null,
    "random_seed": 0,
    "numpy_seed": 1234,
    "torch_seed": 1234,
    "fewshot_seed": 1234
  },
  "git_hash": "8e1bd48d",
  "date": 1735663577.7452598,
  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
  "transformers_version": "4.47.1",
  "upper_git_hash": null,
  "tokenizer_pad_token": [
    "<unk>",
    "0"
  ],
  "tokenizer_eos_token": [
    "</s>",
    "2"
  ],
  "tokenizer_bos_token": [
    "<s>",
    "1"
  ],
  "eot_token_id": 2,
  "max_length": 4096,
  "task_hashes": {
    "openaimmlu_college_physics": "61aa73bd44d8ef4ba6cb739692c6eb8cabf49e8896a7f725463819ef0dbd0132",
    "openaimmlu_astronomy": "4c75961650ca77c7fb21671a45e42b30b2d6737dd89a9dd0f98b5a102a6fd21c",
    "openaimmlu_computer_security": "b3b48aa3be2791a75a9678e21c3f7000c0994934e0892e21be48b61eee9022b1",
    "openaimmlu_elementary_mathematics": "7ea44fa3e77564b6d8928cb20f739393b20c6df004e164290d5d90ef3d0a8b79",
    "openaimmlu_high_school_chemistry": "6d6a118469563d3ce711f5e5ca944e10ed6ea4e52c813217124fc043b7423db6",
    "openaimmlu_college_mathematics": "005761dd03c5fd7ac82e8a629717d9fa02e750f7f4913433240ae0886e421bc6",
    "openaimmlu_college_chemistry": "0539b9d96465df48c1370ac576a07d6d92f0829fe05cc79bd260ff763a74263d",
    "openaimmlu_college_biology": "ac595a195f3fe505c334d8ed12697594cafcbaca1d3247eb8d70a3562f41443e",
    "openaimmlu_conceptual_physics": "7e7cb338548eaf777c9eb7cca310fdf726660871c640842032735cc891816586",
    "openaimmlu_high_school_statistics": "94f1ab74a8bdbd75041a62e1855c3d15b6ade91a7cc96d274bf57c420c5e7a91",
    "openaimmlu_electrical_engineering": "599ce9a4e0938a3911259b3556952c70a5d9ac08df41625179cc73cb45a9797a",
    "openaimmlu_high_school_computer_science": "f89803071a28b442ab2f85f9dc6f5c3acc87118b662626c1e164aae4304f45ea",
    "openaimmlu_high_school_mathematics": "571309ff8e58fb1d5741d2d95ef005ff09d7e1fc61e75a59fd9bf77d1e4ec25e",
    "openaimmlu_abstract_algebra": "18718e53c9eb375b294dc89fddd44e9fec66166611545df741957cb9c3056597",
    "openaimmlu_high_school_physics": "b6a0c08c931f22af3809aba7b65315bf82834cf089961e2cada1bc6dab063306",
    "openaimmlu_college_computer_science": "614eb77451f839f693631aef6269e65c82e88ad3aa7105f665f4e6187723f986",
    "openaimmlu_econometrics": "5b060aa4148ab3c9e801d0591d391b90a21259c436d082120f19a16ce63f7c15",
    "openaimmlu_high_school_biology": "fbd661d888bdfd56e1256684914f1b2d2e90f128b26503e0e0d10af6af678e01",
    "openaimmlu_nutrition": "c44694a990c0a1187712f3e7d83ee10b5682fde624260c4b78bbd33641647f01",
    "openaimmlu_miscellaneous": "a42b3d1263bfa01552f44579362d25e558662731a595c2dec558d8c9ca4d727b",
    "openaimmlu_anatomy": "a2158278024b1e9f8867e30434721221f91cbcebaa34c3ca065cc039f6d9ce56",
    "openaimmlu_virology": "3fd44c94e0170284d5232b194c5604d338d0ace9cd0ff686d2349b0e7c2e19d6",
    "openaimmlu_professional_medicine": "c45c22a09efc77881a194f39b9622414eea01fdd59a4ce6fec12ca0bd542f73b",
    "openaimmlu_human_aging": "4216c0274bb171bdb7c8cc2640cd812401e292271f4ee2b95b73b73a48b061c1",
    "openaimmlu_clinical_knowledge": "3de23a26358560ceb58b2bd43bf1ca0873f1bde03f92f16048a7fe73ef086f3d",
    "openaimmlu_professional_accounting": "b08f816170cc1f742b5d62eca448427a3d57369d56d0db3349d79f0d9da3aec3",
    "openaimmlu_high_school_geography": "5f781f776d42c8b641139ad51da36b50da36c450bbec8a01fa33ae25a684133c",
    "openaimmlu_professional_psychology": "ee9b7102b9f931cb3c7fab155f7b1828160df589ae8b16844039de7ae3c8d064",
    "openaimmlu_high_school_psychology": "f62b828c33fd2ad378bb7aaf081c290210b378f9309aeef1fff01ddc83dd34c6",
    "openaimmlu_machine_learning": "4f9232ed92776579ea24eb5fe1ef1275bb2bf8a290f5f004f749cb64b6feee40",
    "openaimmlu_medical_genetics": "aece3b6adba4255559594b80438a0e80181e24366ff39b1c91456df945b01f4f",
    "openaimmlu_professional_law": "a86f6179dbad6d30286d1b71dafd2799ae6fa219cd7b3a079bb067483bc64b2a",
    "openaimmlu_college_medicine": "36232a0c51f93f761adf0529e7004f2f21479517d238b0c9bc538a1138e7482f",
    "openaimmlu_formal_logic": "5d7a76ba9f40981d143f6fe780a8269bd525cc54aea3fac3dde297b4d4491413",
    "openaimmlu_global_facts": "14d0542f6b985287da88bf4956de680e20754b9d74a009ca463beb66ae081e92",
    "openaimmlu_high_school_microeconomics": "51a598169dbd7a3de2e64558fd952a17cd39b49f0c6cb4de95fdb0e5520245c7",
    "openaimmlu_high_school_government_and_politics": "7c79983103e230916bf4d730743e92feb7f17e308a2897de764d24f49de93f37",
    "openaimmlu_management": "b85920acbd491f4bafeda9de9dc3af4408e63e0d53bfbeade834107dff6c3e3b",
    "openaimmlu_security_studies": "de9eccb24ee7d56897728b9bd30c0159f42a6cac86f3d2090504439efcbb2348",
    "openaimmlu_business_ethics": "5536730d841c70c256991081bd4d8a9c28aaebcab9ac3cdd36ceb1aad896cfae",
    "openaimmlu_sociology": "1fa4ebb60178be200e3b8167e10fec0843964725a6be034e6893ae42d5dd1a3a",
    "openaimmlu_high_school_macroeconomics": "938c0435e322f454a5b1f26f1b53870141a9e311bbc95512cf307ecd007e66db",
    "openaimmlu_moral_scenarios": "c0158287c824e917d6d76a9d4a4e68a53af20dee7bb7c1d372a759546edef562",
    "openaimmlu_public_relations": "545ea7d7b3dbbe04d2c367cd70142c35e1ce585a3c3e4b9d9fa2290d1d25272f",
    "openaimmlu_us_foreign_policy": "cddc366ef735093ff1ab6d3660a19d52ae146b9ca18668d8a878be81466cb626",
    "openaimmlu_moral_disputes": "2b9fd83448202cb343ad8473f9d34194776e73f9fac0fad093610033039e0152",
    "openaimmlu_human_sexuality": "cd4281e8629dd63b57e11fff680c2813bfc156d0807d9a3424670422bb8a8f02",
    "openaimmlu_marketing": "85aaada41a32346c0dce6f252b7e5e50a1bce1641cab3ecf6e1590deb8927db4",
    "openaimmlu_jurisprudence": "ab9de498411479a47a892895a70b20948854fe8c8177f9851da339a984c534f0",
    "openaimmlu_logical_fallacies": "1c809bb030ca1d7256a741cd2f7b3719053d6387df5f89762fca7aa430374461",
    "openaimmlu_philosophy": "e6367d4cab84d33e5ba62f20bc52f72d630c4324c0d34628b546cf72a83eb94f",
    "openaimmlu_high_school_world_history": "f7e18a11fcc4e11b8c758d3227d7e7fc59157c9bd465ade0c8e4707cb3c76c2a",
    "openaimmlu_high_school_european_history": "f6ee95e6dc273ad3d18c110a69772161a9eb250ef81c3202a46228689c5e2071",
    "openaimmlu_prehistory": "88d6574515b52d900aab10f1f22d026fa33c8d910a6528acf3dac384d1e82b0a",
    "openaimmlu_high_school_us_history": "541952c75bfb8c256d813fdfc4d7707ad25448980fd70d39142ec03a15af9d0d",
    "openaimmlu_international_law": "c1acfc8203c4d2f4d5d9245685804c2b7406601dfd8106cc4fac985915559f52",
    "openaimmlu_world_religions": "9b68b777a6bc2b05efee57f75e87792c6b14f39464621c16e4c24b024aeb2630"
  },
  "model_source": "vllm",
  "model_name": "/tmp/7b-alpha-v1.27.2.25",
  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
  "system_instruction": null,
  "system_instruction_sha": null,
  "fewshot_as_multiturn": false,
  "chat_template": null,
  "chat_template_sha": null,
  "start_time": 4238.00553465,
  "end_time": 4722.113520369,
  "total_evaluation_time_seconds": "484.10798571899977"
}