Adding evaluation results

evaluation/ar/acva_5_shot.json

@@ -0,0 +1,119 @@
+{
+  "results": {
+    "acva": {
+      "alias": "acva",
+      "acc,none": 0.7746268656716417,
+      "acc_stderr,none": 0.004477269169728854,
+      "acc_norm,none": 0.7632606199770379,
+      "acc_norm_stderr,none": 0.004554991129754026
+    }
+  },
+  "group_subtasks": {
+    "acva": []
+  },
+  "configs": {
+    "acva": {
+      "task": "acva",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _format_subject(subject):\n        \n        arabic_words = subtasks_ar[subtasks.index(subject)]\n        return arabic_words\n    \n    def _generate_subject(doc):\n        subject = _format_subject(doc[\"id\"].split(\"-\")[0])\n\n        return subject\n        \n    def _process_docs(doc):\n        keys = [\"\u0635\u062d\",\n                 \"\u062e\u0637\u0623\"]\n        subject = _generate_subject(doc)\n        gold = keys.index(doc['answer'])\n        out_doc = {\n            \"id\": doc[\"id\"],\n            \"query\":  \"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" + doc[\"question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\",\n            \"choices\": keys,\n            \"gold\": gold,\n            \"subject\": subject,\n        }\n        \n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d' \u0623\u0648 '\u062e\u0637\u0623' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "acva": 0.0
+  },
+  "n-shot": {
+    "acva": 5
+  },
+  "higher_is_better": {
+    "acva": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "acva": {
+      "original": 8710,
+      "effective": 8710
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735662713.7617116,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "acva": "d007c508f0accdd697f549d7cbe7f960f1470c8f86f1a0969355a6ef33108edb"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 3374.021232778,
+  "end_time": 3578.563943596,
+  "total_evaluation_time_seconds": "204.54271081800016"
+}

evaluation/ar/ar_ifeval_0_shot.json

@@ -0,0 +1,142 @@
+{
+  "results": {
+    "ar_ifeval": {
+      "alias": "ar_ifeval",
+      "prompt_level_strict_acc,none": 0.31343283582089554,
+      "prompt_level_strict_acc_stderr,none": 0.020055655889994813,
+      "inst_level_strict_acc,none": 0.6764505119453925,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.3656716417910448,
+      "prompt_level_loose_acc_stderr,none": 0.020822161638297296,
+      "inst_level_loose_acc,none": 0.7051194539249147,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "ar_ifeval": []
+  },
+  "configs": {
+    "ar_ifeval": {
+      "task": "ar_ifeval",
+      "dataset_path": "lm_eval/tasks/ar_ifeval/ar_ifeval.py",
+      "dataset_name": "ar_ifeval",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "process_results": "def process_results(doc, results):\n\n    response = results[0]\n    out_strict = process_sample(doc, response, 'strict')\n    out_loose = process_sample(doc, response, 'loose')\n\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0
+      }
+    }
+  },
+  "versions": {
+    "ar_ifeval": 4.0
+  },
+  "n-shot": {
+    "ar_ifeval": 0
+  },
+  "higher_is_better": {
+    "ar_ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ar_ifeval": {
+      "original": 536,
+      "effective": 536
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739618378.981141,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "ar_ifeval": "d0db7903ef270d7dc54efe4e7713be0de9864fc3a36c901c6e5777a6a5f69aa9"
+  },
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1393068.333905473,
+  "end_time": 1397143.169266589,
+  "total_evaluation_time_seconds": "4074.8353611161"
+}

evaluation/ar/araMath_5_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "araMath": {
+      "alias": "araMath",
+      "acc,none": 0.6677685950413224,
+      "acc_stderr,none": 0.019165266705090528,
+      "acc_norm,none": 0.6677685950413224,
+      "acc_norm_stderr,none": 0.019165266705090528
+    }
+  },
+  "group_subtasks": {
+    "araMath": []
+  },
+  "configs": {
+    "araMath": {
+      "task": "araMath",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araMath/araMath.py",
+      "dataset_name": "araMath",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def remove_prefix(choice):\n            prefixes = [\"(A)\", \"(B)\", \"(C)\", \"(D)\"]\n            for prefix in prefixes:\n                if choice.startswith(prefix + \" \"):\n                    return choice[len(prefix) + 1:]  \n            return choice \n\n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"options\"])]\n            )\n\n            prompt = f\"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_en.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 \u0628\u064a\u0646 'A\u060c B\u060c C\u060c D' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "araMath": 0.0
+  },
+  "n-shot": {
+    "araMath": 5
+  },
+  "higher_is_better": {
+    "araMath": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araMath": {
+      "original": 605,
+      "effective": 605
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739618269.6292942,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "araMath": "e7f60b63c44ee90c76a61f37207fa1f812622b6662200911fcfd7dabe78ada66"
+  },
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1392959.193182268,
+  "end_time": 1393012.133225703,
+  "total_evaluation_time_seconds": "52.940043434966356"
+}

evaluation/ar/araPro_0_shot.json

@@ -0,0 +1,130 @@
+{
+  "results": {
+    "araPro": {
+      "alias": "araPro",
+      "acc,none": 0.6970605878824235,
+      "acc_stderr,none": 0.006498724870364006,
+      "acc_norm,none": 0.6970605878824235,
+      "acc_norm_stderr,none": 0.006498724870364006
+    }
+  },
+  "group_subtasks": {
+    "araPro": []
+  },
+  "configs": {
+    "araPro": {
+      "task": "araPro",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araPro/araPro.py",
+      "dataset_name": "araPro",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.replace('.', '') if '.' in choice[:2] else choice\n        \n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            \n            choice_num = ['choice1', 'choice2', 'choice3', 'choice4']\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(doc[choice_num[index]])}\\n\" for index, key in enumerate(keys)]\n            )\n\n            prompt = f\"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        #keys = [\"1\", \"2\", \"3\", \"4\"]\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\":  format_example(doc, keys), \n            \"choices\": keys,\n            \"gold\": doc[\"answer\"]-1,\n        }        \n\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question",
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "araPro": 2.0
+  },
+  "n-shot": {
+    "araPro": 0
+  },
+  "higher_is_better": {
+    "araPro": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araPro": {
+      "original": 5001,
+      "effective": 5001
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617164.0204737,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "araPro": "01340c360a1565c46298c4c24dd3fdfe1ea614c6eef6e4d4f021f1da83da2584"
+  },
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1391853.516943726,
+  "end_time": 1392050.054185297,
+  "total_evaluation_time_seconds": "196.5372415711172"
+}

evaluation/ar/etec_0_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "etec": {
+      "alias": "etec",
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.010854826817097195,
+      "acc_norm,none": 0.6666666666666666,
+      "acc_norm_stderr,none": 0.010854826817097195
+    }
+  },
+  "group_subtasks": {
+    "etec": []
+  },
+  "configs": {
+    "etec": {
+      "task": "etec",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/etec/etec.py",
+      "dataset_name": "etec",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n        print(doc[\"label\"])\n        keys_ar = [\"\u0623\", \"\u0628\", \"\u062c\", \"\u062f\"]\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n                \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": int(doc[\"label\"])-1,\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d\n ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "etec": 0.0
+  },
+  "n-shot": {
+    "etec": 0
+  },
+  "higher_is_better": {
+    "etec": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "etec": {
+      "original": 1887,
+      "effective": 1887
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617421.4265695,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "etec": "a0d87bf7eb82815b66ea544cb632aafb803526dee24b399f30fdc751be442b60"
+  },
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1392110.980523203,
+  "end_time": 1392198.883363127,
+  "total_evaluation_time_seconds": "87.90283992397599"
+}

evaluation/ar/exams_ar_5_shot.json

@@ -0,0 +1,121 @@
+{
+  "results": {
+    "exams_ar": {
+      "alias": "exams_ar",
+      "acc,none": 0.515828677839851,
+      "acc_stderr,none": 0.021585885942816244,
+      "acc_norm,none": 0.515828677839851,
+      "acc_norm_stderr,none": 0.021585885942816244
+    }
+  },
+  "group_subtasks": {
+    "exams_ar": []
+  },
+  "configs": {
+    "exams_ar": {
+      "task": "exams_ar",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/exams_ar",
+      "dataset_name": "exams_ar",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            \"\"\"\n            <prompt>\n            \u0633\u0624\u0627\u0644:\n            A. <choice1>\n            B. <choice2>\n            C. <choice3>\n            D. <choice4>\n            \u0627\u062c\u0627\u0628\u0629:\n            \"\"\"\n            \n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        def _format_subject(subject):\n            arabic_words = subtasks_ar[subtasks.index(subject)]\n            return arabic_words\n\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n    \n        subject = doc['id'].split(\"-\")[0]\n        description = f\"\ufed2\ufef4\ufee3\ufe8d \ufef2\ufee0\ufef3 \ufe84\ufeb4\ufe8c\ufedf\ufe93 \ufe8d\ufefc\ufea8\ufe98\ufef3\ufe8d\ufead \ufee2\ufee7 \ufee2\ufe98\ufecb\ufea9\ufea9 (\ufee2\ufecb \ufe8d\ufefa\ufe9f\ufe8e\ufe91\ufe8e\ufe97) \ufea1\ufeee\ufedf {_format_subject(subject)} \\n\" #\ufee2\ufee7 \ufed2\ufec0\ufee0\ufedb \ufe8e\ufea8\ufe97\ufead \ufe88\ufe9f\ufe8e\ufe91\ufe93 \ufeed\ufe8e\ufea3\ufea9\ufe93 \ufee2\ufee7 \ufe90\ufef4\ufee7 'A\u060c B\u060c C\u060c D' \ufea9\ufeee\ufee7 \ufeb5\ufeae\ufea3\\n\"\n\n        out_doc = {\n            \"idx\": doc[\"idx\"],\n            \"id\": doc[\"id\"],\n            'dsecription': description,\n            \"query\":  format_example(doc, keys), # \"Question: \" + doc[\"question\"]['stem'] + \"\\nAnswer:\",\n            \"choices\": keys,\n            \"gold\": [\"A\", \"B\", \"C\", \"D\"].index(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "description",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "exams_ar": 0.0
+  },
+  "n-shot": {
+    "exams_ar": 5
+  },
+  "higher_is_better": {
+    "exams_ar": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "exams_ar": {
+      "original": 537,
+      "effective": 537
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735662207.0830526,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "exams_ar": "b1561abd56354d570ac16bf64163b0ee8dc6c507234b05f678576b09c26c644a"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 2867.397536365,
+  "end_time": 2948.510496752,
+  "total_evaluation_time_seconds": "81.11296038699993"
+}

evaluation/ar/gat_0_shot.json

@@ -0,0 +1,549 @@
+{
+  "results": {
+    "gat": {
+      "acc,none": 0.4452527279568544,
+      "acc_stderr,none": 0.0038711388833064567,
+      "alias": "gat"
+    },
+    "gat_algebra": {
+      "alias": " - gat_algebra",
+      "acc,none": 0.40667903525046384,
+      "acc_stderr,none": 0.009463939247454995
+    },
+    "gat_analogy": {
+      "alias": " - gat_analogy",
+      "acc,none": 0.35919854280510016,
+      "acc_stderr,none": 0.009158766245747282
+    },
+    "gat_arithmetic": {
+      "alias": " - gat_arithmetic",
+      "acc,none": 0.40154582259845417,
+      "acc_stderr,none": 0.009406284814832203
+    },
+    "gat_association": {
+      "alias": " - gat_association",
+      "acc,none": 0.5464114832535886,
+      "acc_stderr,none": 0.015407801869520031
+    },
+    "gat_comparisons": {
+      "alias": " - gat_comparisons",
+      "acc,none": 0.34508196721311474,
+      "acc_stderr,none": 0.013616100682624904
+    },
+    "gat_completion": {
+      "alias": " - gat_completion",
+      "acc,none": 0.6057851239669422,
+      "acc_stderr,none": 0.014054411207805699
+    },
+    "gat_contextual": {
+      "alias": " - gat_contextual",
+      "acc,none": 0.3941717791411043,
+      "acc_stderr,none": 0.013537713096332765
+    },
+    "gat_geometry": {
+      "alias": " - gat_geometry",
+      "acc,none": 0.473972602739726,
+      "acc_stderr,none": 0.026171590093068537
+    },
+    "gat_reading": {
+      "alias": " - gat_reading",
+      "acc,none": 0.5727788279773157,
+      "acc_stderr,none": 0.009620311542503682
+    }
+  },
+  "groups": {
+    "gat": {
+      "acc,none": 0.4452527279568544,
+      "acc_stderr,none": 0.0038711388833064567,
+      "alias": "gat"
+    }
+  },
+  "group_subtasks": {
+    "gat": [
+      "gat_analogy",
+      "gat_association",
+      "gat_completion",
+      "gat_reading",
+      "gat_algebra",
+      "gat_arithmetic",
+      "gat_comparisons",
+      "gat_contextual",
+      "gat_geometry"
+    ]
+  },
+  "configs": {
+    "gat_algebra": {
+      "task": "gat_algebra",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_analogy": {
+      "task": "gat_analogy",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "analogy",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_arithmetic": {
+      "task": "gat_arithmetic",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "arithmetic",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_association": {
+      "task": "gat_association",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "association",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_comparisons": {
+      "task": "gat_comparisons",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "comparisons",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_completion": {
+      "task": "gat_completion",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "completion",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_contextual": {
+      "task": "gat_contextual",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "contextual",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "\u0627\u0648\u062c\u062f \u0627\u0644\u062e\u0637\u0623 \u0627\u0644\u0633\u064a\u0627\u0642\u064a \u0641\u064a \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0627\u0644\u062a\u0627\u0644\u064a\u0629 \u0645\u0646 \u0628\u064a\u0646 \u0627\u0644\u062e\u064a\u0627\u0631\u0627\u062a:",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_geometry": {
+      "task": "gat_geometry",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "geometry",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_reading": {
+      "task": "gat_reading",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "reading",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "gat": 0,
+    "gat_algebra": 0.0,
+    "gat_analogy": 0.0,
+    "gat_arithmetic": 0.0,
+    "gat_association": 0.0,
+    "gat_comparisons": 0.0,
+    "gat_completion": 0.0,
+    "gat_contextual": 0.0,
+    "gat_geometry": 0.0,
+    "gat_reading": 0.0
+  },
+  "n-shot": {
+    "gat_algebra": 0,
+    "gat_analogy": 0,
+    "gat_arithmetic": 0,
+    "gat_association": 0,
+    "gat_comparisons": 0,
+    "gat_completion": 0,
+    "gat_contextual": 0,
+    "gat_geometry": 0,
+    "gat_reading": 0
+  },
+  "higher_is_better": {
+    "gat": {
+      "acc": true
+    },
+    "gat_algebra": {
+      "acc": true
+    },
+    "gat_analogy": {
+      "acc": true
+    },
+    "gat_arithmetic": {
+      "acc": true
+    },
+    "gat_association": {
+      "acc": true
+    },
+    "gat_comparisons": {
+      "acc": true
+    },
+    "gat_completion": {
+      "acc": true
+    },
+    "gat_contextual": {
+      "acc": true
+    },
+    "gat_geometry": {
+      "acc": true
+    },
+    "gat_reading": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "gat_analogy": {
+      "original": 2745,
+      "effective": 2745
+    },
+    "gat_association": {
+      "original": 1045,
+      "effective": 1045
+    },
+    "gat_completion": {
+      "original": 1210,
+      "effective": 1210
+    },
+    "gat_reading": {
+      "original": 2645,
+      "effective": 2645
+    },
+    "gat_algebra": {
+      "original": 2695,
+      "effective": 2695
+    },
+    "gat_arithmetic": {
+      "original": 2717,
+      "effective": 2717
+    },
+    "gat_comparisons": {
+      "original": 1220,
+      "effective": 1220
+    },
+    "gat_contextual": {
+      "original": 1304,
+      "effective": 1304
+    },
+    "gat_geometry": {
+      "original": 365,
+      "effective": 365
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735664096.2650902,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "gat_analogy": "ede28dec097bfebe8a85a19fa27d001696858276df66254bdb70fc63231f1a83",
+    "gat_association": "5d82550d46c4f3cabf370185a8a23cc2eb5b08f1f0c5e210a8a712562a44bd08",
+    "gat_completion": "fc3c19dd7f1896696fec1bffc21182804c9b2f1fb8d8c882428a6bb4bb61e370",
+    "gat_reading": "93053b187a750d2e87f5488f2d0fda944f3da9195bb04d1c4dee9c4b56fa626a",
+    "gat_algebra": "77832c595eaaf156775c3dbb27da0915ef600ebf46a7113ae32a202b0359e8a6",
+    "gat_arithmetic": "6a498f75f5cc0ffd1b30f7a6293ba80d08f2a8876d5558d8e934bf57355ff0cc",
+    "gat_comparisons": "acb80c0ed8dd07e916a471189aef3a546efc289824b2cc50a32c11dc4c97c9c1",
+    "gat_contextual": "de063ed3b94011d74ee24a6532122c9d344fc15e42800db44f0849995a0bc37a",
+    "gat_geometry": "3e482885559a4404ee9e97556edc6e49959770a499f4ae2c58f18ad85b91a363"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 4756.376698655,
+  "end_time": 5124.76942052,
+  "total_evaluation_time_seconds": "368.39272186499966"
+}

evaluation/ar/moe_ien_mcq_0_shot.json

@@ -0,0 +1,127 @@
+{
+  "results": {
+    "moe_ien_mcq": {
+      "alias": "moe_ien_mcq",
+      "acc,none": 0.9177177177177177,
+      "acc_stderr,none": 0.002749455634736978,
+      "acc_norm,none": 0.9177177177177177,
+      "acc_norm_stderr,none": 0.002749455634736978
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_mcq": []
+  },
+  "configs": {
+    "moe_ien_mcq": {
+      "task": "moe_ien_mcq",
+      "dataset_path": "lm_eval/tasks/moe_ien_mcq/ien_moe_mcq.py",
+      "dataset_name": "moe_ien_mcq",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.split(\". \", 1)[1] if \". \" in choice else choice\n\n        def format_example(doc, keys):\n            question = doc[\"Question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"Choices\"])]\n                \n            )\n            prompt = f\"\\n\\n\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\"][0:len(doc[\"Choices\"])]\n        out_doc = {\n            \"Query\":  format_example(doc, keys), \n            \"Choices\": keys,\n            \"gold\": int(doc[\"Answer\"])-1, ## \n        }        \n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "Query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{Choices}}",
+      "description": "\u0641\u064a\u0645\u0627\u202f\u064a\u0644\u064a\u202f\u0623\u0633\u0626\u0644\u0629\u202f\u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631\u202f\u0645\u0646\u202f\u0645\u062a\u0639\u062f\u062f\u202f(\u0645\u0639\u202f\u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a)\u202f\u0641\u064a\u202f{{Subject}}",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_mcq": 0.0
+  },
+  "n-shot": {
+    "moe_ien_mcq": 0
+  },
+  "higher_is_better": {
+    "moe_ien_mcq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_mcq": {
+      "original": 9990,
+      "effective": 9990
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617571.8184838,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "moe_ien_mcq": "504533b140426f12c89d975ef421328fc89d69af8719c420a1bf897ed4724191"
+  },
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1392261.292633723,
+  "end_time": 1392626.942167409,
+  "total_evaluation_time_seconds": "365.64953368599527"
+}

evaluation/ar/moe_ien_tf_0_shot.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "moe_ien_tf": {
+      "alias": "moe_ien_tf",
+      "acc,none": 0.8294693456980937,
+      "acc_stderr,none": 0.004929073554117403,
+      "acc_norm,none": 0.8294693456980937,
+      "acc_norm_stderr,none": 0.004929073554117403
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_tf": []
+  },
+  "configs": {
+    "moe_ien_tf": {
+      "task": "moe_ien_tf",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/moe_ien_tf/moe_ien_tf.py",
+      "dataset_name": "moe_ien_tf",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        keys=[\"\u0635\u062d\u064a\u062d\u0629\",\n              \"\u062e\u0627\u0637\u0626\u0629\"\n              ]\n        #keys =[\"\u0635\u0648\u0627\u0628\",\n        #         \"\u062e\u0637\u0623\"]\n        target_key = int(doc[\"Answer\"])-1\n\n        out_doc = {\n            \"query\":  \"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" +doc[\"Question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\", \n            \"choices\": keys,\n            \"gold\": target_key,\n        }\n        return out_doc\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{Subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d\u064a\u062d\u0629' \u0623\u0648 '\u062e\u0627\u0637\u0626\u0629' \u062f\u0648\u0646 \u0634\u0631\u062d ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_tf": 2.0
+  },
+  "n-shot": {
+    "moe_ien_tf": 0
+  },
+  "higher_is_better": {
+    "moe_ien_tf": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_tf": {
+      "original": 5823,
+      "effective": 5823
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617995.3462336,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "moe_ien_tf": "8701a646f6ea8b9bb96c028f817fbeabfb9031580f5054368b43d14d4a5a1270"
+  },
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1392684.818305694,
+  "end_time": 1392900.218863064,
+  "total_evaluation_time_seconds": "215.40055736992508"
+}