Adding evaluation results

evaluation/ar/acva_5_shot.json

@@ -0,0 +1,119 @@
+{
+  "results": {
+    "acva": {
+      "alias": "acva",
+      "acc,none": 0.7746268656716417,
+      "acc_stderr,none": 0.004477269169728854,
+      "acc_norm,none": 0.7632606199770379,
+      "acc_norm_stderr,none": 0.004554991129754026
+    }
+  },
+  "group_subtasks": {
+    "acva": []
+  },
+  "configs": {
+    "acva": {
+      "task": "acva",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _format_subject(subject):\n        \n        arabic_words = subtasks_ar[subtasks.index(subject)]\n        return arabic_words\n    \n    def _generate_subject(doc):\n        subject = _format_subject(doc[\"id\"].split(\"-\")[0])\n\n        return subject\n        \n    def _process_docs(doc):\n        keys = [\"\u0635\u062d\",\n                 \"\u062e\u0637\u0623\"]\n        subject = _generate_subject(doc)\n        gold = keys.index(doc['answer'])\n        out_doc = {\n            \"id\": doc[\"id\"],\n            \"query\":  \"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" + doc[\"question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\",\n            \"choices\": keys,\n            \"gold\": gold,\n            \"subject\": subject,\n        }\n        \n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d' \u0623\u0648 '\u062e\u0637\u0623' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "acva": 0.0
+  },
+  "n-shot": {
+    "acva": 5
+  },
+  "higher_is_better": {
+    "acva": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "acva": {
+      "original": 8710,
+      "effective": 8710
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735662713.7617116,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "acva": "d007c508f0accdd697f549d7cbe7f960f1470c8f86f1a0969355a6ef33108edb"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 3374.021232778,
+  "end_time": 3578.563943596,
+  "total_evaluation_time_seconds": "204.54271081800016"
+}

evaluation/ar/araMath_v2_5_shot.json

@@ -0,0 +1,123 @@
+{
+  "results": {
+    "araMath_v2": {
+      "alias": "araMath_v2",
+      "acc,none": 0.655,
+      "acc_stderr,none": 0.019423021295885703,
+      "acc_norm,none": 0.655,
+      "acc_norm_stderr,none": 0.019423021295885703
+    }
+  },
+  "group_subtasks": {
+    "araMath_v2": []
+  },
+  "configs": {
+    "araMath_v2": {
+      "task": "araMath_v2",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araMath_v2",
+      "dataset_name": "araMath_v2",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def remove_prefix(choice):\n            prefixes = [\"(A)\", \"(B)\", \"(C)\", \"(D)\"]\n            for prefix in prefixes:\n                if choice.startswith(prefix + \" \"):\n                    return choice[len(prefix) + 1:]  \n            return choice \n\n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"options\"])]\n            )\n\n            prompt = f\"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_en.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 \u0628\u064a\u0646 'A\u060c B\u060c C\u060c D' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "araMath_v2": 0.0
+  },
+  "n-shot": {
+    "araMath_v2": 5
+  },
+  "higher_is_better": {
+    "araMath_v2": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araMath_v2": {
+      "original": 600,
+      "effective": 600
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "parallelize=True,pretrained=/ALLaM-7B-Instruct,trust_remote_code=True",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "5e10e017",
+  "date": 1736774062.2964265,
+  "pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.86\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
+  "transformers_version": "4.38.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 72495.638596469,
+  "end_time": 72556.179139124,
+  "total_evaluation_time_seconds": "60.54054265499872"
+}

evaluation/ar/etec_0_shot.json

@@ -0,0 +1,121 @@
+{
+  "results": {
+    "etec": {
+      "alias": "etec",
+      "acc,none": 0.6680761099365751,
+      "acc_stderr,none": 0.010828952839616325,
+      "acc_norm,none": 0.6680761099365751,
+      "acc_norm_stderr,none": 0.010828952839616325
+    }
+  },
+  "group_subtasks": {
+    "etec": []
+  },
+  "configs": {
+    "etec": {
+      "task": "etec",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/etec",
+      "dataset_name": "etec",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_ar = [\"\u0623\", \"\u0628\", \"\u062c\", \"\u062f\"]\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n                \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_ar.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d\n ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "etec": 0.0
+  },
+  "n-shot": {
+    "etec": 0
+  },
+  "higher_is_better": {
+    "etec": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "etec": {
+      "original": 1892,
+      "effective": 1892
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735662950.8344455,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "etec": "8937d87b09ed63604ed9f64a02b8ba75ee9c43b9acebd5dd58a797e187916bbf"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 3611.154007204,
+  "end_time": 3697.095375819,
+  "total_evaluation_time_seconds": "85.94136861499965"
+}

evaluation/ar/exams_ar_5_shot.json

@@ -0,0 +1,121 @@
+{
+  "results": {
+    "exams_ar": {
+      "alias": "exams_ar",
+      "acc,none": 0.515828677839851,
+      "acc_stderr,none": 0.021585885942816244,
+      "acc_norm,none": 0.515828677839851,
+      "acc_norm_stderr,none": 0.021585885942816244
+    }
+  },
+  "group_subtasks": {
+    "exams_ar": []
+  },
+  "configs": {
+    "exams_ar": {
+      "task": "exams_ar",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/exams_ar",
+      "dataset_name": "exams_ar",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            \"\"\"\n            <prompt>\n            \u0633\u0624\u0627\u0644:\n            A. <choice1>\n            B. <choice2>\n            C. <choice3>\n            D. <choice4>\n            \u0627\u062c\u0627\u0628\u0629:\n            \"\"\"\n            \n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        def _format_subject(subject):\n            arabic_words = subtasks_ar[subtasks.index(subject)]\n            return arabic_words\n\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n    \n        subject = doc['id'].split(\"-\")[0]\n        description = f\"\ufed2\ufef4\ufee3\ufe8d \ufef2\ufee0\ufef3 \ufe84\ufeb4\ufe8c\ufedf\ufe93 \ufe8d\ufefc\ufea8\ufe98\ufef3\ufe8d\ufead \ufee2\ufee7 \ufee2\ufe98\ufecb\ufea9\ufea9 (\ufee2\ufecb \ufe8d\ufefa\ufe9f\ufe8e\ufe91\ufe8e\ufe97) \ufea1\ufeee\ufedf {_format_subject(subject)} \\n\" #\ufee2\ufee7 \ufed2\ufec0\ufee0\ufedb \ufe8e\ufea8\ufe97\ufead \ufe88\ufe9f\ufe8e\ufe91\ufe93 \ufeed\ufe8e\ufea3\ufea9\ufe93 \ufee2\ufee7 \ufe90\ufef4\ufee7 'A\u060c B\u060c C\u060c D' \ufea9\ufeee\ufee7 \ufeb5\ufeae\ufea3\\n\"\n\n        out_doc = {\n            \"idx\": doc[\"idx\"],\n            \"id\": doc[\"id\"],\n            'dsecription': description,\n            \"query\":  format_example(doc, keys), # \"Question: \" + doc[\"question\"]['stem'] + \"\\nAnswer:\",\n            \"choices\": keys,\n            \"gold\": [\"A\", \"B\", \"C\", \"D\"].index(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "description",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "exams_ar": 0.0
+  },
+  "n-shot": {
+    "exams_ar": 5
+  },
+  "higher_is_better": {
+    "exams_ar": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "exams_ar": {
+      "original": 537,
+      "effective": 537
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735662207.0830526,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "exams_ar": "b1561abd56354d570ac16bf64163b0ee8dc6c507234b05f678576b09c26c644a"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 2867.397536365,
+  "end_time": 2948.510496752,
+  "total_evaluation_time_seconds": "81.11296038699993"
+}

evaluation/ar/gat_0_shot.json

@@ -0,0 +1,549 @@
+{
+  "results": {
+    "gat": {
+      "acc,none": 0.4452527279568544,
+      "acc_stderr,none": 0.0038711388833064567,
+      "alias": "gat"
+    },
+    "gat_algebra": {
+      "alias": " - gat_algebra",
+      "acc,none": 0.40667903525046384,
+      "acc_stderr,none": 0.009463939247454995
+    },
+    "gat_analogy": {
+      "alias": " - gat_analogy",
+      "acc,none": 0.35919854280510016,
+      "acc_stderr,none": 0.009158766245747282
+    },
+    "gat_arithmetic": {
+      "alias": " - gat_arithmetic",
+      "acc,none": 0.40154582259845417,
+      "acc_stderr,none": 0.009406284814832203
+    },
+    "gat_association": {
+      "alias": " - gat_association",
+      "acc,none": 0.5464114832535886,
+      "acc_stderr,none": 0.015407801869520031
+    },
+    "gat_comparisons": {
+      "alias": " - gat_comparisons",
+      "acc,none": 0.34508196721311474,
+      "acc_stderr,none": 0.013616100682624904
+    },
+    "gat_completion": {
+      "alias": " - gat_completion",
+      "acc,none": 0.6057851239669422,
+      "acc_stderr,none": 0.014054411207805699
+    },
+    "gat_contextual": {
+      "alias": " - gat_contextual",
+      "acc,none": 0.3941717791411043,
+      "acc_stderr,none": 0.013537713096332765
+    },
+    "gat_geometry": {
+      "alias": " - gat_geometry",
+      "acc,none": 0.473972602739726,
+      "acc_stderr,none": 0.026171590093068537
+    },
+    "gat_reading": {
+      "alias": " - gat_reading",
+      "acc,none": 0.5727788279773157,
+      "acc_stderr,none": 0.009620311542503682
+    }
+  },
+  "groups": {
+    "gat": {
+      "acc,none": 0.4452527279568544,
+      "acc_stderr,none": 0.0038711388833064567,
+      "alias": "gat"
+    }
+  },
+  "group_subtasks": {
+    "gat": [
+      "gat_analogy",
+      "gat_association",
+      "gat_completion",
+      "gat_reading",
+      "gat_algebra",
+      "gat_arithmetic",
+      "gat_comparisons",
+      "gat_contextual",
+      "gat_geometry"
+    ]
+  },
+  "configs": {
+    "gat_algebra": {
+      "task": "gat_algebra",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_analogy": {
+      "task": "gat_analogy",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "analogy",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_arithmetic": {
+      "task": "gat_arithmetic",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "arithmetic",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_association": {
+      "task": "gat_association",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "association",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_comparisons": {
+      "task": "gat_comparisons",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "comparisons",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_completion": {
+      "task": "gat_completion",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "completion",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_contextual": {
+      "task": "gat_contextual",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "contextual",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "\u0627\u0648\u062c\u062f \u0627\u0644\u062e\u0637\u0623 \u0627\u0644\u0633\u064a\u0627\u0642\u064a \u0641\u064a \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0627\u0644\u062a\u0627\u0644\u064a\u0629 \u0645\u0646 \u0628\u064a\u0646 \u0627\u0644\u062e\u064a\u0627\u0631\u0627\u062a:",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_geometry": {
+      "task": "gat_geometry",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "geometry",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_reading": {
+      "task": "gat_reading",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "reading",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "gat": 0,
+    "gat_algebra": 0.0,
+    "gat_analogy": 0.0,
+    "gat_arithmetic": 0.0,
+    "gat_association": 0.0,
+    "gat_comparisons": 0.0,
+    "gat_completion": 0.0,
+    "gat_contextual": 0.0,
+    "gat_geometry": 0.0,
+    "gat_reading": 0.0
+  },
+  "n-shot": {
+    "gat_algebra": 0,
+    "gat_analogy": 0,
+    "gat_arithmetic": 0,
+    "gat_association": 0,
+    "gat_comparisons": 0,
+    "gat_completion": 0,
+    "gat_contextual": 0,
+    "gat_geometry": 0,
+    "gat_reading": 0
+  },
+  "higher_is_better": {
+    "gat": {
+      "acc": true
+    },
+    "gat_algebra": {
+      "acc": true
+    },
+    "gat_analogy": {
+      "acc": true
+    },
+    "gat_arithmetic": {
+      "acc": true
+    },
+    "gat_association": {
+      "acc": true
+    },
+    "gat_comparisons": {
+      "acc": true
+    },
+    "gat_completion": {
+      "acc": true
+    },
+    "gat_contextual": {
+      "acc": true
+    },
+    "gat_geometry": {
+      "acc": true
+    },
+    "gat_reading": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "gat_analogy": {
+      "original": 2745,
+      "effective": 2745
+    },
+    "gat_association": {
+      "original": 1045,
+      "effective": 1045
+    },
+    "gat_completion": {
+      "original": 1210,
+      "effective": 1210
+    },
+    "gat_reading": {
+      "original": 2645,
+      "effective": 2645
+    },
+    "gat_algebra": {
+      "original": 2695,
+      "effective": 2695
+    },
+    "gat_arithmetic": {
+      "original": 2717,
+      "effective": 2717
+    },
+    "gat_comparisons": {
+      "original": 1220,
+      "effective": 1220
+    },
+    "gat_contextual": {
+      "original": 1304,
+      "effective": 1304
+    },
+    "gat_geometry": {
+      "original": 365,
+      "effective": 365
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735664096.2650902,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "gat_analogy": "ede28dec097bfebe8a85a19fa27d001696858276df66254bdb70fc63231f1a83",
+    "gat_association": "5d82550d46c4f3cabf370185a8a23cc2eb5b08f1f0c5e210a8a712562a44bd08",
+    "gat_completion": "fc3c19dd7f1896696fec1bffc21182804c9b2f1fb8d8c882428a6bb4bb61e370",
+    "gat_reading": "93053b187a750d2e87f5488f2d0fda944f3da9195bb04d1c4dee9c4b56fa626a",
+    "gat_algebra": "77832c595eaaf156775c3dbb27da0915ef600ebf46a7113ae32a202b0359e8a6",
+    "gat_arithmetic": "6a498f75f5cc0ffd1b30f7a6293ba80d08f2a8876d5558d8e934bf57355ff0cc",
+    "gat_comparisons": "acb80c0ed8dd07e916a471189aef3a546efc289824b2cc50a32c11dc4c97c9c1",
+    "gat_contextual": "de063ed3b94011d74ee24a6532122c9d344fc15e42800db44f0849995a0bc37a",
+    "gat_geometry": "3e482885559a4404ee9e97556edc6e49959770a499f4ae2c58f18ad85b91a363"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 4756.376698655,
+  "end_time": 5124.76942052,
+  "total_evaluation_time_seconds": "368.39272186499966"
+}

evaluation/ar/moe_ien_mcq_0_shot.json

@@ -0,0 +1,118 @@
+{
+  "results": {
+    "moe_ien_mcq": {
+      "alias": "moe_ien_mcq",
+      "acc,none": 0.9154154154154154,
+      "acc_stderr,none": 0.0027841569543517694,
+      "acc_norm,none": 0.9154154154154154,
+      "acc_norm_stderr,none": 0.0027841569543517694
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_mcq": []
+  },
+  "configs": {
+    "moe_ien_mcq": {
+      "task": "moe_ien_mcq",
+      "dataset_path": "lm_eval/tasks/moe_ien_mcq/ien_moe_mcq.py",
+      "dataset_name": "moe_ien_mcq",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.split(\". \", 1)[1] if \". \" in choice else choice\n\n        def format_example(doc, keys):\n            question = doc[\"Question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"Choices\"])]\n                \n            )\n            prompt = f\"\\n\\n\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\"][0:len(doc[\"Choices\"])]\n        out_doc = {\n            \"Query\":  format_example(doc, keys), \n            \"Choices\": keys,\n            \"gold\": int(doc[\"Answer\"])-1, ## \n            \"Speciality\": doc['Speciality']\n        }        \n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "Query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{Choices}}",
+      "description": "\u0641\u064a\u0645\u0627\u202f\u064a\u0644\u064a\u202f\u0623\u0633\u0626\u0644\u0629\u202f\u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631\u202f\u0645\u0646\u202f\u0645\u062a\u0639\u062f\u062f\u202f(\u0645\u0639\u202f\u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a)\u202f\u0641\u064a\u202f{{Speciality}}",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_mcq": 0.0
+  },
+  "n-shot": {
+    "moe_ien_mcq": 0
+  },
+  "higher_is_better": {
+    "moe_ien_mcq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_mcq": {
+      "original": 9990,
+      "effective": 9990
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735663068.5370116,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "moe_ien_mcq": "554899322e5b78369683b10024d90dc868f768d310530589a6167541e8f9d594"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 3728.910211786,
+  "end_time": 3947.718352837,
+  "total_evaluation_time_seconds": "218.8081410509999"
+}

evaluation/ar/moe_ien_tf_0_shot.json

@@ -0,0 +1,119 @@
+{
+  "results": {
+    "moe_ien_tf": {
+      "alias": "moe_ien_tf",
+      "acc,none": 0.8557082967729356,
+      "acc_stderr,none": 0.0034697209254064324,
+      "acc_norm,none": 0.8557082967729356,
+      "acc_norm_stderr,none": 0.0034697209254064324
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_tf": []
+  },
+  "configs": {
+    "moe_ien_tf": {
+      "task": "moe_ien_tf",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/moe_ien_tf",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        keys=[\"\u0635\u062d\u064a\u062d\u0629\",\n              \"\u062e\u0627\u0637\u0626\u0629\"\n              ]\n        #keys =[\"\u0635\u0648\u0627\u0628\",\n        #         \"\u062e\u0637\u0623\"]\n        target_key = int(doc[\"Answer\"])-1\n\n        out_doc = {\n            \"query\":  \"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" +doc[\"Question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\", \n            \"choices\": keys,\n            \"gold\": target_key,\n        }\n        return out_doc\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{Speciality}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d\u064a\u062d\u0629' \u0623\u0648 '\u062e\u0627\u0637\u0626\u0629' \u062f\u0648\u0646 \u0634\u0631\u062d ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_tf": 0.0
+  },
+  "n-shot": {
+    "moe_ien_tf": 0
+  },
+  "higher_is_better": {
+    "moe_ien_tf": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_tf": {
+      "original": 10257,
+      "effective": 10257
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735663321.6141305,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "moe_ien_tf": "bf29d6fb290755a9dc7c5aaf1263e4cd1e9d82a62085aa6279661d8b84fd5ab6"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 3981.83990155,
+  "end_time": 4097.740745391,
+  "total_evaluation_time_seconds": "115.9008438410001"
+}

evaluation/ar/sdaia_mcq_0_shot.json

@@ -0,0 +1,121 @@
+{
+  "results": {
+    "sdaia_mcq": {
+      "alias": "sdaia_mcq",
+      "acc,none": 0.739021329987453,
+      "acc_stderr,none": 0.011003303841318535,
+      "acc_norm,none": 0.739021329987453,
+      "acc_norm_stderr,none": 0.011003303841318535
+    }
+  },
+  "group_subtasks": {
+    "sdaia_mcq": []
+  },
+  "configs": {
+    "sdaia_mcq": {
+      "task": "sdaia_mcq",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/sdaia_mcq/sdaia_mcq.py",
+      "dataset_name": "sdaia_mcq",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.replace('.', '') if '.' in choice[:2] else choice\n        \n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            \n            choice_num = ['choice1', 'choice2', 'choice3', 'choice4']\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(doc[choice_num[index]])}\\n\" for index, key in enumerate(keys)]\n            )\n\n            prompt = f\"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        #keys = [\"1\", \"2\", \"3\", \"4\"]\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\":  format_example(doc, keys), \n            \"choices\": keys,\n            \"gold\": doc[\"answer\"]-1,\n        }        \n\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "sdaia_mcq": 0.0
+  },
+  "n-shot": {
+    "sdaia_mcq": 0
+  },
+  "higher_is_better": {
+    "sdaia_mcq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "sdaia_mcq": {
+      "original": 1594,
+      "effective": 1594
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735663470.0459642,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "sdaia_mcq": "c69b252ba97d5f402c302aadb4d06d0293774e38250e701b1d0c7984fa47dd24"
+  },
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 4130.43217211,
+  "end_time": 4204.747507708,
+  "total_evaluation_time_seconds": "74.31533559800027"
+}

evaluation/en/agieval_0_shot.json

@@ -0,0 +1,1108 @@
+{
+  "results": {
+    "agieval": {
+      "acc,none": 0.4175133043057571,
+      "acc_stderr,none": 0.0050080978184310855,
+      "alias": "agieval"
+    },
+    "agieval_aqua_rat": {
+      "alias": " - agieval_aqua_rat",
+      "acc,none": 0.28346456692913385,
+      "acc_stderr,none": 0.028334004921307634,
+      "acc_norm,none": 0.28346456692913385,
+      "acc_norm_stderr,none": 0.02833400492130763
+    },
+    "agieval_gaokao_biology": {
+      "alias": " - agieval_gaokao_biology",
+      "acc,none": 0.319047619047619,
+      "acc_stderr,none": 0.03224133248962465,
+      "acc_norm,none": 0.3619047619047619,
+      "acc_norm_stderr,none": 0.03324043951593503
+    },
+    "agieval_gaokao_chemistry": {
+      "alias": " - agieval_gaokao_chemistry",
+      "acc,none": 0.33816425120772947,
+      "acc_stderr,none": 0.03296137710480074,
+      "acc_norm,none": 0.32367149758454106,
+      "acc_norm_stderr,none": 0.03259848850179343
+    },
+    "agieval_gaokao_chinese": {
+      "alias": " - agieval_gaokao_chinese",
+      "acc,none": 0.3089430894308943,
+      "acc_stderr,none": 0.02951977938940491,
+      "acc_norm,none": 0.3048780487804878,
+      "acc_norm_stderr,none": 0.029411050550756265
+    },
+    "agieval_gaokao_english": {
+      "alias": " - agieval_gaokao_english",
+      "acc,none": 0.7352941176470589,
+      "acc_stderr,none": 0.025261691219729494,
+      "acc_norm,none": 0.7516339869281046,
+      "acc_norm_stderr,none": 0.02473998135511359
+    },
+    "agieval_gaokao_geography": {
+      "alias": " - agieval_gaokao_geography",
+      "acc,none": 0.4472361809045226,
+      "acc_stderr,none": 0.035335047084973224,
+      "acc_norm,none": 0.4472361809045226,
+      "acc_norm_stderr,none": 0.035335047084973224
+    },
+    "agieval_gaokao_history": {
+      "alias": " - agieval_gaokao_history",
+      "acc,none": 0.43829787234042555,
+      "acc_stderr,none": 0.03243618636108102,
+      "acc_norm,none": 0.39574468085106385,
+      "acc_norm_stderr,none": 0.03196758697835362
+    },
+    "agieval_gaokao_mathcloze": {
+      "alias": " - agieval_gaokao_mathcloze",
+      "acc,none": 0.0423728813559322,
+      "acc_stderr,none": 0.018622984668462274
+    },
+    "agieval_gaokao_mathqa": {
+      "alias": " - agieval_gaokao_mathqa",
+      "acc,none": 0.27635327635327633,
+      "acc_stderr,none": 0.02390350500312722,
+      "acc_norm,none": 0.2678062678062678,
+      "acc_norm_stderr,none": 0.023669514493780283
+    },
+    "agieval_gaokao_physics": {
+      "alias": " - agieval_gaokao_physics",
+      "acc,none": 0.36,
+      "acc_stderr,none": 0.034026297840400156,
+      "acc_norm,none": 0.355,
+      "acc_norm_stderr,none": 0.03392091008070853
+    },
+    "agieval_jec_qa_ca": {
+      "alias": " - agieval_jec_qa_ca",
+      "acc,none": 0.5025025025025025,
+      "acc_stderr,none": 0.015827025208013587,
+      "acc_norm,none": 0.4924924924924925,
+      "acc_norm_stderr,none": 0.015825439216141556
+    },
+    "agieval_jec_qa_kd": {
+      "alias": " - agieval_jec_qa_kd",
+      "acc,none": 0.568,
+      "acc_stderr,none": 0.01567232023733621,
+      "acc_norm,none": 0.518,
+      "acc_norm_stderr,none": 0.015809045699406728
+    },
+    "agieval_logiqa_en": {
+      "alias": " - agieval_logiqa_en",
+      "acc,none": 0.42242703533026116,
+      "acc_stderr,none": 0.01937414753071922,
+      "acc_norm,none": 0.42857142857142855,
+      "acc_norm_stderr,none": 0.01941046344247875
+    },
+    "agieval_logiqa_zh": {
+      "alias": " - agieval_logiqa_zh",
+      "acc,none": 0.38095238095238093,
+      "acc_stderr,none": 0.01904761904761897,
+      "acc_norm,none": 0.3717357910906298,
+      "acc_norm_stderr,none": 0.01895534398822881
+    },
+    "agieval_lsat_ar": {
+      "alias": " - agieval_lsat_ar",
+      "acc,none": 0.17391304347826086,
+      "acc_stderr,none": 0.02504731738604971,
+      "acc_norm,none": 0.1826086956521739,
+      "acc_norm_stderr,none": 0.02553042195273417
+    },
+    "agieval_lsat_lr": {
+      "alias": " - agieval_lsat_lr",
+      "acc,none": 0.696078431372549,
+      "acc_stderr,none": 0.0203868890006473,
+      "acc_norm,none": 0.6647058823529411,
+      "acc_norm_stderr,none": 0.020925162390233513
+    },
+    "agieval_lsat_rc": {
+      "alias": " - agieval_lsat_rc",
+      "acc,none": 0.5836431226765799,
+      "acc_stderr,none": 0.030111969407536524,
+      "acc_norm,none": 0.5464684014869888,
+      "acc_norm_stderr,none": 0.03041017404275444
+    },
+    "agieval_math": {
+      "alias": " - agieval_math",
+      "acc,none": 0.086,
+      "acc_stderr,none": 0.008870325962594766
+    },
+    "agieval_sat_en": {
+      "alias": " - agieval_sat_en",
+      "acc,none": 0.8155339805825242,
+      "acc_stderr,none": 0.02708958103176961,
+      "acc_norm,none": 0.7912621359223301,
+      "acc_norm_stderr,none": 0.028384671935185523
+    },
+    "agieval_sat_en_without_passage": {
+      "alias": " - agieval_sat_en_without_passage",
+      "acc,none": 0.44660194174757284,
+      "acc_stderr,none": 0.03472179658263948,
+      "acc_norm,none": 0.4174757281553398,
+      "acc_norm_stderr,none": 0.034442581739193366
+    },
+    "agieval_sat_math": {
+      "alias": " - agieval_sat_math",
+      "acc,none": 0.38636363636363635,
+      "acc_stderr,none": 0.03290270539316666,
+      "acc_norm,none": 0.37272727272727274,
+      "acc_norm_stderr,none": 0.0326739568483895
+    }
+  },
+  "groups": {
+    "agieval": {
+      "acc,none": 0.4175133043057571,
+      "acc_stderr,none": 0.0050080978184310855,
+      "alias": "agieval"
+    }
+  },
+  "group_subtasks": {
+    "agieval": [
+      "agieval_gaokao_biology",
+      "agieval_gaokao_chemistry",
+      "agieval_gaokao_chinese",
+      "agieval_gaokao_geography",
+      "agieval_gaokao_history",
+      "agieval_gaokao_mathcloze",
+      "agieval_gaokao_mathqa",
+      "agieval_gaokao_physics",
+      "agieval_jec_qa_ca",
+      "agieval_jec_qa_kd",
+      "agieval_logiqa_zh",
+      "agieval_aqua_rat",
+      "agieval_gaokao_english",
+      "agieval_logiqa_en",
+      "agieval_lsat_ar",
+      "agieval_lsat_lr",
+      "agieval_lsat_rc",
+      "agieval_math",
+      "agieval_sat_en_without_passage",
+      "agieval_sat_en",
+      "agieval_sat_math"
+    ]
+  },
+  "configs": {
+    "agieval_aqua_rat": {
+      "task": "agieval_aqua_rat",
+      "dataset_path": "hails/agieval-aqua-rat",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_biology": {
+      "task": "agieval_gaokao_biology",
+      "dataset_path": "hails/agieval-gaokao-biology",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_chemistry": {
+      "task": "agieval_gaokao_chemistry",
+      "dataset_path": "hails/agieval-gaokao-chemistry",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_chinese": {
+      "task": "agieval_gaokao_chinese",
+      "dataset_path": "hails/agieval-gaokao-chinese",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_english": {
+      "task": "agieval_gaokao_english",
+      "dataset_path": "hails/agieval-gaokao-english",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_geography": {
+      "task": "agieval_gaokao_geography",
+      "dataset_path": "hails/agieval-gaokao-geography",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_history": {
+      "task": "agieval_gaokao_history",
+      "dataset_path": "hails/agieval-gaokao-history",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_mathcloze": {
+      "task": "agieval_gaokao_mathcloze",
+      "dataset_path": "hails/agieval-gaokao-mathcloze",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{answer}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidate = results[0]\n\n    gold = doc[\"answer\"]\n\n    if not gold:\n        print(doc, candidate, gold)\n    if is_equiv(candidate, gold):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"acc\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "max_gen_toks": 32,
+        "do_sample": false,
+        "temperature": 0.0,
+        "until": [
+          "Q:"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_mathqa": {
+      "task": "agieval_gaokao_mathqa",
+      "dataset_path": "hails/agieval-gaokao-mathqa",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_gaokao_physics": {
+      "task": "agieval_gaokao_physics",
+      "dataset_path": "hails/agieval-gaokao-physics",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_jec_qa_ca": {
+      "task": "agieval_jec_qa_ca",
+      "dataset_path": "hails/agieval-jec-qa-ca",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_jec_qa_kd": {
+      "task": "agieval_jec_qa_kd",
+      "dataset_path": "hails/agieval-jec-qa-kd",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_logiqa_en": {
+      "task": "agieval_logiqa_en",
+      "dataset_path": "hails/agieval-logiqa-en",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_logiqa_zh": {
+      "task": "agieval_logiqa_zh",
+      "dataset_path": "hails/agieval-logiqa-zh",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_lsat_ar": {
+      "task": "agieval_lsat_ar",
+      "dataset_path": "hails/agieval-lsat-ar",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_lsat_lr": {
+      "task": "agieval_lsat_lr",
+      "dataset_path": "hails/agieval-lsat-lr",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_lsat_rc": {
+      "task": "agieval_lsat_rc",
+      "dataset_path": "hails/agieval-lsat-rc",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_math": {
+      "task": "agieval_math",
+      "dataset_path": "hails/agieval-math",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{answer}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidate = results[0]\n\n    gold = doc[\"answer\"]\n\n    if not gold:\n        print(doc, candidate, gold)\n    if is_equiv(candidate, gold):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"acc\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "max_gen_toks": 32,
+        "do_sample": false,
+        "temperature": 0.0,
+        "until": [
+          "Q:"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_sat_en": {
+      "task": "agieval_sat_en",
+      "dataset_path": "hails/agieval-sat-en",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_sat_en_without_passage": {
+      "task": "agieval_sat_en_without_passage",
+      "dataset_path": "hails/agieval-sat-en-without-passage",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "agieval_sat_math": {
+      "task": "agieval_sat_math",
+      "dataset_path": "hails/agieval-sat-math",
+      "test_split": "test",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{gold}}",
+      "doc_to_choice": "{{choices}}",
+      "process_results": "def process_results_mcqa(doc, results):\n    results = [result[0] for result in results]\n\n    gold = doc[\"gold\"]\n\n    acc = 1.0 if int(np.argmax(results)) in gold else 0.0\n    completion_len = np.array([float(len(i)) for i in doc[\"choices\"]])\n    acc_norm = 1.0 if int(np.argmax(results / completion_len)) in gold else 0.0\n\n    return {\n        \"acc\": acc,\n        \"acc_norm\": acc_norm,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "agieval": 0.0,
+    "agieval_aqua_rat": 1.0,
+    "agieval_gaokao_biology": 1.0,
+    "agieval_gaokao_chemistry": 1.0,
+    "agieval_gaokao_chinese": 1.0,
+    "agieval_gaokao_english": 1.0,
+    "agieval_gaokao_geography": 1.0,
+    "agieval_gaokao_history": 1.0,
+    "agieval_gaokao_mathcloze": 1.0,
+    "agieval_gaokao_mathqa": 1.0,
+    "agieval_gaokao_physics": 1.0,
+    "agieval_jec_qa_ca": 1.0,
+    "agieval_jec_qa_kd": 1.0,
+    "agieval_logiqa_en": 1.0,
+    "agieval_logiqa_zh": 1.0,
+    "agieval_lsat_ar": 1.0,
+    "agieval_lsat_lr": 1.0,
+    "agieval_lsat_rc": 1.0,
+    "agieval_math": 1.0,
+    "agieval_sat_en": 1.0,
+    "agieval_sat_en_without_passage": 1.0,
+    "agieval_sat_math": 1.0
+  },
+  "n-shot": {
+    "agieval_aqua_rat": 0,
+    "agieval_gaokao_biology": 0,
+    "agieval_gaokao_chemistry": 0,
+    "agieval_gaokao_chinese": 0,
+    "agieval_gaokao_english": 0,
+    "agieval_gaokao_geography": 0,
+    "agieval_gaokao_history": 0,
+    "agieval_gaokao_mathcloze": 0,
+    "agieval_gaokao_mathqa": 0,
+    "agieval_gaokao_physics": 0,
+    "agieval_jec_qa_ca": 0,
+    "agieval_jec_qa_kd": 0,
+    "agieval_logiqa_en": 0,
+    "agieval_logiqa_zh": 0,
+    "agieval_lsat_ar": 0,
+    "agieval_lsat_lr": 0,
+    "agieval_lsat_rc": 0,
+    "agieval_math": 0,
+    "agieval_sat_en": 0,
+    "agieval_sat_en_without_passage": 0,
+    "agieval_sat_math": 0
+  },
+  "higher_is_better": {
+    "agieval": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_aqua_rat": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_gaokao_biology": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_gaokao_chemistry": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_gaokao_chinese": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_gaokao_english": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_gaokao_geography": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_gaokao_history": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_gaokao_mathcloze": {
+      "acc": true
+    },
+    "agieval_gaokao_mathqa": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_gaokao_physics": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_jec_qa_ca": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_jec_qa_kd": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_logiqa_en": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_logiqa_zh": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_lsat_ar": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_lsat_lr": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_lsat_rc": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_math": {
+      "acc": true
+    },
+    "agieval_sat_en": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_sat_en_without_passage": {
+      "acc": true,
+      "acc_norm": true
+    },
+    "agieval_sat_math": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "agieval_gaokao_biology": {
+      "original": 210,
+      "effective": 210
+    },
+    "agieval_gaokao_chemistry": {
+      "original": 207,
+      "effective": 207
+    },
+    "agieval_gaokao_chinese": {
+      "original": 246,
+      "effective": 246
+    },
+    "agieval_gaokao_geography": {
+      "original": 199,
+      "effective": 199
+    },
+    "agieval_gaokao_history": {
+      "original": 235,
+      "effective": 235
+    },
+    "agieval_gaokao_mathcloze": {
+      "original": 118,
+      "effective": 118
+    },
+    "agieval_gaokao_mathqa": {
+      "original": 351,
+      "effective": 351
+    },
+    "agieval_gaokao_physics": {
+      "original": 200,
+      "effective": 200
+    },
+    "agieval_jec_qa_ca": {
+      "original": 999,
+      "effective": 999
+    },
+    "agieval_jec_qa_kd": {
+      "original": 1000,
+      "effective": 1000
+    },
+    "agieval_logiqa_zh": {
+      "original": 651,
+      "effective": 651
+    },
+    "agieval_aqua_rat": {
+      "original": 254,
+      "effective": 254
+    },
+    "agieval_gaokao_english": {
+      "original": 306,
+      "effective": 306
+    },
+    "agieval_logiqa_en": {
+      "original": 651,
+      "effective": 651
+    },
+    "agieval_lsat_ar": {
+      "original": 230,
+      "effective": 230
+    },
+    "agieval_lsat_lr": {
+      "original": 510,
+      "effective": 510
+    },
+    "agieval_lsat_rc": {
+      "original": 269,
+      "effective": 269
+    },
+    "agieval_math": {
+      "original": 1000,
+      "effective": 1000
+    },
+    "agieval_sat_en_without_passage": {
+      "original": 206,
+      "effective": 206
+    },
+    "agieval_sat_en": {
+      "original": 206,
+      "effective": 206
+    },
+    "agieval_sat_math": {
+      "original": 220,
+      "effective": 220
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735956443.5467572,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.90\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 23113.003334144,
+  "end_time": 23735.631059832,
+  "total_evaluation_time_seconds": "622.6277256880021"
+}

evaluation/en/gpqa_main_n_shot_0_shot.json

@@ -0,0 +1,123 @@
+{
+  "results": {
+    "gpqa_main_n_shot": {
+      "alias": "gpqa_main_n_shot",
+      "acc,none": 0.22098214285714285,
+      "acc_stderr,none": 0.01962449705224272,
+      "acc_norm,none": 0.22098214285714285,
+      "acc_norm_stderr,none": 0.01962449705224272
+    }
+  },
+  "group_subtasks": {
+    "gpqa_main_n_shot": []
+  },
+  "configs": {
+    "gpqa_main_n_shot": {
+      "task": "gpqa_main_n_shot",
+      "tag": "gpqa",
+      "dataset_path": "Idavidrein/gpqa",
+      "dataset_name": "gpqa_main",
+      "training_split": "train",
+      "validation_split": "train",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        choices = [\n            preprocess(doc[\"Incorrect Answer 1\"]),\n            preprocess(doc[\"Incorrect Answer 2\"]),\n            preprocess(doc[\"Incorrect Answer 3\"]),\n            preprocess(doc[\"Correct Answer\"]),\n        ]\n\n        rng.shuffle(choices)\n        correct_answer_index = choices.index(preprocess(doc[\"Correct Answer\"]))\n\n        out_doc = {\n            \"choice1\": choices[0],\n            \"choice2\": choices[1],\n            \"choice3\": choices[2],\n            \"choice4\": choices[3],\n            \"answer\": f\"({chr(65 + correct_answer_index)})\",\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "Question: {{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer:",
+      "doc_to_target": "answer",
+      "doc_to_choice": [
+        "(A)",
+        "(B)",
+        "(C)",
+        "(D)"
+      ],
+      "description": "Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "gpqa_main_n_shot": 2.0
+  },
+  "n-shot": {
+    "gpqa_main_n_shot": 0
+  },
+  "higher_is_better": {
+    "gpqa_main_n_shot": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "gpqa_main_n_shot": {
+      "original": 448,
+      "effective": 448
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1734941625.7186382,
+  "pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": "18b53334e0494773088a01c543e721a58f958e0d",
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 66386.780938561,
+  "end_time": 66441.200832346,
+  "total_evaluation_time_seconds": "54.41989378500148"
+}

evaluation/en/gsm8k_5_shot.json

@@ -0,0 +1,153 @@
+{
+  "results": {
+    "gsm8k": {
+      "alias": "gsm8k",
+      "exact_match,strict-match": 0.620166793025019,
+      "exact_match_stderr,strict-match": 0.013368818096960501,
+      "exact_match,flexible-extract": 0.623199393479909,
+      "exact_match_stderr,flexible-extract": 0.01334785875782916
+    }
+  },
+  "group_subtasks": {
+    "gsm8k": []
+  },
+  "configs": {
+    "gsm8k": {
+      "task": "gsm8k",
+      "tag": [
+        "math_word_problems"
+      ],
+      "dataset_path": "gsm8k",
+      "dataset_name": "main",
+      "training_split": "train",
+      "test_split": "test",
+      "fewshot_split": "train",
+      "doc_to_text": "Question: {{question}}\nAnswer:",
+      "doc_to_target": "{{answer}}",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": false,
+          "regexes_to_ignore": [
+            ",",
+            "\\$",
+            "(?s).*#### ",
+            "\\.$"
+          ]
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Question:",
+          "</s>",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "strict-match",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "#### (\\-?[0-9\\.\\,]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        },
+        {
+          "name": "flexible-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "group_select": -1,
+              "regex_pattern": "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 3.0
+      }
+    }
+  },
+  "versions": {
+    "gsm8k": 3.0
+  },
+  "n-shot": {
+    "gsm8k": 5
+  },
+  "higher_is_better": {
+    "gsm8k": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "gsm8k": {
+      "original": 1319,
+      "effective": 1319
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735956272.5546186,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.90\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 22942.105525776,
+  "end_time": 23057.183463458,
+  "total_evaluation_time_seconds": "115.07793768199917"
+}

evaluation/en/hellaswag_0_shot.json

@@ -0,0 +1,118 @@
+{
+  "results": {
+    "hellaswag": {
+      "alias": "hellaswag",
+      "acc,none": 0.5771758613821948,
+      "acc_stderr,none": 0.00492998369279507,
+      "acc_norm,none": 0.7625970922127067,
+      "acc_norm_stderr,none": 0.0042462162299898715
+    }
+  },
+  "group_subtasks": {
+    "hellaswag": []
+  },
+  "configs": {
+    "hellaswag": {
+      "task": "hellaswag",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735957117.4813576,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.90\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 23786.943776673,
+  "end_time": 23998.958401018,
+  "total_evaluation_time_seconds": "212.0146243449999"
+}

evaluation/en/hendrycks_ethics_0_shot.json

@@ -0,0 +1,307 @@
+{
+  "results": {
+    "ethics_cm": {
+      "alias": "ethics_cm",
+      "acc,none": 0.7392535392535392,
+      "acc_stderr,none": 0.007044761695158352
+    },
+    "ethics_deontology": {
+      "alias": "ethics_deontology",
+      "acc,none": 0.5786985539488321,
+      "acc_stderr,none": 0.00823518246369769
+    },
+    "ethics_justice": {
+      "alias": "ethics_justice",
+      "acc,none": 0.771819526627219,
+      "acc_stderr,none": 0.00807186884011459
+    },
+    "ethics_utilitarianism": {
+      "alias": "ethics_utilitarianism",
+      "acc,none": 0.6541181364392679,
+      "acc_stderr,none": 0.006860486742815242
+    },
+    "ethics_virtue": {
+      "alias": "ethics_virtue",
+      "acc,none": 0.9147738693467337,
+      "acc_stderr,none": 0.003959044383441912
+    }
+  },
+  "group_subtasks": {
+    "ethics_deontology": [],
+    "ethics_virtue": [],
+    "ethics_cm": [],
+    "ethics_utilitarianism": [],
+    "ethics_justice": []
+  },
+  "configs": {
+    "ethics_cm": {
+      "task": "ethics_cm",
+      "tag": [
+        "hendrycks_ethics"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_ethics",
+      "dataset_name": "commonsense",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "doc_to_text": "{{input}}\nQuestion: Is this wrong?\nAnswer:",
+      "doc_to_target": "label",
+      "doc_to_choice": [
+        "no",
+        "yes"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "ethics_deontology": {
+      "task": "ethics_deontology",
+      "tag": [
+        "hendrycks_ethics"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_ethics",
+      "dataset_name": "deontology",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}} {{excuse.rstrip()}}\"\nAnswer:",
+      "doc_to_target": "label",
+      "doc_to_choice": [
+        "unreasonable",
+        "reasonable"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "ethics_justice": {
+      "task": "ethics_justice",
+      "tag": [
+        "hendrycks_ethics"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_ethics",
+      "dataset_name": "justice",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "doc_to_text": "Question: Would most people believe this reasonable or unreasonable to say? \"{{scenario}}\"\nAnswer:",
+      "doc_to_target": "label",
+      "doc_to_choice": [
+        "unreasonable",
+        "reasonable"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "ethics_utilitarianism": {
+      "task": "ethics_utilitarianism",
+      "tag": [
+        "hendrycks_ethics"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_ethics",
+      "dataset_name": "utilitarianism",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "doc_to_text": "def doc_to_text(doc) -> str:\n    doc = _preproc_doc(doc)\n    return f\"Scenario 1: {doc['scenarios'][0]}\\nScenario 2: {doc['scenarios'][1]}\\nQuestion: Is Scenario 1 preferable?\\nAnswer:\"\n",
+      "doc_to_target": "def doc_to_target(doc):\n    doc = _preproc_doc(doc)\n    return doc[\"label\"]\n",
+      "doc_to_choice": [
+        "no",
+        "yes"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "ethics_virtue": {
+      "task": "ethics_virtue",
+      "tag": [
+        "hendrycks_ethics"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_ethics",
+      "dataset_name": "virtue",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "doc_to_text": "Sentence: {{scenario}}\nQuestion: Does the character in this sentence exhibit the trait \"{{trait}}\"?\nAnswer:",
+      "doc_to_target": "label",
+      "doc_to_choice": [
+        "no",
+        "yes"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc"
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "ethics_cm": 1.0,
+    "ethics_deontology": 1.0,
+    "ethics_justice": 1.0,
+    "ethics_utilitarianism": 1.0,
+    "ethics_virtue": 1.0
+  },
+  "n-shot": {
+    "ethics_cm": 0,
+    "ethics_deontology": 0,
+    "ethics_justice": 0,
+    "ethics_utilitarianism": 0,
+    "ethics_virtue": 0
+  },
+  "higher_is_better": {
+    "ethics_cm": {
+      "acc": true
+    },
+    "ethics_deontology": {
+      "acc": true
+    },
+    "ethics_justice": {
+      "acc": true
+    },
+    "ethics_utilitarianism": {
+      "acc": true
+    },
+    "ethics_virtue": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "ethics_justice": {
+      "original": 2704,
+      "effective": 2704
+    },
+    "ethics_utilitarianism": {
+      "original": 4808,
+      "effective": 4808
+    },
+    "ethics_cm": {
+      "original": 3885,
+      "effective": 3885
+    },
+    "ethics_virtue": {
+      "original": 4975,
+      "effective": 4975
+    },
+    "ethics_deontology": {
+      "original": 3596,
+      "effective": 3596
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735957382.509422,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.90\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 24051.95882374,
+  "end_time": 24251.353762318,
+  "total_evaluation_time_seconds": "199.3949385779997"
+}

evaluation/en/ifeval_0_shot.json

@@ -0,0 +1,132 @@
+{
+  "results": {
+    "ifeval": {
+      "alias": "ifeval",
+      "prompt_level_strict_acc,none": 0.37707948243992606,
+      "prompt_level_strict_acc_stderr,none": 0.020856233918528456,
+      "inst_level_strict_acc,none": 0.486810551558753,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.41404805914972276,
+      "prompt_level_loose_acc_stderr,none": 0.021196272552471213,
+      "inst_level_loose_acc,none": 0.5239808153477218,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "ifeval": []
+  },
+  "configs": {
+    "ifeval": {
+      "task": "ifeval",
+      "dataset_path": "google/IFEval",
+      "test_split": "train",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "process_results": "def process_results(doc, results):\n    inp = InputExample(\n        key=doc[\"key\"],\n        instruction_id_list=doc[\"instruction_id_list\"],\n        prompt=doc[\"prompt\"],\n        kwargs=doc[\"kwargs\"],\n    )\n    response = results[0]\n\n    out_strict = test_instruction_following_strict(inp, response)\n    out_loose = test_instruction_following_loose(inp, response)\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0
+      }
+    }
+  },
+  "versions": {
+    "ifeval": 4.0
+  },
+  "n-shot": {
+    "ifeval": 0
+  },
+  "higher_is_better": {
+    "ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ifeval": {
+      "original": 541,
+      "effective": 541
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735955103.211484,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.90\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 21772.672146886,
+  "end_time": 21897.362057308,
+  "total_evaluation_time_seconds": "124.68991042199923"
+}

evaluation/en/minerva_math_4_shot.json

@@ -0,0 +1,525 @@
+{
+  "results": {
+    "minerva_math": {
+      "exact_match,none": 0.1742,
+      "exact_match_stderr,none": 0.005167735460596966,
+      "alias": "minerva_math"
+    },
+    "minerva_math_algebra": {
+      "alias": " - minerva_math_algebra",
+      "exact_match,none": 0.2443133951137321,
+      "exact_match_stderr,none": 0.012476769647814658
+    },
+    "minerva_math_counting_and_prob": {
+      "alias": " - minerva_math_counting_and_prob",
+      "exact_match,none": 0.16666666666666666,
+      "exact_match_stderr,none": 0.01713575252401387
+    },
+    "minerva_math_geometry": {
+      "alias": " - minerva_math_geometry",
+      "exact_match,none": 0.11899791231732777,
+      "exact_match_stderr,none": 0.014809629428535889
+    },
+    "minerva_math_intermediate_algebra": {
+      "alias": " - minerva_math_intermediate_algebra",
+      "exact_match,none": 0.058693244739756366,
+      "exact_match_stderr,none": 0.00782629796703524
+    },
+    "minerva_math_num_theory": {
+      "alias": " - minerva_math_num_theory",
+      "exact_match,none": 0.11481481481481481,
+      "exact_match_stderr,none": 0.013731616019404622
+    },
+    "minerva_math_prealgebra": {
+      "alias": " - minerva_math_prealgebra",
+      "exact_match,none": 0.3409873708381171,
+      "exact_match_stderr,none": 0.016071499145682847
+    },
+    "minerva_math_precalc": {
+      "alias": " - minerva_math_precalc",
+      "exact_match,none": 0.06043956043956044,
+      "exact_match_stderr,none": 0.010207626216646911
+    }
+  },
+  "groups": {
+    "minerva_math": {
+      "exact_match,none": 0.1742,
+      "exact_match_stderr,none": 0.005167735460596966,
+      "alias": "minerva_math"
+    }
+  },
+  "group_subtasks": {
+    "minerva_math": [
+      "minerva_math_algebra",
+      "minerva_math_counting_and_prob",
+      "minerva_math_geometry",
+      "minerva_math_intermediate_algebra",
+      "minerva_math_num_theory",
+      "minerva_math_prealgebra",
+      "minerva_math_precalc"
+    ]
+  },
+  "configs": {
+    "minerva_math_algebra": {
+      "task": "minerva_math_algebra",
+      "tag": [
+        "math_word_problems"
+      ],
+      "group": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"exact_match\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x146d9c03c820>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "minerva_math_counting_and_prob": {
+      "task": "minerva_math_counting_and_prob",
+      "tag": [
+        "math_word_problems"
+      ],
+      "group": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "counting_and_probability",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"exact_match\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x146d9c04e830>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "minerva_math_geometry": {
+      "task": "minerva_math_geometry",
+      "tag": [
+        "math_word_problems"
+      ],
+      "group": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "geometry",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"exact_match\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x146d9c04c1f0>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "minerva_math_intermediate_algebra": {
+      "task": "minerva_math_intermediate_algebra",
+      "tag": [
+        "math_word_problems"
+      ],
+      "group": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "intermediate_algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"exact_match\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x146d9c0eecb0>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "minerva_math_num_theory": {
+      "task": "minerva_math_num_theory",
+      "tag": [
+        "math_word_problems"
+      ],
+      "group": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "number_theory",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"exact_match\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x146d9c0ec040>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "minerva_math_prealgebra": {
+      "task": "minerva_math_prealgebra",
+      "tag": [
+        "math_word_problems"
+      ],
+      "group": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "prealgebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"exact_match\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x146d996368c0>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "minerva_math_precalc": {
+      "task": "minerva_math_precalc",
+      "tag": [
+        "math_word_problems"
+      ],
+      "group": [
+        "math_word_problems"
+      ],
+      "dataset_path": "EleutherAI/hendrycks_math",
+      "dataset_name": "precalculus",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc: dict) -> dict:\n        out_doc = {\n            \"problem\": doc[\"problem\"],\n            \"solution\": doc[\"solution\"],\n            \"answer\": normalize_final_answer(\n                remove_boxed(last_boxed_only_string(doc[\"solution\"]))\n            ),\n        }\n        if getattr(doc, \"few_shot\", None) is not None:\n            out_doc[\"few_shot\"] = True\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "def doc_to_text(doc: dict) -> str:\n    return \"Problem:\" + \"\\n\" + doc[\"problem\"] + \"\\n\\n\" + \"Solution:\"\n",
+      "doc_to_target": "{{answer if few_shot is undefined else solution}}",
+      "process_results": "def process_results(doc: dict, results: List[str]) -> Dict[str, int]:\n    candidates = results[0]\n\n    unnormalized_answer = get_unnormalized_answer(candidates)\n    answer = normalize_final_answer(unnormalized_answer)\n\n    if is_equiv(answer, doc[\"answer\"]):\n        retval = 1\n    else:\n        retval = 0\n\n    results = {\n        \"exact_match\": retval,\n    }\n    return results\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "samples": "<function list_fewshot_samples at 0x146d995cb490>"
+      },
+      "num_fewshot": 4,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "Problem:"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "minerva_math": 1.0,
+    "minerva_math_algebra": 1.0,
+    "minerva_math_counting_and_prob": 1.0,
+    "minerva_math_geometry": 1.0,
+    "minerva_math_intermediate_algebra": 1.0,
+    "minerva_math_num_theory": 1.0,
+    "minerva_math_prealgebra": 1.0,
+    "minerva_math_precalc": 1.0
+  },
+  "n-shot": {
+    "minerva_math_algebra": 4,
+    "minerva_math_counting_and_prob": 4,
+    "minerva_math_geometry": 4,
+    "minerva_math_intermediate_algebra": 4,
+    "minerva_math_num_theory": 4,
+    "minerva_math_prealgebra": 4,
+    "minerva_math_precalc": 4
+  },
+  "higher_is_better": {
+    "minerva_math": {
+      "exact_match": true
+    },
+    "minerva_math_algebra": {
+      "exact_match": true
+    },
+    "minerva_math_counting_and_prob": {
+      "exact_match": true
+    },
+    "minerva_math_geometry": {
+      "exact_match": true
+    },
+    "minerva_math_intermediate_algebra": {
+      "exact_match": true
+    },
+    "minerva_math_num_theory": {
+      "exact_match": true
+    },
+    "minerva_math_prealgebra": {
+      "exact_match": true
+    },
+    "minerva_math_precalc": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "minerva_math_algebra": {
+      "original": 1187,
+      "effective": 1187
+    },
+    "minerva_math_counting_and_prob": {
+      "original": 474,
+      "effective": 474
+    },
+    "minerva_math_geometry": {
+      "original": 479,
+      "effective": 479
+    },
+    "minerva_math_intermediate_algebra": {
+      "original": 903,
+      "effective": 903
+    },
+    "minerva_math_num_theory": {
+      "original": 540,
+      "effective": 540
+    },
+    "minerva_math_prealgebra": {
+      "original": 871,
+      "effective": 871
+    },
+    "minerva_math_precalc": {
+      "original": 546,
+      "effective": 546
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "auto",
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735683439.646248,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 29617.613485255,
+  "end_time": 33957.45925665,
+  "total_evaluation_time_seconds": "4339.845771395001"
+}

evaluation/en/mmlu_pro_5_shot.json

@@ -0,0 +1,1088 @@
+{
+  "results": {
+    "mmlu_pro": {
+      "exact_match,custom-extract": 0.3042719414893617,
+      "exact_match_stderr,custom-extract": 0.00404763190810295,
+      "alias": "mmlu_pro"
+    },
+    "mmlu_pro_biology": {
+      "alias": " - biology",
+      "exact_match,custom-extract": 0.5788005578800558,
+      "exact_match_stderr,custom-extract": 0.01845235719744687
+    },
+    "mmlu_pro_business": {
+      "alias": " - business",
+      "exact_match,custom-extract": 0.2915082382762991,
+      "exact_match_stderr,custom-extract": 0.016189361099463357
+    },
+    "mmlu_pro_chemistry": {
+      "alias": " - chemistry",
+      "exact_match,custom-extract": 0.14752650176678445,
+      "exact_match_stderr,custom-extract": 0.010544941212928488
+    },
+    "mmlu_pro_computer_science": {
+      "alias": " - computer_science",
+      "exact_match,custom-extract": 0.2975609756097561,
+      "exact_match_stderr,custom-extract": 0.022606360476532427
+    },
+    "mmlu_pro_economics": {
+      "alias": " - economics",
+      "exact_match,custom-extract": 0.44549763033175355,
+      "exact_match_stderr,custom-extract": 0.017118299286531986
+    },
+    "mmlu_pro_engineering": {
+      "alias": " - engineering",
+      "exact_match,custom-extract": 0.17337461300309598,
+      "exact_match_stderr,custom-extract": 0.012167726609185038
+    },
+    "mmlu_pro_health": {
+      "alias": " - health",
+      "exact_match,custom-extract": 0.3753056234718826,
+      "exact_match_stderr,custom-extract": 0.0169400741062406
+    },
+    "mmlu_pro_history": {
+      "alias": " - history",
+      "exact_match,custom-extract": 0.3438320209973753,
+      "exact_match_stderr,custom-extract": 0.024366260232577264
+    },
+    "mmlu_pro_law": {
+      "alias": " - law",
+      "exact_match,custom-extract": 0.21525885558583105,
+      "exact_match_stderr,custom-extract": 0.012392170573599742
+    },
+    "mmlu_pro_math": {
+      "alias": " - math",
+      "exact_match,custom-extract": 0.26350851221317545,
+      "exact_match_stderr,custom-extract": 0.011989865356312482
+    },
+    "mmlu_pro_other": {
+      "alias": " - other",
+      "exact_match,custom-extract": 0.38203463203463206,
+      "exact_match_stderr,custom-extract": 0.015993097507618206
+    },
+    "mmlu_pro_philosophy": {
+      "alias": " - philosophy",
+      "exact_match,custom-extract": 0.2865731462925852,
+      "exact_match_stderr,custom-extract": 0.02026178957298461
+    },
+    "mmlu_pro_physics": {
+      "alias": " - physics",
+      "exact_match,custom-extract": 0.20323325635103925,
+      "exact_match_stderr,custom-extract": 0.01116929190053331
+    },
+    "mmlu_pro_psychology": {
+      "alias": " - psychology",
+      "exact_match,custom-extract": 0.49122807017543857,
+      "exact_match_stderr,custom-extract": 0.017708182870812612
+    }
+  },
+  "groups": {
+    "mmlu_pro": {
+      "exact_match,custom-extract": 0.3042719414893617,
+      "exact_match_stderr,custom-extract": 0.00404763190810295,
+      "alias": "mmlu_pro"
+    }
+  },
+  "group_subtasks": {
+    "mmlu_pro": [
+      "mmlu_pro_biology",
+      "mmlu_pro_business",
+      "mmlu_pro_chemistry",
+      "mmlu_pro_computer_science",
+      "mmlu_pro_economics",
+      "mmlu_pro_engineering",
+      "mmlu_pro_health",
+      "mmlu_pro_history",
+      "mmlu_pro_law",
+      "mmlu_pro_math",
+      "mmlu_pro_other",
+      "mmlu_pro_philosophy",
+      "mmlu_pro_physics",
+      "mmlu_pro_psychology"
+    ]
+  },
+  "configs": {
+    "mmlu_pro_biology": {
+      "task": "mmlu_pro_biology",
+      "task_alias": "biology",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541d3696c0>, subject='biology')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541d36a710>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about biology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541d369240>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_business": {
+      "task": "mmlu_pro_business",
+      "task_alias": "business",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541d3683a0>, subject='business')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541d369d80>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about business. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541d36b910>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_chemistry": {
+      "task": "mmlu_pro_chemistry",
+      "task_alias": "chemistry",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541d3681f0>, subject='chemistry')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541d36a200>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about chemistry. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541d369900>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_computer_science": {
+      "task": "mmlu_pro_computer_science",
+      "task_alias": "computer_science",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541d368040>, subject='computer science')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541d3680d0>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about computer science. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541d368dc0>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_economics": {
+      "task": "mmlu_pro_economics",
+      "task_alias": "economics",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cf66f80>, subject='economics')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf66830>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about economics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf66b00>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_engineering": {
+      "task": "mmlu_pro_engineering",
+      "task_alias": "engineering",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cf641f0>, subject='engineering')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf653f0>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about engineering. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf67f40>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_health": {
+      "task": "mmlu_pro_health",
+      "task_alias": "health",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cf65f30>, subject='health')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf65b40>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about health. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf65e10>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_history": {
+      "task": "mmlu_pro_history",
+      "task_alias": "history",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cf67d00>, subject='history')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf66710>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about history. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf64820>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_law": {
+      "task": "mmlu_pro_law",
+      "task_alias": "law",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cf65bd0>, subject='law')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf66a70>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about law. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf66320>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_math": {
+      "task": "mmlu_pro_math",
+      "task_alias": "math",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cf64b80>, subject='math')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf66dd0>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about math. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf66c20>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_other": {
+      "task": "mmlu_pro_other",
+      "task_alias": "other",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cf64d30>, subject='other')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf66560>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about other. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf65c60>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_philosophy": {
+      "task": "mmlu_pro_philosophy",
+      "task_alias": "philosophy",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cf64940>, subject='philosophy')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf65750>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about philosophy. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cf64e50>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_physics": {
+      "task": "mmlu_pro_physics",
+      "task_alias": "physics",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x14541cfa3eb0>, subject='physics')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cfa3be0>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about physics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x14541cfa3d90>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    },
+    "mmlu_pro_psychology": {
+      "task": "mmlu_pro_psychology",
+      "task_alias": "psychology",
+      "dataset_path": "TIGER-Lab/MMLU-Pro",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "functools.partial(<function process_docs at 0x1454204afb50>, subject='psychology')",
+      "doc_to_text": "functools.partial(<function format_cot_example at 0x1454204afbe0>, including_answer=False)",
+      "doc_to_target": "answer",
+      "description": "The following are multiple choice questions (with answers) about psychology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "first_n",
+        "doc_to_text": "functools.partial(<function format_cot_example at 0x1454204afd00>, including_answer=True)",
+        "doc_to_target": ""
+      },
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "exact_match",
+          "aggregation": "mean",
+          "higher_is_better": true,
+          "ignore_case": true,
+          "ignore_punctuation": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [
+          "</s>",
+          "Q:",
+          "<|im_end|>"
+        ],
+        "do_sample": false,
+        "temperature": 0.0
+      },
+      "repeats": 1,
+      "filter_list": [
+        {
+          "name": "custom-extract",
+          "filter": [
+            {
+              "function": "regex",
+              "regex_pattern": "answer is \\(?([ABCDEFGHIJ])\\)?"
+            },
+            {
+              "function": "take_first"
+            }
+          ]
+        }
+      ],
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "mmlu_pro": 2.0,
+    "mmlu_pro_biology": 1.0,
+    "mmlu_pro_business": 1.0,
+    "mmlu_pro_chemistry": 1.0,
+    "mmlu_pro_computer_science": 1.0,
+    "mmlu_pro_economics": 1.0,
+    "mmlu_pro_engineering": 1.0,
+    "mmlu_pro_health": 1.0,
+    "mmlu_pro_history": 1.0,
+    "mmlu_pro_law": 1.0,
+    "mmlu_pro_math": 1.0,
+    "mmlu_pro_other": 1.0,
+    "mmlu_pro_philosophy": 1.0,
+    "mmlu_pro_physics": 1.0,
+    "mmlu_pro_psychology": 1.0
+  },
+  "n-shot": {
+    "mmlu_pro_biology": 5,
+    "mmlu_pro_business": 5,
+    "mmlu_pro_chemistry": 5,
+    "mmlu_pro_computer_science": 5,
+    "mmlu_pro_economics": 5,
+    "mmlu_pro_engineering": 5,
+    "mmlu_pro_health": 5,
+    "mmlu_pro_history": 5,
+    "mmlu_pro_law": 5,
+    "mmlu_pro_math": 5,
+    "mmlu_pro_other": 5,
+    "mmlu_pro_philosophy": 5,
+    "mmlu_pro_physics": 5,
+    "mmlu_pro_psychology": 5
+  },
+  "higher_is_better": {
+    "mmlu_pro": {
+      "exact_match": true
+    },
+    "mmlu_pro_biology": {
+      "exact_match": true
+    },
+    "mmlu_pro_business": {
+      "exact_match": true
+    },
+    "mmlu_pro_chemistry": {
+      "exact_match": true
+    },
+    "mmlu_pro_computer_science": {
+      "exact_match": true
+    },
+    "mmlu_pro_economics": {
+      "exact_match": true
+    },
+    "mmlu_pro_engineering": {
+      "exact_match": true
+    },
+    "mmlu_pro_health": {
+      "exact_match": true
+    },
+    "mmlu_pro_history": {
+      "exact_match": true
+    },
+    "mmlu_pro_law": {
+      "exact_match": true
+    },
+    "mmlu_pro_math": {
+      "exact_match": true
+    },
+    "mmlu_pro_other": {
+      "exact_match": true
+    },
+    "mmlu_pro_philosophy": {
+      "exact_match": true
+    },
+    "mmlu_pro_physics": {
+      "exact_match": true
+    },
+    "mmlu_pro_psychology": {
+      "exact_match": true
+    }
+  },
+  "n-samples": {
+    "mmlu_pro_biology": {
+      "original": 717,
+      "effective": 717
+    },
+    "mmlu_pro_business": {
+      "original": 789,
+      "effective": 789
+    },
+    "mmlu_pro_chemistry": {
+      "original": 1132,
+      "effective": 1132
+    },
+    "mmlu_pro_computer_science": {
+      "original": 410,
+      "effective": 410
+    },
+    "mmlu_pro_economics": {
+      "original": 844,
+      "effective": 844
+    },
+    "mmlu_pro_engineering": {
+      "original": 969,
+      "effective": 969
+    },
+    "mmlu_pro_health": {
+      "original": 818,
+      "effective": 818
+    },
+    "mmlu_pro_history": {
+      "original": 381,
+      "effective": 381
+    },
+    "mmlu_pro_law": {
+      "original": 1101,
+      "effective": 1101
+    },
+    "mmlu_pro_math": {
+      "original": 1351,
+      "effective": 1351
+    },
+    "mmlu_pro_other": {
+      "original": 924,
+      "effective": 924
+    },
+    "mmlu_pro_philosophy": {
+      "original": 499,
+      "effective": 499
+    },
+    "mmlu_pro_physics": {
+      "original": 1299,
+      "effective": 1299
+    },
+    "mmlu_pro_psychology": {
+      "original": 798,
+      "effective": 798
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735955547.4293072,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.90\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 22216.794737072,
+  "end_time": 22732.624102917,
+  "total_evaluation_time_seconds": "515.829365845002"
+}

evaluation/en/truthfulqa_mc2_0_shot.json

@@ -0,0 +1,108 @@
+{
+  "results": {
+    "truthfulqa_mc2": {
+      "alias": "truthfulqa_mc2",
+      "acc,none": 0.4667466051524712,
+      "acc_stderr,none": 0.015605585169281691
+    }
+  },
+  "group_subtasks": {
+    "truthfulqa_mc2": []
+  },
+  "configs": {
+    "truthfulqa_mc2": {
+      "task": "truthfulqa_mc2",
+      "tag": [
+        "truthfulqa"
+      ],
+      "dataset_path": "truthful_qa",
+      "dataset_name": "multiple_choice",
+      "validation_split": "validation",
+      "doc_to_text": "{% set prompt_qa = 'Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.'%}{{prompt_qa + '\n\nQ: ' + question + '\nA:'}}",
+      "doc_to_target": 0,
+      "doc_to_choice": "{{mc2_targets.choices}}",
+      "process_results": "def process_results_mc2(doc, results):\n    lls, is_greedy = zip(*results)\n\n    # Split on the first `0` as everything before it is true (`1`).\n    split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n    # Compute the normalized probability mass for the correct answer.\n    ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n    p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n    p_true = p_true / (sum(p_true) + sum(p_false))\n\n    return {\"acc\": sum(p_true)}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "question",
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "truthfulqa_mc2": 2.0
+  },
+  "n-shot": {
+    "truthfulqa_mc2": 0
+  },
+  "higher_is_better": {
+    "truthfulqa_mc2": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "truthfulqa_mc2": {
+      "original": 817,
+      "effective": 817
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735957764.7570622,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.90\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 24434.078025398,
+  "end_time": 24545.624577618,
+  "total_evaluation_time_seconds": "111.54655221999928"
+}

evaluation/en/winogrande_0_shot.json

@@ -0,0 +1,108 @@
+{
+  "results": {
+    "winogrande": {
+      "alias": "winogrande",
+      "acc,none": 0.7048145224940805,
+      "acc_stderr,none": 0.012819410741754765
+    }
+  },
+  "group_subtasks": {
+    "winogrande": []
+  },
+  "configs": {
+    "winogrande": {
+      "task": "winogrande",
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "training_split": "train",
+      "validation_split": "validation",
+      "doc_to_text": "def doc_to_text(doc):\n    answer_to_num = {\"1\": 0, \"2\": 1}\n    return answer_to_num[doc[\"answer\"]]\n",
+      "doc_to_target": "def doc_to_target(doc):\n    idx = doc[\"sentence\"].index(\"_\") + 1\n    return doc[\"sentence\"][idx:].strip()\n",
+      "doc_to_choice": "def doc_to_choice(doc):\n    idx = doc[\"sentence\"].index(\"_\")\n    options = [doc[\"option1\"], doc[\"option2\"]]\n    return [doc[\"sentence\"][:idx] + opt for opt in options]\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "sentence",
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "winogrande": 1.0
+  },
+  "n-shot": {
+    "winogrande": 0
+  },
+  "higher_is_better": {
+    "winogrande": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "winogrande": {
+      "original": 1267,
+      "effective": 1267
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=4,data_parallel_size=2,gpu_memory_utilization=0.5,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735957928.9213855,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.90\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "/ALLaM-7B-Instruct",
+  "model_name_sanitized": "/ALLaM-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 24598.479043164,
+  "end_time": 24674.97354231,
+  "total_evaluation_time_seconds": "76.49449914599973"
+}