Delete evaluation/ar

evaluation/ar/acva_5_shot.json

@@ -1,119 +0,0 @@
-{
-  "results": {
-    "acva": {
-      "alias": "acva",
-      "acc,none": 0.7746268656716417,
-      "acc_stderr,none": 0.004477269169728854,
-      "acc_norm,none": 0.7632606199770379,
-      "acc_norm_stderr,none": 0.004554991129754026
-    }
-  },
-  "group_subtasks": {
-    "acva": []
-  },
-  "configs": {
-    "acva": {
-      "task": "acva",
-      "tag": [
-        "multiple_choice"
-      ],
-      "dataset_path": "FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _format_subject(subject):\n        \n        arabic_words = subtasks_ar[subtasks.index(subject)]\n        return arabic_words\n    \n    def _generate_subject(doc):\n        subject = _format_subject(doc[\"id\"].split(\"-\")[0])\n\n        return subject\n        \n    def _process_docs(doc):\n        keys = [\"\u0635\u062d\",\n                 \"\u062e\u0637\u0623\"]\n        subject = _generate_subject(doc)\n        gold = keys.index(doc['answer'])\n        out_doc = {\n            \"id\": doc[\"id\"],\n            \"query\":  \"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" + doc[\"question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\",\n            \"choices\": keys,\n            \"gold\": gold,\n            \"subject\": subject,\n        }\n        \n        return out_doc\n\n    return dataset.map(_process_docs)\n",
-      "doc_to_text": "query",
-      "doc_to_target": "gold",
-      "doc_to_choice": "choices",
-      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d' \u0623\u0648 '\u062e\u0637\u0623' \u062f\u0648\u0646 \u0634\u0631\u062d",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 5,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    }
-  },
-  "versions": {
-    "acva": 0.0
-  },
-  "n-shot": {
-    "acva": 5
-  },
-  "higher_is_better": {
-    "acva": {
-      "acc": true,
-      "acc_norm": true
-    }
-  },
-  "n-samples": {
-    "acva": {
-      "original": 8710,
-      "effective": 8710
-    }
-  },
-  "config": {
-    "model": "vllm",
-    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
-    "batch_size": 1,
-    "batch_sizes": [],
-    "device": null,
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": null,
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "8e1bd48d",
-  "date": 1735662713.7617116,
-  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
-  "transformers_version": "4.47.1",
-  "upper_git_hash": null,
-  "tokenizer_pad_token": [
-    "<unk>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "</s>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<s>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 4096,
-  "task_hashes": {
-    "acva": "d007c508f0accdd697f549d7cbe7f960f1470c8f86f1a0969355a6ef33108edb"
-  },
-  "model_source": "vllm",
-  "model_name": "/ALLaM-7B-Instruct",
-  "model_name_sanitized": "/ALLaM-7B-Instruct",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": false,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "start_time": 3374.021232778,
-  "end_time": 3578.563943596,
-  "total_evaluation_time_seconds": "204.54271081800016"
-}

evaluation/ar/araMath_v2_5_shot.json

@@ -1,123 +0,0 @@
-{
-  "results": {
-    "araMath_v2": {
-      "alias": "araMath_v2",
-      "acc,none": 0.655,
-      "acc_stderr,none": 0.019423021295885703,
-      "acc_norm,none": 0.655,
-      "acc_norm_stderr,none": 0.019423021295885703
-    }
-  },
-  "group_subtasks": {
-    "araMath_v2": []
-  },
-  "configs": {
-    "araMath_v2": {
-      "task": "araMath_v2",
-      "tag": [
-        "multiple_choice"
-      ],
-      "dataset_path": "lm_eval/tasks/araMath_v2",
-      "dataset_name": "araMath_v2",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def remove_prefix(choice):\n            prefixes = [\"(A)\", \"(B)\", \"(C)\", \"(D)\"]\n            for prefix in prefixes:\n                if choice.startswith(prefix + \" \"):\n                    return choice[len(prefix) + 1:]  \n            return choice \n\n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"options\"])]\n            )\n\n            prompt = f\"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_en.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
-      "doc_to_text": "query",
-      "doc_to_target": "gold",
-      "doc_to_choice": "{{choices}}",
-      "description": "\u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 \u0628\u064a\u0646 'A\u060c B\u060c C\u060c D' \u062f\u0648\u0646 \u0634\u0631\u062d",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 5,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": true,
-      "doc_to_decontamination_query": "query",
-      "metadata": {
-        "version": 0.0
-      }
-    }
-  },
-  "versions": {
-    "araMath_v2": 0.0
-  },
-  "n-shot": {
-    "araMath_v2": 5
-  },
-  "higher_is_better": {
-    "araMath_v2": {
-      "acc": true,
-      "acc_norm": true
-    }
-  },
-  "n-samples": {
-    "araMath_v2": {
-      "original": 600,
-      "effective": 600
-    }
-  },
-  "config": {
-    "model": "hf",
-    "model_args": "parallelize=True,pretrained=/ALLaM-7B-Instruct,trust_remote_code=True",
-    "model_num_parameters": 7000559616,
-    "model_dtype": "torch.bfloat16",
-    "model_revision": "main",
-    "model_sha": "",
-    "batch_size": 1,
-    "batch_sizes": [],
-    "device": null,
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": null,
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "5e10e017",
-  "date": 1736774062.2964265,
-  "pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.86\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
-  "transformers_version": "4.38.2",
-  "upper_git_hash": null,
-  "tokenizer_pad_token": [
-    "<unk>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "</s>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<s>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 4096,
-  "task_hashes": {},
-  "model_source": "hf",
-  "model_name": "/ALLaM-7B-Instruct",
-  "model_name_sanitized": "/ALLaM-7B-Instruct",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": false,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "start_time": 72495.638596469,
-  "end_time": 72556.179139124,
-  "total_evaluation_time_seconds": "60.54054265499872"
-}

evaluation/ar/etec_0_shot.json

@@ -1,121 +0,0 @@
-{
-  "results": {
-    "etec": {
-      "alias": "etec",
-      "acc,none": 0.6680761099365751,
-      "acc_stderr,none": 0.010828952839616325,
-      "acc_norm,none": 0.6680761099365751,
-      "acc_norm_stderr,none": 0.010828952839616325
-    }
-  },
-  "group_subtasks": {
-    "etec": []
-  },
-  "configs": {
-    "etec": {
-      "task": "etec",
-      "tag": [
-        "multiple_choice"
-      ],
-      "dataset_path": "lm_eval/tasks/etec",
-      "dataset_name": "etec",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_ar = [\"\u0623\", \"\u0628\", \"\u062c\", \"\u062f\"]\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n                \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_ar.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
-      "doc_to_text": "query",
-      "doc_to_target": "gold",
-      "doc_to_choice": "choices",
-      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d\n ",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": true,
-      "doc_to_decontamination_query": "query",
-      "metadata": {
-        "version": 0.0
-      }
-    }
-  },
-  "versions": {
-    "etec": 0.0
-  },
-  "n-shot": {
-    "etec": 0
-  },
-  "higher_is_better": {
-    "etec": {
-      "acc": true,
-      "acc_norm": true
-    }
-  },
-  "n-samples": {
-    "etec": {
-      "original": 1892,
-      "effective": 1892
-    }
-  },
-  "config": {
-    "model": "vllm",
-    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
-    "batch_size": 1,
-    "batch_sizes": [],
-    "device": null,
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": null,
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "8e1bd48d",
-  "date": 1735662950.8344455,
-  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
-  "transformers_version": "4.47.1",
-  "upper_git_hash": null,
-  "tokenizer_pad_token": [
-    "<unk>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "</s>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<s>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 4096,
-  "task_hashes": {
-    "etec": "8937d87b09ed63604ed9f64a02b8ba75ee9c43b9acebd5dd58a797e187916bbf"
-  },
-  "model_source": "vllm",
-  "model_name": "/ALLaM-7B-Instruct",
-  "model_name_sanitized": "/ALLaM-7B-Instruct",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": false,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "start_time": 3611.154007204,
-  "end_time": 3697.095375819,
-  "total_evaluation_time_seconds": "85.94136861499965"
-}

evaluation/ar/exams_ar_5_shot.json

@@ -1,121 +0,0 @@
-{
-  "results": {
-    "exams_ar": {
-      "alias": "exams_ar",
-      "acc,none": 0.515828677839851,
-      "acc_stderr,none": 0.021585885942816244,
-      "acc_norm,none": 0.515828677839851,
-      "acc_norm_stderr,none": 0.021585885942816244
-    }
-  },
-  "group_subtasks": {
-    "exams_ar": []
-  },
-  "configs": {
-    "exams_ar": {
-      "task": "exams_ar",
-      "tag": [
-        "multiple_choice"
-      ],
-      "dataset_path": "lm_eval/tasks/exams_ar",
-      "dataset_name": "exams_ar",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            \"\"\"\n            <prompt>\n            \u0633\u0624\u0627\u0644:\n            A. <choice1>\n            B. <choice2>\n            C. <choice3>\n            D. <choice4>\n            \u0627\u062c\u0627\u0628\u0629:\n            \"\"\"\n            \n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        def _format_subject(subject):\n            arabic_words = subtasks_ar[subtasks.index(subject)]\n            return arabic_words\n\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n    \n        subject = doc['id'].split(\"-\")[0]\n        description = f\"\ufed2\ufef4\ufee3\ufe8d \ufef2\ufee0\ufef3 \ufe84\ufeb4\ufe8c\ufedf\ufe93 \ufe8d\ufefc\ufea8\ufe98\ufef3\ufe8d\ufead \ufee2\ufee7 \ufee2\ufe98\ufecb\ufea9\ufea9 (\ufee2\ufecb \ufe8d\ufefa\ufe9f\ufe8e\ufe91\ufe8e\ufe97) \ufea1\ufeee\ufedf {_format_subject(subject)} \\n\" #\ufee2\ufee7 \ufed2\ufec0\ufee0\ufedb \ufe8e\ufea8\ufe97\ufead \ufe88\ufe9f\ufe8e\ufe91\ufe93 \ufeed\ufe8e\ufea3\ufea9\ufe93 \ufee2\ufee7 \ufe90\ufef4\ufee7 'A\u060c B\u060c C\u060c D' \ufea9\ufeee\ufee7 \ufeb5\ufeae\ufea3\\n\"\n\n        out_doc = {\n            \"idx\": doc[\"idx\"],\n            \"id\": doc[\"id\"],\n            'dsecription': description,\n            \"query\":  format_example(doc, keys), # \"Question: \" + doc[\"question\"]['stem'] + \"\\nAnswer:\",\n            \"choices\": keys,\n            \"gold\": [\"A\", \"B\", \"C\", \"D\"].index(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_docs)\n",
-      "doc_to_text": "query",
-      "doc_to_target": "gold",
-      "doc_to_choice": "choices",
-      "description": "description",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 5,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": true,
-      "doc_to_decontamination_query": "query",
-      "metadata": {
-        "version": 0.0
-      }
-    }
-  },
-  "versions": {
-    "exams_ar": 0.0
-  },
-  "n-shot": {
-    "exams_ar": 5
-  },
-  "higher_is_better": {
-    "exams_ar": {
-      "acc": true,
-      "acc_norm": true
-    }
-  },
-  "n-samples": {
-    "exams_ar": {
-      "original": 537,
-      "effective": 537
-    }
-  },
-  "config": {
-    "model": "vllm",
-    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
-    "batch_size": 1,
-    "batch_sizes": [],
-    "device": null,
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": null,
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "8e1bd48d",
-  "date": 1735662207.0830526,
-  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
-  "transformers_version": "4.47.1",
-  "upper_git_hash": null,
-  "tokenizer_pad_token": [
-    "<unk>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "</s>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<s>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 4096,
-  "task_hashes": {
-    "exams_ar": "b1561abd56354d570ac16bf64163b0ee8dc6c507234b05f678576b09c26c644a"
-  },
-  "model_source": "vllm",
-  "model_name": "/ALLaM-7B-Instruct",
-  "model_name_sanitized": "/ALLaM-7B-Instruct",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": false,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "start_time": 2867.397536365,
-  "end_time": 2948.510496752,
-  "total_evaluation_time_seconds": "81.11296038699993"
-}

evaluation/ar/gat_0_shot.json

@@ -1,549 +0,0 @@
-{
-  "results": {
-    "gat": {
-      "acc,none": 0.4452527279568544,
-      "acc_stderr,none": 0.0038711388833064567,
-      "alias": "gat"
-    },
-    "gat_algebra": {
-      "alias": " - gat_algebra",
-      "acc,none": 0.40667903525046384,
-      "acc_stderr,none": 0.009463939247454995
-    },
-    "gat_analogy": {
-      "alias": " - gat_analogy",
-      "acc,none": 0.35919854280510016,
-      "acc_stderr,none": 0.009158766245747282
-    },
-    "gat_arithmetic": {
-      "alias": " - gat_arithmetic",
-      "acc,none": 0.40154582259845417,
-      "acc_stderr,none": 0.009406284814832203
-    },
-    "gat_association": {
-      "alias": " - gat_association",
-      "acc,none": 0.5464114832535886,
-      "acc_stderr,none": 0.015407801869520031
-    },
-    "gat_comparisons": {
-      "alias": " - gat_comparisons",
-      "acc,none": 0.34508196721311474,
-      "acc_stderr,none": 0.013616100682624904
-    },
-    "gat_completion": {
-      "alias": " - gat_completion",
-      "acc,none": 0.6057851239669422,
-      "acc_stderr,none": 0.014054411207805699
-    },
-    "gat_contextual": {
-      "alias": " - gat_contextual",
-      "acc,none": 0.3941717791411043,
-      "acc_stderr,none": 0.013537713096332765
-    },
-    "gat_geometry": {
-      "alias": " - gat_geometry",
-      "acc,none": 0.473972602739726,
-      "acc_stderr,none": 0.026171590093068537
-    },
-    "gat_reading": {
-      "alias": " - gat_reading",
-      "acc,none": 0.5727788279773157,
-      "acc_stderr,none": 0.009620311542503682
-    }
-  },
-  "groups": {
-    "gat": {
-      "acc,none": 0.4452527279568544,
-      "acc_stderr,none": 0.0038711388833064567,
-      "alias": "gat"
-    }
-  },
-  "group_subtasks": {
-    "gat": [
-      "gat_analogy",
-      "gat_association",
-      "gat_completion",
-      "gat_reading",
-      "gat_algebra",
-      "gat_arithmetic",
-      "gat_comparisons",
-      "gat_contextual",
-      "gat_geometry"
-    ]
-  },
-  "configs": {
-    "gat_algebra": {
-      "task": "gat_algebra",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "algebra",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    },
-    "gat_analogy": {
-      "task": "gat_analogy",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "analogy",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    },
-    "gat_arithmetic": {
-      "task": "gat_arithmetic",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "arithmetic",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    },
-    "gat_association": {
-      "task": "gat_association",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "association",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    },
-    "gat_comparisons": {
-      "task": "gat_comparisons",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "comparisons",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    },
-    "gat_completion": {
-      "task": "gat_completion",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "completion",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    },
-    "gat_contextual": {
-      "task": "gat_contextual",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "contextual",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "\u0627\u0648\u062c\u062f \u0627\u0644\u062e\u0637\u0623 \u0627\u0644\u0633\u064a\u0627\u0642\u064a \u0641\u064a \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0627\u0644\u062a\u0627\u0644\u064a\u0629 \u0645\u0646 \u0628\u064a\u0646 \u0627\u0644\u062e\u064a\u0627\u0631\u0627\u062a:",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    },
-    "gat_geometry": {
-      "task": "gat_geometry",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "geometry",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    },
-    "gat_reading": {
-      "task": "gat_reading",
-      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
-      "dataset_name": "reading",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "fewshot_split": "validation",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
-      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
-      "doc_to_target": "{{label}}",
-      "doc_to_choice": [
-        "\u0623",
-        "\u0628",
-        "\u062c",
-        "\u062f"
-      ],
-      "description": "",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    }
-  },
-  "versions": {
-    "gat": 0,
-    "gat_algebra": 0.0,
-    "gat_analogy": 0.0,
-    "gat_arithmetic": 0.0,
-    "gat_association": 0.0,
-    "gat_comparisons": 0.0,
-    "gat_completion": 0.0,
-    "gat_contextual": 0.0,
-    "gat_geometry": 0.0,
-    "gat_reading": 0.0
-  },
-  "n-shot": {
-    "gat_algebra": 0,
-    "gat_analogy": 0,
-    "gat_arithmetic": 0,
-    "gat_association": 0,
-    "gat_comparisons": 0,
-    "gat_completion": 0,
-    "gat_contextual": 0,
-    "gat_geometry": 0,
-    "gat_reading": 0
-  },
-  "higher_is_better": {
-    "gat": {
-      "acc": true
-    },
-    "gat_algebra": {
-      "acc": true
-    },
-    "gat_analogy": {
-      "acc": true
-    },
-    "gat_arithmetic": {
-      "acc": true
-    },
-    "gat_association": {
-      "acc": true
-    },
-    "gat_comparisons": {
-      "acc": true
-    },
-    "gat_completion": {
-      "acc": true
-    },
-    "gat_contextual": {
-      "acc": true
-    },
-    "gat_geometry": {
-      "acc": true
-    },
-    "gat_reading": {
-      "acc": true
-    }
-  },
-  "n-samples": {
-    "gat_analogy": {
-      "original": 2745,
-      "effective": 2745
-    },
-    "gat_association": {
-      "original": 1045,
-      "effective": 1045
-    },
-    "gat_completion": {
-      "original": 1210,
-      "effective": 1210
-    },
-    "gat_reading": {
-      "original": 2645,
-      "effective": 2645
-    },
-    "gat_algebra": {
-      "original": 2695,
-      "effective": 2695
-    },
-    "gat_arithmetic": {
-      "original": 2717,
-      "effective": 2717
-    },
-    "gat_comparisons": {
-      "original": 1220,
-      "effective": 1220
-    },
-    "gat_contextual": {
-      "original": 1304,
-      "effective": 1304
-    },
-    "gat_geometry": {
-      "original": 365,
-      "effective": 365
-    }
-  },
-  "config": {
-    "model": "vllm",
-    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
-    "batch_size": 1,
-    "batch_sizes": [],
-    "device": null,
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": null,
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "8e1bd48d",
-  "date": 1735664096.2650902,
-  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
-  "transformers_version": "4.47.1",
-  "upper_git_hash": null,
-  "tokenizer_pad_token": [
-    "<unk>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "</s>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<s>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 4096,
-  "task_hashes": {
-    "gat_analogy": "ede28dec097bfebe8a85a19fa27d001696858276df66254bdb70fc63231f1a83",
-    "gat_association": "5d82550d46c4f3cabf370185a8a23cc2eb5b08f1f0c5e210a8a712562a44bd08",
-    "gat_completion": "fc3c19dd7f1896696fec1bffc21182804c9b2f1fb8d8c882428a6bb4bb61e370",
-    "gat_reading": "93053b187a750d2e87f5488f2d0fda944f3da9195bb04d1c4dee9c4b56fa626a",
-    "gat_algebra": "77832c595eaaf156775c3dbb27da0915ef600ebf46a7113ae32a202b0359e8a6",
-    "gat_arithmetic": "6a498f75f5cc0ffd1b30f7a6293ba80d08f2a8876d5558d8e934bf57355ff0cc",
-    "gat_comparisons": "acb80c0ed8dd07e916a471189aef3a546efc289824b2cc50a32c11dc4c97c9c1",
-    "gat_contextual": "de063ed3b94011d74ee24a6532122c9d344fc15e42800db44f0849995a0bc37a",
-    "gat_geometry": "3e482885559a4404ee9e97556edc6e49959770a499f4ae2c58f18ad85b91a363"
-  },
-  "model_source": "vllm",
-  "model_name": "/ALLaM-7B-Instruct",
-  "model_name_sanitized": "/ALLaM-7B-Instruct",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": false,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "start_time": 4756.376698655,
-  "end_time": 5124.76942052,
-  "total_evaluation_time_seconds": "368.39272186499966"
-}

evaluation/ar/moe_ien_mcq_0_shot.json

@@ -1,118 +0,0 @@
-{
-  "results": {
-    "moe_ien_mcq": {
-      "alias": "moe_ien_mcq",
-      "acc,none": 0.9154154154154154,
-      "acc_stderr,none": 0.0027841569543517694,
-      "acc_norm,none": 0.9154154154154154,
-      "acc_norm_stderr,none": 0.0027841569543517694
-    }
-  },
-  "group_subtasks": {
-    "moe_ien_mcq": []
-  },
-  "configs": {
-    "moe_ien_mcq": {
-      "task": "moe_ien_mcq",
-      "dataset_path": "lm_eval/tasks/moe_ien_mcq/ien_moe_mcq.py",
-      "dataset_name": "moe_ien_mcq",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.split(\". \", 1)[1] if \". \" in choice else choice\n\n        def format_example(doc, keys):\n            question = doc[\"Question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"Choices\"])]\n                \n            )\n            prompt = f\"\\n\\n\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\"][0:len(doc[\"Choices\"])]\n        out_doc = {\n            \"Query\":  format_example(doc, keys), \n            \"Choices\": keys,\n            \"gold\": int(doc[\"Answer\"])-1, ## \n            \"Speciality\": doc['Speciality']\n        }        \n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
-      "doc_to_text": "Query",
-      "doc_to_target": "gold",
-      "doc_to_choice": "{{Choices}}",
-      "description": "\u0641\u064a\u0645\u0627\u202f\u064a\u0644\u064a\u202f\u0623\u0633\u0626\u0644\u0629\u202f\u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631\u202f\u0645\u0646\u202f\u0645\u062a\u0639\u062f\u062f\u202f(\u0645\u0639\u202f\u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a)\u202f\u0641\u064a\u202f{{Speciality}}",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": true,
-      "doc_to_decontamination_query": "Query",
-      "metadata": {
-        "version": 0.0
-      }
-    }
-  },
-  "versions": {
-    "moe_ien_mcq": 0.0
-  },
-  "n-shot": {
-    "moe_ien_mcq": 0
-  },
-  "higher_is_better": {
-    "moe_ien_mcq": {
-      "acc": true,
-      "acc_norm": true
-    }
-  },
-  "n-samples": {
-    "moe_ien_mcq": {
-      "original": 9990,
-      "effective": 9990
-    }
-  },
-  "config": {
-    "model": "vllm",
-    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
-    "batch_size": 1,
-    "batch_sizes": [],
-    "device": null,
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": null,
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "8e1bd48d",
-  "date": 1735663068.5370116,
-  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
-  "transformers_version": "4.47.1",
-  "upper_git_hash": null,
-  "tokenizer_pad_token": [
-    "<unk>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "</s>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<s>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 4096,
-  "task_hashes": {
-    "moe_ien_mcq": "554899322e5b78369683b10024d90dc868f768d310530589a6167541e8f9d594"
-  },
-  "model_source": "vllm",
-  "model_name": "/ALLaM-7B-Instruct",
-  "model_name_sanitized": "/ALLaM-7B-Instruct",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": false,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "start_time": 3728.910211786,
-  "end_time": 3947.718352837,
-  "total_evaluation_time_seconds": "218.8081410509999"
-}

evaluation/ar/moe_ien_tf_0_shot.json

@@ -1,119 +0,0 @@
-{
-  "results": {
-    "moe_ien_tf": {
-      "alias": "moe_ien_tf",
-      "acc,none": 0.8557082967729356,
-      "acc_stderr,none": 0.0034697209254064324,
-      "acc_norm,none": 0.8557082967729356,
-      "acc_norm_stderr,none": 0.0034697209254064324
-    }
-  },
-  "group_subtasks": {
-    "moe_ien_tf": []
-  },
-  "configs": {
-    "moe_ien_tf": {
-      "task": "moe_ien_tf",
-      "tag": [
-        "multiple_choice"
-      ],
-      "dataset_path": "lm_eval/tasks/moe_ien_tf",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        keys=[\"\u0635\u062d\u064a\u062d\u0629\",\n              \"\u062e\u0627\u0637\u0626\u0629\"\n              ]\n        #keys =[\"\u0635\u0648\u0627\u0628\",\n        #         \"\u062e\u0637\u0623\"]\n        target_key = int(doc[\"Answer\"])-1\n\n        out_doc = {\n            \"query\":  \"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" +doc[\"Question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\", \n            \"choices\": keys,\n            \"gold\": target_key,\n        }\n        return out_doc\n    return dataset.map(_process_docs)\n",
-      "doc_to_text": "query",
-      "doc_to_target": "gold",
-      "doc_to_choice": "choices",
-      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{Speciality}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d\u064a\u062d\u0629' \u0623\u0648 '\u062e\u0627\u0637\u0626\u0629' \u062f\u0648\u0646 \u0634\u0631\u062d ",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": false,
-      "metadata": {
-        "version": 0.0
-      }
-    }
-  },
-  "versions": {
-    "moe_ien_tf": 0.0
-  },
-  "n-shot": {
-    "moe_ien_tf": 0
-  },
-  "higher_is_better": {
-    "moe_ien_tf": {
-      "acc": true,
-      "acc_norm": true
-    }
-  },
-  "n-samples": {
-    "moe_ien_tf": {
-      "original": 10257,
-      "effective": 10257
-    }
-  },
-  "config": {
-    "model": "vllm",
-    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
-    "batch_size": 1,
-    "batch_sizes": [],
-    "device": null,
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": null,
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "8e1bd48d",
-  "date": 1735663321.6141305,
-  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
-  "transformers_version": "4.47.1",
-  "upper_git_hash": null,
-  "tokenizer_pad_token": [
-    "<unk>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "</s>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<s>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 4096,
-  "task_hashes": {
-    "moe_ien_tf": "bf29d6fb290755a9dc7c5aaf1263e4cd1e9d82a62085aa6279661d8b84fd5ab6"
-  },
-  "model_source": "vllm",
-  "model_name": "/ALLaM-7B-Instruct",
-  "model_name_sanitized": "/ALLaM-7B-Instruct",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": false,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "start_time": 3981.83990155,
-  "end_time": 4097.740745391,
-  "total_evaluation_time_seconds": "115.9008438410001"
-}

evaluation/ar/sdaia_mcq_0_shot.json

@@ -1,121 +0,0 @@
-{
-  "results": {
-    "sdaia_mcq": {
-      "alias": "sdaia_mcq",
-      "acc,none": 0.739021329987453,
-      "acc_stderr,none": 0.011003303841318535,
-      "acc_norm,none": 0.739021329987453,
-      "acc_norm_stderr,none": 0.011003303841318535
-    }
-  },
-  "group_subtasks": {
-    "sdaia_mcq": []
-  },
-  "configs": {
-    "sdaia_mcq": {
-      "task": "sdaia_mcq",
-      "tag": [
-        "multiple_choice"
-      ],
-      "dataset_path": "lm_eval/tasks/sdaia_mcq/sdaia_mcq.py",
-      "dataset_name": "sdaia_mcq",
-      "dataset_kwargs": {
-        "trust_remote_code": true
-      },
-      "test_split": "test",
-      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.replace('.', '') if '.' in choice[:2] else choice\n        \n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            \n            choice_num = ['choice1', 'choice2', 'choice3', 'choice4']\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(doc[choice_num[index]])}\\n\" for index, key in enumerate(keys)]\n            )\n\n            prompt = f\"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        #keys = [\"1\", \"2\", \"3\", \"4\"]\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\":  format_example(doc, keys), \n            \"choices\": keys,\n            \"gold\": doc[\"answer\"]-1,\n        }        \n\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
-      "doc_to_text": "query",
-      "doc_to_target": "gold",
-      "doc_to_choice": "{{choices}}",
-      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d",
-      "target_delimiter": " ",
-      "fewshot_delimiter": "\n\n",
-      "num_fewshot": 0,
-      "metric_list": [
-        {
-          "metric": "acc",
-          "aggregation": "mean",
-          "higher_is_better": true
-        },
-        {
-          "metric": "acc_norm",
-          "aggregation": "mean",
-          "higher_is_better": true
-        }
-      ],
-      "output_type": "multiple_choice",
-      "repeats": 1,
-      "should_decontaminate": true,
-      "doc_to_decontamination_query": "Question",
-      "metadata": {
-        "version": 0.0
-      }
-    }
-  },
-  "versions": {
-    "sdaia_mcq": 0.0
-  },
-  "n-shot": {
-    "sdaia_mcq": 0
-  },
-  "higher_is_better": {
-    "sdaia_mcq": {
-      "acc": true,
-      "acc_norm": true
-    }
-  },
-  "n-samples": {
-    "sdaia_mcq": {
-      "original": 1594,
-      "effective": 1594
-    }
-  },
-  "config": {
-    "model": "vllm",
-    "model_args": "pretrained=/ALLaM-7B-Instruct,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
-    "batch_size": 1,
-    "batch_sizes": [],
-    "device": null,
-    "use_cache": null,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "gen_kwargs": null,
-    "random_seed": 0,
-    "numpy_seed": 1234,
-    "torch_seed": 1234,
-    "fewshot_seed": 1234
-  },
-  "git_hash": "8e1bd48d",
-  "date": 1735663470.0459642,
-  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
-  "transformers_version": "4.47.1",
-  "upper_git_hash": null,
-  "tokenizer_pad_token": [
-    "<unk>",
-    "0"
-  ],
-  "tokenizer_eos_token": [
-    "</s>",
-    "2"
-  ],
-  "tokenizer_bos_token": [
-    "<s>",
-    "1"
-  ],
-  "eot_token_id": 2,
-  "max_length": 4096,
-  "task_hashes": {
-    "sdaia_mcq": "c69b252ba97d5f402c302aadb4d06d0293774e38250e701b1d0c7984fa47dd24"
-  },
-  "model_source": "vllm",
-  "model_name": "/ALLaM-7B-Instruct",
-  "model_name_sanitized": "/ALLaM-7B-Instruct",
-  "system_instruction": null,
-  "system_instruction_sha": null,
-  "fewshot_as_multiturn": false,
-  "chat_template": null,
-  "chat_template_sha": null,
-  "start_time": 4130.43217211,
-  "end_time": 4204.747507708,
-  "total_evaluation_time_seconds": "74.31533559800027"
-}