diff --git "a/results.json" "b/results.json" --- "a/results.json" +++ "b/results.json" @@ -2,107 +2,51 @@ { "model":"amazon\/nova-micro-v1", "bcp_47":"ar", - "task":"mmlu", + "task":"classification", "metric":"accuracy", - "score":0.6 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2144530934 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.499609058 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3307884966 + "score":0.8 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5479156427 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", - "score":0.3 + "score":0.6 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"bn", + "bcp_47":"ar", "task":"translation_from", "metric":"bleu", - "score":0.193036151 + "score":0.2294045445 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"bn", + "bcp_47":"ar", "task":"translation_from", "metric":"chrf", - "score":0.4366975641 + "score":0.4952691669 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"bn", + "bcp_47":"ar", "task":"translation_to", "metric":"bleu", - "score":0.2512647986 + "score":0.3288016428 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"bn", + "bcp_47":"ar", "task":"translation_to", "metric":"chrf", - "score":0.4387026201 + "score":0.5463705477 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"de", - "task":"mmlu", + "bcp_47":"en", + "task":"classification", "metric":"accuracy", - "score":0.5 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.249655311 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4987192142 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.3783450256 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.5970718288 + "score":0.9 }, { "model":"amazon\/nova-micro-v1", @@ -116,28 +60,35 @@ "bcp_47":"en", "task":"translation_from", "metric":"bleu", - "score":0.4813757383 + "score":0.480740436 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"translation_from", "metric":"chrf", - "score":0.6701342658 + "score":0.6653209358 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"translation_to", "metric":"bleu", - "score":0.5512931433 + "score":0.5761493028 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"translation_to", "metric":"chrf", - "score":0.763108119 + "score":0.7850415995 + }, + { + "model":"amazon\/nova-micro-v1", + "bcp_47":"es", + "task":"classification", + "metric":"accuracy", + "score":0.8 }, { "model":"amazon\/nova-micro-v1", @@ -151,63 +102,35 @@ "bcp_47":"es", "task":"translation_from", "metric":"bleu", - "score":0.2057312363 + "score":0.2334919219 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"translation_from", "metric":"chrf", - "score":0.4645708907 + "score":0.488808171 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"translation_to", "metric":"bleu", - "score":0.3335475603 + "score":0.3542820003 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"translation_to", "metric":"chrf", - "score":0.5654042924 + "score":0.5855988752 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"fr", - "task":"mmlu", + "bcp_47":"hi", + "task":"classification", "metric":"accuracy", - "score":0.6 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2522597451 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5010466238 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4197992651 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.608198896 + "score":0.9 }, { "model":"amazon\/nova-micro-v1", @@ -221,16344 +144,909 @@ "bcp_47":"hi", "task":"translation_from", "metric":"bleu", - "score":0.3499000643 + "score":0.352711035 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", - "score":0.5900050545 + "score":0.5881371851 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", - "score":0.3771214193 + "score":0.3866780476 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", - "score":0.5828460278 + "score":0.6045266419 + }, + { + "model":"amazon\/nova-micro-v1", + "bcp_47":"zh", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"id", + "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"id", + "bcp_47":"zh", "task":"translation_from", "metric":"bleu", - "score":0.1774899549 + "score":0.1724985295 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"id", + "bcp_47":"zh", "task":"translation_from", "metric":"chrf", - "score":0.4606519655 + "score":0.4704383702 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"id", + "bcp_47":"zh", "task":"translation_to", "metric":"bleu", - "score":0.1527125759 + "score":0.2855931625 }, { "model":"amazon\/nova-micro-v1", - "bcp_47":"id", + "bcp_47":"zh", "task":"translation_to", "metric":"chrf", - "score":0.5471305127 + "score":0.3243323503 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ja", + "model":"deepseek\/deepseek-chat", + "bcp_47":"ar", + "task":"classification", + "metric":"accuracy", + "score":0.9 + }, + { + "model":"deepseek\/deepseek-chat", + "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", - "score":0.5 + "score":0.9 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ja", + "model":"deepseek\/deepseek-chat", + "bcp_47":"ar", "task":"translation_from", "metric":"bleu", - "score":0.2535676165 + "score":0.2738315246 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ja", + "model":"deepseek\/deepseek-chat", + "bcp_47":"ar", "task":"translation_from", "metric":"chrf", - "score":0.5161759227 + "score":0.5469357403 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ja", + "model":"deepseek\/deepseek-chat", + "bcp_47":"ar", "task":"translation_to", "metric":"bleu", - "score":0.3421090766 + "score":0.3350156046 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ja", + "model":"deepseek\/deepseek-chat", + "bcp_47":"ar", "task":"translation_to", "metric":"chrf", - "score":0.4832285924 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.2559752722 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.4824824422 + "score":0.5718599465 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.1620132742 + "model":"deepseek\/deepseek-chat", + "bcp_47":"en", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.4958991194 + "model":"deepseek\/deepseek-chat", + "bcp_47":"en", + "task":"mmlu", + "metric":"accuracy", + "score":1.0 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"mr", + "model":"deepseek\/deepseek-chat", + "bcp_47":"en", "task":"translation_from", "metric":"bleu", - "score":0.1417986092 + "score":0.5729991861 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"mr", + "model":"deepseek\/deepseek-chat", + "bcp_47":"en", "task":"translation_from", "metric":"chrf", - "score":0.4193263814 + "score":0.7160877321 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"mr", + "model":"deepseek\/deepseek-chat", + "bcp_47":"en", "task":"translation_to", "metric":"bleu", - "score":0.1222453955 + "score":0.6673869986 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"mr", + "model":"deepseek\/deepseek-chat", + "bcp_47":"en", "task":"translation_to", "metric":"chrf", - "score":0.3265542088 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.3643913289 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5978521406 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.3487197391 + "score":0.8276942212 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.4994988579 + "model":"deepseek\/deepseek-chat", + "bcp_47":"es", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pt", + "model":"deepseek\/deepseek-chat", + "bcp_47":"es", "task":"mmlu", "metric":"accuracy", - "score":0.7 + "score":0.9 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pt", + "model":"deepseek\/deepseek-chat", + "bcp_47":"es", "task":"translation_from", "metric":"bleu", - "score":0.2674020405 + "score":0.354793442 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pt", + "model":"deepseek\/deepseek-chat", + "bcp_47":"es", "task":"translation_from", "metric":"chrf", - "score":0.4916276462 + "score":0.5751433215 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pt", + "model":"deepseek\/deepseek-chat", + "bcp_47":"es", "task":"translation_to", "metric":"bleu", - "score":0.449593878 + "score":0.2990292431 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"pt", + "model":"deepseek\/deepseek-chat", + "bcp_47":"es", "task":"translation_to", "metric":"chrf", - "score":0.6517684253 + "score":0.5879890464 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ru", + "model":"deepseek\/deepseek-chat", + "bcp_47":"hi", + "task":"classification", + "metric":"accuracy", + "score":0.9 + }, + { + "model":"deepseek\/deepseek-chat", + "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", - "score":0.6 + "score":1.0 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ru", + "model":"deepseek\/deepseek-chat", + "bcp_47":"hi", "task":"translation_from", "metric":"bleu", - "score":0.1691106182 + "score":0.3618244878 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ru", + "model":"deepseek\/deepseek-chat", + "bcp_47":"hi", "task":"translation_from", "metric":"chrf", - "score":0.4371008423 + "score":0.5955921046 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ru", + "model":"deepseek\/deepseek-chat", + "bcp_47":"hi", "task":"translation_to", "metric":"bleu", - "score":0.3132986377 + "score":0.4464861773 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ru", + "model":"deepseek\/deepseek-chat", + "bcp_47":"hi", "task":"translation_to", "metric":"chrf", - "score":0.559742916 + "score":0.6528285136 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"sw", + "model":"deepseek\/deepseek-chat", + "bcp_47":"zh", + "task":"classification", + "metric":"accuracy", + "score":1.0 + }, + { + "model":"deepseek\/deepseek-chat", + "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", - "score":0.5 + "score":0.9 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"sw", + "model":"deepseek\/deepseek-chat", + "bcp_47":"zh", "task":"translation_from", "metric":"bleu", - "score":0.1944514657 + "score":0.2411074799 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"sw", + "model":"deepseek\/deepseek-chat", + "bcp_47":"zh", "task":"translation_from", "metric":"chrf", - "score":0.4363680283 + "score":0.524514003 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"sw", + "model":"deepseek\/deepseek-chat", + "bcp_47":"zh", "task":"translation_to", "metric":"bleu", - "score":0.2507279258 + "score":0.2895271183 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"sw", + "model":"deepseek\/deepseek-chat", + "bcp_47":"zh", "task":"translation_to", "metric":"chrf", - "score":0.5456882386 + "score":0.3718777788 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"te", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"ar", + "task":"classification", + "metric":"accuracy", + "score":0.9 + }, + { + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", - "score":0.4 + "score":0.8 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"te", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"ar", "task":"translation_from", "metric":"bleu", - "score":0.2351879022 + "score":0.286402856 }, { - "model":"amazon\/nova-micro-v1", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.5015135666 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2443279394 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4616934816 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2239109073 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4664590257 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.1004419383 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.3466465332 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1579119983 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4176784236 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.2143148158 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.4814311669 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1837575642 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4701178189 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2769852176 - }, - { - "model":"amazon\/nova-micro-v1", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3191136721 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2700460204 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5538421491 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3429486101 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.585338958 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2518742297 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4808653185 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.3262591736 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.5167211951 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.296094691 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5405203037 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.4521703872 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6510137978 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.5350598812 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6920702898 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6197100322 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7935529886 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.3328054743 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5621761866 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3056069517 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5864625476 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.3183976094 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5810941349 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4748705901 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6771774372 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3380504047 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5712827315 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.4099090208 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.6013300104 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2977952957 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5374664029 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3176178107 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6263788011 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2817398654 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.5582480948 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.4310206038 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.5648545023 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.3061246587 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.523750141 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2727123426 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5748905525 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2242946033 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.5097037063 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2137264994 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.440475376 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.43589074 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.6610423251 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.3781104049 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5478926249 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.3335410498 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5672032656 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4580630889 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6657392427 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2587147192 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.5313277465 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3724063678 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5962061241 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2269997511 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.4842755902 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.3044437459 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.6117061265 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.290563665 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.5273017096 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2358621489 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4806590073 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2429196873 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5008155104 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.3049495338 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4791605551 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2408476334 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4988564858 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.2703596899 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5116028165 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2376852279 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.5256489144 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2792207353 - }, - { - "model":"deepseek\/deepseek-chat", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3511763987 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.201730622 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5230392495 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3852105985 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.605693575 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2515106137 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.5085911882 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.2989934852 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4923392967 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2947779952 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5363074175 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.3523906614 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.5431292252 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3549731332 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.516110221 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.5306797494 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7137131706 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2686376487 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5150941143 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.2858074351 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5391420281 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2992701671 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5475268989 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.3905037889 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6068698391 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3538422525 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5862254357 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.41385038 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.6049826714 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.3007180384 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5023246329 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.334501536 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6312038844 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2664302142 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.5511307577 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3695096312 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.5016072256 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.2557256216 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.4694930912 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2301023035 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5333883608 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2389669394 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4384602668 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1906304694 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.43321511 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.2890532438 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5744900625 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.2023348505 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.4209687673 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.3077173175 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5684889974 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.3988769765 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6526950425 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1939421958 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4613061334 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.2954667205 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.4928272924 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2244277924 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.4632403501 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.2475167578 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.5859145482 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2491270517 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.4971850787 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.230423487 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4572263206 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2528912211 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5471673071 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2890836127 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4638645889 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2182193511 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4918690502 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3051142824 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5736498723 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1267866191 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.3661653956 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2217838799 - }, - { - "model":"deepseek\/deepseek-chat-v3-0324", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3212219671 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.1955321366 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.4452496338 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3157575918 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5012233036 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.1735477119 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4036037512 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.2603256448 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4715465891 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2350996731 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4539868888 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.4694847332 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6499890785 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.4790246882 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6389223375 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6184989115 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.8037117471 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.1802447888 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.352852694 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.32221956 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5945828154 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.1396007132 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.2948285656 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4626443841 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6453178011 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3248801173 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5368976197 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3477596714 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.542033928 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2034777205 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4787094647 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.4161506758 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.675862263 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2148759695 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4372780553 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.1725311936 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.3103103139 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.3165472295 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.5345475991 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2983221296 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5707529624 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1247777236 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.2923162337 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1050598836 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.2683360352 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.1844224826 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.3422693726 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.1807039126 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.2632902709 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.2434841819 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.4532930974 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4043131386 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.5936961583 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1866121683 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.3995426351 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3589759487 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5459455358 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2525543056 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.4855834718 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.3353132654 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.5534047662 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.1505793252 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.269111455 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.1313190377 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.3349177081 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.1514267917 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.3397549606 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2593168504 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4181171937 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1737123456 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3975781626 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.320003713 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5179634386 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2147159409 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4884471405 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2856153111 - }, - { - "model":"deepseek\/deepseek-r1", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3570853581 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2957143991 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5733819398 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3975775226 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.6026539032 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2722037865 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.5441644725 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.3756817753 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.5431366991 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.3669235156 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.61602266 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.515407684 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6945040285 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.5394941139 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.7176459206 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6542806821 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.8016677558 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2647784787 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.534131581 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3885319246 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.6122434715 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.3106669108 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5636006618 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.5258244924 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.7167464827 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3579075772 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.6194831237 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.4158534882 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.6174987325 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.3245383993 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5729744645 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.4448839139 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6973440754 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.3117179672 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.5777870563 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3252873649 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4988074019 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.3569868465 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.5843991239 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.3135334271 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5896292046 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.310376381 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.553246687 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2835101619 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4885179598 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.4330249307 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.6595328005 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4034194489 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5814569981 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.3163929238 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5723494086 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.5307526172 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.7110385531 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2574428009 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.5385430576 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.4379543642 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.6143170099 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.3703323281 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.5881678812 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.5012096467 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.7067312972 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.3855742221 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.6089215443 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.3338485428 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.5599923303 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.256633436 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5540218114 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.283656908 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4700660306 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2924845131 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.5532241595 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3842345819 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.6178494251 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2857271778 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.5503996958 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.3778780802 - }, - { - "model":"google\/gemini-2.0-flash-001", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.397093475 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2261993605 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.4713910334 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.4047621232 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.6016984486 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2193638362 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4698224614 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.3331571997 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.5230347118 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2768085602 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4912101702 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.5341715111 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.7072193548 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3830044788 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.5606920251 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.5668030405 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7339158252 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2943762224 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5358537364 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3601264217 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.6055265571 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2647971272 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5503506292 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.5019602858 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6907968386 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3907958067 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.6168614685 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.4116631597 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.6260711406 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2723472362 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5512694018 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.4033292298 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6688453679 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2213115571 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4716655163 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.2097990462 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.3915561417 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.3653472959 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.5797628494 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.3433849381 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.600500681 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2443836816 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4868851136 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2414882248 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4707414144 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.3529476629 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5689952282 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4285508909 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5918084941 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.311051651 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5763190766 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.5012048785 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6909453526 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2525201789 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4998034041 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3876399108 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5526901501 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2544576251 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.510033557 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.463366902 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.6862513407 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2891932187 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.5401801911 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2956394846 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.5301794135 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2772013738 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5352561426 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.3143344562 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.5013762031 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2023911962 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4478296607 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3591790299 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5997773195 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2384160236 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.490052749 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2454938644 - }, - { - "model":"google\/gemini-2.0-flash-lite-001", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2975998115 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.3372721649 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5838783957 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.4321344547 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.6264247806 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2938960839 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.5517113066 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.3782283657 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.5487977464 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.3186073711 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5411120296 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.5051785366 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6920207566 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.4956880203 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6377883658 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6849600072 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.836046383 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2675890166 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5349872169 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.4090938759 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.6229850426 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.3316785101 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5862021915 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.5917547348 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.7493377856 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.4181179966 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.6225342018 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.4326454895 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.6447613926 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.3651687929 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5938680131 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3831360784 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6646674328 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.3772905023 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.5716747693 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.4648684731 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.5702790562 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.4284391483 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.6436107817 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.3533000629 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.6161033443 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2317334189 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.5101013277 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2648144898 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4919964495 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.4595839503 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.678498887 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.478748482 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.6196055546 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.3741937374 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.6015951936 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.5289737883 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.7055677715 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2597045639 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.5415502492 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.5240240402 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.677637684 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.3411098531 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.5298146709 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.479199485 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.6917999035 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.4076773709 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.6331898021 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.3546746435 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.5752621807 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.3221779371 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5831693751 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2652097773 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4365593229 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.3357390025 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.5736288237 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.4227150213 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.6237856247 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2672185947 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.5405753178 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.3184157374 - }, - { - "model":"google\/gemini-2.5-flash-preview", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3790142738 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2627473985 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5441433858 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.2905461034 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.533714307 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2482076583 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.5203197505 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.2943876522 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.5061827933 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2926727761 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5382695102 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.4992927063 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.68458461 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3544545007 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6065493785 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.4605012594 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7071085486 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.3062605383 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5185293421 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3509297564 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5918368155 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2672257271 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5461942844 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.5445047026 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.7187017022 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3658457853 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.6104356234 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3523205262 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.5683614987 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.3445897108 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5707894245 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3561921353 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6497121165 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.3395831717 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.6027437002 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.4020848766 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4849709417 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.3178340836 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.5358308503 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2594137172 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5439108225 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.231055869 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4704905603 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2167113972 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.446095738 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.4063183084 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.6360820916 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4147226756 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5528779397 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.3026809635 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5703886674 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4781663764 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6799080797 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2557079153 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.5311675905 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3835973558 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5821962599 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2577141288 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.5044946448 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.359025417 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.6186979309 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2906881879 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.5350481537 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.236600544 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4972520076 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2771747402 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5443382904 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.1736835138 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.3883707647 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2410514007 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4876248025 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3887220068 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.6136859301 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2111467812 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4809814755 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2521327553 - }, - { - "model":"google\/gemini-flash-1.5", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3112350874 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2467308763 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.534220382 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.2743392971 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5037551764 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.1944999711 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4268510127 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.259115468 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.494497039 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2829478955 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4871806301 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.4110829383 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6232948404 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3785767671 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.609933858 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.51911398 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7392276316 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.237555247 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4538651122 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3655739185 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.6222977455 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2666374602 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.499416925 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4596821942 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6596069595 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.243987548 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.4795583662 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3694040944 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.5844559185 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2068662913 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4574195897 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3502935028 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6435312578 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2621288101 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.5227903424 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3001729338 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4543813968 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.2413161859 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.4889410755 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2769851422 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5709543626 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1815113298 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4391057397 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2353762412 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4455494128 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.3485084476 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5939961743 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4125142357 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5513553952 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.2628896238 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5183595448 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4596356921 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6605740155 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.182466759 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4404528166 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3555783157 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5498712278 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2359761779 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.4965595451 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.2850822623 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.5802793004 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2700708569 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.5216974993 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2200201561 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4752363489 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2468482501 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5030755763 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2478229019 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4152088296 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1594908644 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3987254781 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3413161409 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.6036920979 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2452562567 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4947687861 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2210892317 - }, - { - "model":"google\/gemini-flash-1.5-8b", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2889702622 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2378760525 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5183150258 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3230924319 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5538406612 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.1 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.261796363 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.5061044452 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.3392340733 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.527149208 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.3574957943 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5949602318 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.5362416556 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.7003378351 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.4224513828 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6281695357 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.5351379766 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7598417142 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.279398514 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5677163743 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3796449789 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.6098050112 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2586433932 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5545655476 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.5217173445 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.706560562 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3147708697 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5735957081 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3839899572 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.6091652356 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2843117161 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5625584849 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3330091043 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6452178521 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.1 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2694680234 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.534930693 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.2886629345 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4441001762 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.2889499493 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.4946977693 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2708892281 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5462808569 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2909291965 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.5445276883 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2268785589 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4679091917 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.4076785456 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.6390667798 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4008672864 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5746982049 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.28456543 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5644895825 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.5029529971 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.7001896317 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2624491573 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.5200383542 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.5021166201 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.6735886 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2077591586 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.484103396 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.3186094389 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.6101838123 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2690763704 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.5194720687 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2606721725 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4856589279 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2738179594 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5231933041 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2402519591 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4398186462 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.3 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2009145559 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4683501544 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3564695765 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.6122018967 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1887088881 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4674871904 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2493415308 - }, - { - "model":"google\/gemma-3-27b-it", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3083774204 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.0771743646 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.2973457605 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.107373793 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.2699953852 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.0133841771 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.1998161123 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.0571375232 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.2236086675 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.1885986583 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.3942008265 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.208640787 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.4336553669 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.2635281374 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.4503496048 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.3327540401 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.5634975097 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.161908961 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.3791550712 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.1756752106 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.4286169476 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.1391333463 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.3906284149 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.2221133172 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.439476083 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.1927642098 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.3900862934 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.1921622852 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.3921656103 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.1450314351 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.349096966 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.1678766463 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.4572154046 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.122127159 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.3592521793 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.1445573759 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.2746217781 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.0385853258 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.2297130712 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.0327175312 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.1978832002 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.0606459804 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.2530129366 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.0379902356 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.2129894179 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.2215887293 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.3740953101 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.2063143842 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.3059460254 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.1491417042 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.3641087791 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.2212237119 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.4443362942 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1053993699 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.339909617 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.1027968239 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.3552639409 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.04894473 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.2403687794 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.0081616469 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.2256342788 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.1038281469 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.2848836056 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.1264888672 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.2764453485 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.0896680673 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.2815141335 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.0393651445 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.2318870316 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1221410689 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3760902563 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.1740511381 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.3679951776 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1065732415 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.3712929662 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.1553037377 - }, - { - "model":"gryphe\/mythomax-l2-13b", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2272771523 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2106827678 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.4729315216 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.1833822876 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.4592369142 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.1849279476 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4313097332 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.2409740184 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4412144476 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2234676555 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4567541675 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.3260331458 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.5706133276 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.1748225206 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.3715602465 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.3077182547 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.5349807426 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.1850637388 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4446871408 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3369466851 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5877033036 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2046187032 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.4459730588 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4666795067 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6677200822 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3304821544 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5478705434 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3530460218 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.558479654 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.1852494633 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4413568977 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.2908938694 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.5892878712 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.18258136 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4537379112 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.1225657534 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.199008122 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.2245721935 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.4151722512 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2248935322 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5061056827 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1740373958 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.45132413 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1932088011 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4185660328 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.1523314336 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.3644985981 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.1654328825 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.3306856997 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.1849607455 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.4432514335 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.3735403041 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6258974903 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1542357564 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.407792192 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3023477748 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5524877157 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.1851974615 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.3965623482 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.1508209944 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.4882487732 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.1379772803 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.4107039585 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2016846201 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4266778738 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.175281535 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4308484341 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2117497213 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.3938011024 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1357518844 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3722986269 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.2732555305 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5379510442 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1738291666 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4357500531 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.1521172398 - }, - { - "model":"meta-llama\/llama-3-70b-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2131109353 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2599940834 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5344413165 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3535082667 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5618696854 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2354383205 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4808197348 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.3199525151 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.499911007 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.3087730646 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5446110065 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.4363239189 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6606949091 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.5339078139 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6892570142 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6674605684 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.804371781 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2419716374 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.507956668 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3634368286 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5899847818 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2344633197 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5330179435 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4911890131 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6961788043 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3846763493 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.6011964597 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.4163962042 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.6196043433 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2757294842 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5164990805 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.2979743631 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.621505647 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.3952954307 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.6095092587 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3550137141 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4750629972 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.2908958438 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.5056906446 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.278835165 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.4936217166 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2506634169 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4822339063 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1977398 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4512527967 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.4150075487 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5859289925 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4055714442 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5640351838 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.3043299439 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.535196717 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4680489994 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6589781804 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.222732943 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4930414776 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.338298129 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5638349443 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2821010087 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.5068790976 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.2757507483 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.5718564064 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2421646919 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.4745308575 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2617143353 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.5319635422 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2376372003 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4703124968 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2981093662 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4567172276 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2436649491 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4800401855 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3190461612 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5738845596 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.241579409 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4834754638 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2521936889 - }, - { - "model":"meta-llama\/llama-3.1-70b-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3101863746 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2693485281 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.526972026 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.299118757 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5209634099 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2349115906 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4958748831 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.3037551079 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.5104197052 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2940412827 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5289123372 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.4945548288 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6883045723 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.5254997328 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6651833212 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6367218418 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7901906482 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2547596952 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5121489935 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3970366728 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.6218247901 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2933660925 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5571361609 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.5068230898 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6748037018 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.343226497 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5766444084 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3910462734 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.5964148868 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2823009881 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5398328113 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3103565141 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6174908793 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.3323087124 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.5573643434 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3398063689 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.439612207 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.3202476462 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.5170575936 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2655261196 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5078431653 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2672781462 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.5076383126 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1627869169 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4085690767 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.4343597896 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.612284022 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4056628693 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.577470079 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.2561623538 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5084673727 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4822378549 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6805267649 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2002083293 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4701113234 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3404957825 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5535956904 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2378532768 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.488547843 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.265213578 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.5688332618 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2212863699 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.4840600136 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2690593843 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.5097708439 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2689156709 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.5166231119 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2395415037 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4291961624 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1514958599 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.393503436 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3126864004 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5612804645 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2120665528 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.473531494 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2450622208 - }, - { - "model":"meta-llama\/llama-3.3-70b-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3218418642 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.296505833 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5400572519 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.340461259 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5619514904 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2759322818 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.520565338 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.338451158 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.5187985166 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.3759519266 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5897372465 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.4241869609 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6369588457 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.5411864684 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6967802953 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6482731713 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.8116769221 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2922155838 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5497719853 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.338476596 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5745067933 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2953732107 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5548969581 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.5304234655 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.7109575095 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3775698519 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5982891247 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3986854808 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.591115524 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2279863353 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5022597218 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3892094891 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6520141168 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.3788371121 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.6055484505 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3351360505 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4702609655 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.3432613501 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.5702557631 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2987779883 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5727527461 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.245468889 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.5195139142 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2246556712 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4485577055 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.4116477856 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.6206199646 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4995455509 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.619445933 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.3804523937 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5992357582 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4118835999 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6402783366 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2711737241 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.5223358319 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3600231378 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5851662403 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2816996366 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.5425859416 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.3295094623 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.6093047217 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.3372428289 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.5687146545 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.3362998921 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.5480115484 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2130096226 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4588260908 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2672672187 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4657426772 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1973356715 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4598464437 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3356973056 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.6057343387 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2839664983 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.5496970367 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2404693379 - }, - { - "model":"meta-llama\/llama-4-maverick", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2741729656 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.1333199947 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.3787420394 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.2639924417 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.4508419338 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.0781077372 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.3675270991 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.1386482919 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.3721300124 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2182705396 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4808885751 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.3009940804 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.5563863941 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3417051248 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.5564910783 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.505573911 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7036775988 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.162581322 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4452830938 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3120471917 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5521181062 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.1841410036 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.4498873761 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.3957930078 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.5908491177 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.2781818374 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5317078224 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.2082459717 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.3612751063 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.14109102 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4206086243 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.2541374481 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.5738191205 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.217631862 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4810988251 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3314270444 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4579736168 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.1369006135 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.3726552499 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.1358290891 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.4485195299 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1220522034 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.394432105 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.0877294163 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.3228035874 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.2872682457 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5343897569 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.2344910913 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.4302635187 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.1578818745 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.4563510644 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.3548341797 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.5851699932 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1518810674 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4229373334 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.280180123 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5388785418 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.1197479462 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.3847229786 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.1284176951 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.4471488498 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.2 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.1221237902 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.3753302424 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.1342135875 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.3921252544 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.1013597082 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.3226565783 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.1055049805 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.3270359823 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1212000479 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3499332758 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.2215772318 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.4885306046 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1515209994 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4261650622 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.1646480678 - }, - { - "model":"microsoft\/phi-4", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2556911118 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.2 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.12805161 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.3001336989 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.1701118305 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.2705414464 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.3 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.0274986656 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.2715478832 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.0364128057 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.1916781774 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.1092874906 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.3511512521 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.1229942688 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.3801618008 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3724031885 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.5022830131 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.4547673266 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.6611296308 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.1090053669 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.3312266479 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.2527998075 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.4585982852 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.0725417811 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.2990918671 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.2151316826 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.4131412973 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.1923702178 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.4177039104 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.1946792857 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.3652693419 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.141847306 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.3708481852 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.0995355066 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.3111314573 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2032657995 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.3735202727 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.2250820799 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.3303987825 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.0965458717 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.2876791143 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.0955767676 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.2170187808 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.0250471784 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.191193874 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.0430511796 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.2530147609 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.2939382507 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.4236905058 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.2038997189 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.3029720916 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.1164585656 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.3513786374 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.1159124937 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.3709389273 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.0726892156 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.2863134478 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.1856825973 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.4059546251 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.3 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.0285853745 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.258743147 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.025156184 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.1783483023 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.0776108509 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.2968212875 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.0381758727 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.2127544397 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.0750463099 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.2214284722 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.070850694 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.2944265552 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.3 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.0771444238 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.2978139061 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.0799080562 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.2533330302 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.0844065166 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.3102353728 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.1424724277 - }, - { - "model":"microsoft\/phi-4-multimodal-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.1997648583 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.1 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.1411079383 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.4288778553 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.0953750383 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.3064512757 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.1 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.1056007136 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.3471561552 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.1826317159 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4009621221 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.1779380635 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4500965094 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.2121723095 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.5030225058 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3044634871 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.5185245123 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.4511358274 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7059119826 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.1945088953 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4507966567 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3128110525 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.564309507 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.1503854665 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.4086984161 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4089231963 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6185522756 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.2552169122 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.473817666 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.245531449 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.49602593 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.183278736 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4259117743 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.1644315134 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.4582343316 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2091363742 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4571081356 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.1736443062 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.2944908594 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.1871758781 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.3534639201 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.1246961548 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.3016674377 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1225817213 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.3113005758 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1352680788 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.2974034458 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.3657744488 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5584856442 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.2676684918 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.4408840729 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.1448893824 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.4141759018 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.3802451797 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.5985788184 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1462210673 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4241453806 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.2074330919 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.4438291243 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.0700151652 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.267224603 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.0414991243 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.3151327738 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.0841044753 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.3329048731 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.1093199271 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.3565208631 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.0712938477 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.34111588 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.1282666873 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.355242737 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1008653377 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3602737727 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.0837032908 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.2497686258 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.1 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1835816123 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4240772425 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.1204225303 - }, - { - "model":"mistralai\/mistral-nemo", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2073656478 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2307115512 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.4984040511 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3687026411 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5939572307 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2038963891 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4710421194 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.3131013355 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4896188194 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2711678576 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5010413002 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.3851397268 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6299318639 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3971499204 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.5704846467 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.5236846455 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.750594995 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2325003342 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4556556517 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.2894670078 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.552498138 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.1905858019 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.4419502294 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4778596825 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6586250533 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3285111598 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5456949766 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3807039375 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.5585736378 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.266611987 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4806521836 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.280362444 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.5856798399 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2716028557 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.5228700434 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.2362873295 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.3988160859 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.1351110513 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.3282404022 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.1135047432 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.3702437183 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2524012374 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4786192289 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2105658138 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4350780053 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.3889163911 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5826179394 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.3921555968 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5536315548 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.2943696763 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5224446861 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4674713564 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6782176965 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1704112037 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4377240493 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.2666948821 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.4926215679 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.07741038 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.2658688973 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.1116660561 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.4225621416 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.3226731558 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.4991511328 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2633784516 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4918671935 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.1788289199 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4270187713 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2622858638 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4513610959 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1261643561 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3351014303 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.2593001367 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5207002255 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.2331909906 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4659908973 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2401879449 - }, - { - "model":"mistralai\/mistral-saba", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2984854823 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.1538277929 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.4198691069 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.2808764056 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.4945291951 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.1503587087 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.3519203949 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.1971487973 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.3802325565 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2282524895 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4762659907 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.310849797 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.5162495445 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3862933118 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.5649666335 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.4654982999 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.6658845521 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.153039152 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.3787678566 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.2956525612 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5355759847 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2366178299 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.496953378 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4332629029 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6444037956 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.2794593031 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5163454129 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3170576155 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.5211214847 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.1707579789 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4033109614 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.2188248408 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.5492911343 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2285487924 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.471295757 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3374389141 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.454206736 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.1830822648 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.3748256188 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.156719726 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.3565335339 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1396612724 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.3788068496 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1238899835 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.3481069187 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.3486285299 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5488183895 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.3765245892 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5298772939 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.1859039826 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.4364419603 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.3813303912 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.5887135238 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1311303005 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.3844997287 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.2777412876 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.4986551548 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.2 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.123417194 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.3074793247 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.1862273327 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.4964265847 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.1871161773 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.3959828918 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.1541981122 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.3534389775 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.1380435832 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.3409229162 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.1042021716 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.3262262617 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1450000268 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.384195795 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.2211289124 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.4485571181 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1336281725 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.3849552359 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2484908362 - }, - { - "model":"mistralai\/mistral-small-3.1-24b-instruct", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3028612614 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.1903586334 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.446387516 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.2809263669 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.4646534423 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.1096890189 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.3793955783 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.1159907317 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.3788984673 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2549144567 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4826972193 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.34224065 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.5863783658 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3883780582 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.5704308494 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.4668069531 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.672145186 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.184032222 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4351218193 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3557841931 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5719636086 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.1916132677 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.4640172061 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4034048767 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.5890628474 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.3 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3160468813 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5380845065 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3086022651 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.503357724 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.182638701 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4629618645 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.26058128 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.5416150753 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2482898345 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4829019738 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3097130959 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4348908804 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.1939894603 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.366697572 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2131488291 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.4853868303 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1138885419 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.3670742528 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1416518083 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.3217149096 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.2942498706 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5514597671 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.3152534079 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5017440919 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.2158772583 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.4780892551 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4052445466 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6186215895 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1501099752 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4157469966 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.2685247919 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.4840654691 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.1900353684 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.4405322363 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.2981157309 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.5535919581 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.2 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.1437549858 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.3497530813 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.1088443126 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.350662674 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.1592716898 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4051863522 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.1515473933 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.3710678709 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1245150014 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3649923323 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.1931933918 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.397517622 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1633023298 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4277663973 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2192873662 - }, - { - "model":"openai\/gpt-3.5-turbo", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2909960698 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.1829429954 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.4486992309 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3288082098 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5189698121 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.0865856519 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.3670408519 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.1246701174 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.3844904766 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.255733121 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.4745347601 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.3450910905 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.593965846 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.3751847049 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.5546252024 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.4781043171 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.6714236508 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":1.0 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.1971959428 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4525212046 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3678222384 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5884629819 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.1903326188 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.4492193992 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.3974812512 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.5872620046 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3091104445 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5239666365 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.308658779 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.4943494838 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.1558351322 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4339633925 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.2667443989 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.5471646551 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.2512075408 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4781485122 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3294110726 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4419761829 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.1874700943 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.3668478037 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2179816276 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.4537309029 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1293703417 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.3740601424 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.1316417996 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.3104567715 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.223477129 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.4562631144 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.2876949296 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.4857260461 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.1820530207 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.4181188719 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.417401324 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6189673269 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1434775736 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4131789962 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.2902641383 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5067661761 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.1838174977 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.446335194 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.2979831166 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.5546563707 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.3 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.1576206892 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.3790307912 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.1221954617 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.3360330123 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.1353612029 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.36632891 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.1357613569 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.3657578763 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1119834897 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.3394106853 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.1986904894 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.3757062086 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1621574248 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4283684768 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2167142755 - }, - { - "model":"openai\/gpt-3.5-turbo-0613", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.2830134241 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2353582779 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5006899773 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3664710407 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5606190587 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2553082196 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.488512704 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.2688722519 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4454248715 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2566663279 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5174970249 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.4322161955 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6526227117 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.4934809373 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6722291033 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6327969379 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.8005146414 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.235589316 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.5063165811 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3911713119 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.6242017348 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.311885366 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5786679545 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.5123128835 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6920494347 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3452221913 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5902403977 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.381895318 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.5898272046 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2855978621 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5472063766 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3721127194 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6463029364 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.288264373 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.5359086837 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3670619927 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.5068433812 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.3051034461 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.5471092112 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2747623728 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5861307631 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2464469919 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4954666446 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2249035276 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4254292104 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.4045917852 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.6244183732 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.4219453949 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5730380868 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.2852961117 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5549713299 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.4724645835 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6736886701 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2092871331 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4879651984 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.4104606671 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.6062298906 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.5 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2119039225 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.4718779729 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.3482867389 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.6083272302 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.292187148 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.5292968897 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.2456913217 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4548022965 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2215148184 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4881854895 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2641606665 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4480825493 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2163224434 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4949463826 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.3086902722 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5554756761 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.207708073 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4816364739 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2818179571 - }, - { - "model":"openai\/gpt-4.1-mini", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3369663048 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2325698513 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.4900910318 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3273886017 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5507594814 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.1825189442 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4341218556 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.2624070312 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4731044972 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2633872752 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5250805862 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.3914347425 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6094934561 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.4631079117 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6442114257 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.5299774376 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7380736765 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.4 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2274715114 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4837759577 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.3124440889 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.5261084633 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2312533162 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.4815491459 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.455086296 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6474755177 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.2652800435 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5175523714 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3386268714 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.5633372903 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.1907857214 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.4650956372 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.3541709408 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.6291579219 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.1469461673 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4392693708 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.2078261662 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.3748504876 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.2484943284 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.4654993953 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2497434284 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5490071169 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.1625330023 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4495751195 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2157605321 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.456208027 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.3790606838 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.6045377359 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.3874235881 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5593657469 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.2760123498 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5134357242 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.416752628 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6215563908 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.1853930017 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.465375097 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.2860173151 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5480059055 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.3 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2010282492 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.4493062791 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.2458134572 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.5476246368 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2087672996 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.4716980224 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.1943752237 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4205022979 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.1759186135 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4432474082 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.242532014 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4305022436 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.1547443459 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4483479472 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.2971306268 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5596664411 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1404632343 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4382880735 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2230925442 - }, - { - "model":"openai\/gpt-4.1-nano", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3010591018 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.2260572099 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.5164260772 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3098248698 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.536672296 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.1851732693 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4399312653 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.305694311 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4860162598 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.2442519451 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"de", - "task":"translation_from", - "metric":"chrf", - "score":0.5003799076 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"de", - "task":"translation_to", - "metric":"bleu", - "score":0.3647222325 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"de", - "task":"translation_to", - "metric":"chrf", - "score":0.6024672844 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"en", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"en", - "task":"translation_from", - "metric":"bleu", - "score":0.4797368411 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"en", - "task":"translation_from", - "metric":"chrf", - "score":0.6374317341 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"en", - "task":"translation_to", - "metric":"bleu", - "score":0.6311582936 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"en", - "task":"translation_to", - "metric":"chrf", - "score":0.7997371179 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"es", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"es", - "task":"translation_from", - "metric":"bleu", - "score":0.2600341328 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"es", - "task":"translation_from", - "metric":"chrf", - "score":0.4963209525 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"es", - "task":"translation_to", - "metric":"bleu", - "score":0.417992948 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"es", - "task":"translation_to", - "metric":"chrf", - "score":0.640138245 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"fr", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.2376904256 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.5011085853 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4849594812 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6766251104 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"hi", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"hi", - "task":"translation_from", - "metric":"bleu", - "score":0.3234457974 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"hi", - "task":"translation_from", - "metric":"chrf", - "score":0.5370126468 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"hi", - "task":"translation_to", - "metric":"bleu", - "score":0.3996080448 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"hi", - "task":"translation_to", - "metric":"chrf", - "score":0.6023656967 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"id", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"id", - "task":"translation_from", - "metric":"bleu", - "score":0.2125374969 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"id", - "task":"translation_from", - "metric":"chrf", - "score":0.5022908944 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"id", - "task":"translation_to", - "metric":"bleu", - "score":0.2960726977 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"id", - "task":"translation_to", - "metric":"chrf", - "score":0.5881822388 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ja", - "task":"mmlu", - "metric":"accuracy", - "score":0.7 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ja", - "task":"translation_from", - "metric":"bleu", - "score":0.281578672 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ja", - "task":"translation_from", - "metric":"chrf", - "score":0.4963026763 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ja", - "task":"translation_to", - "metric":"bleu", - "score":0.3509105353 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ja", - "task":"translation_to", - "metric":"chrf", - "score":0.4772595136 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.244859193 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.4719413896 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.2375487692 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.5454443303 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"mr", - "task":"translation_from", - "metric":"bleu", - "score":0.2071473102 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"mr", - "task":"translation_from", - "metric":"chrf", - "score":0.4103819252 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"mr", - "task":"translation_to", - "metric":"bleu", - "score":0.2256200492 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"mr", - "task":"translation_to", - "metric":"chrf", - "score":0.4419952711 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pa", - "task":"translation_from", - "metric":"bleu", - "score":0.3531564234 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pa", - "task":"translation_from", - "metric":"chrf", - "score":0.5805142329 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pa", - "task":"translation_to", - "metric":"bleu", - "score":0.3707122189 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pa", - "task":"translation_to", - "metric":"chrf", - "score":0.5524474478 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pt", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pt", - "task":"translation_from", - "metric":"bleu", - "score":0.2673214525 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pt", - "task":"translation_from", - "metric":"chrf", - "score":0.5155609027 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pt", - "task":"translation_to", - "metric":"bleu", - "score":0.440844237 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"pt", - "task":"translation_to", - "metric":"chrf", - "score":0.6696954647 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ru", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ru", - "task":"translation_from", - "metric":"bleu", - "score":0.2144471986 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ru", - "task":"translation_from", - "metric":"chrf", - "score":0.4935159532 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ru", - "task":"translation_to", - "metric":"bleu", - "score":0.3268453207 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ru", - "task":"translation_to", - "metric":"chrf", - "score":0.5715958119 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"sw", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"sw", - "task":"translation_from", - "metric":"bleu", - "score":0.2086488438 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"sw", - "task":"translation_from", - "metric":"chrf", - "score":0.4503413027 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"sw", - "task":"translation_to", - "metric":"bleu", - "score":0.3062254829 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"sw", - "task":"translation_to", - "metric":"chrf", - "score":0.585118766 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"te", - "task":"mmlu", - "metric":"accuracy", - "score":0.6 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"te", - "task":"translation_from", - "metric":"bleu", - "score":0.2751005462 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"te", - "task":"translation_from", - "metric":"chrf", - "score":0.4938464228 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"te", - "task":"translation_to", - "metric":"bleu", - "score":0.200746746 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"te", - "task":"translation_to", - "metric":"chrf", - "score":0.4753275787 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ur", - "task":"translation_from", - "metric":"bleu", - "score":0.2271037726 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ur", - "task":"translation_from", - "metric":"chrf", - "score":0.4884796205 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ur", - "task":"translation_to", - "metric":"bleu", - "score":0.2821385341 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"ur", - "task":"translation_to", - "metric":"chrf", - "score":0.4794553557 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"vi", - "task":"mmlu", - "metric":"accuracy", - "score":0.8 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"vi", - "task":"translation_from", - "metric":"bleu", - "score":0.2177425012 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"vi", - "task":"translation_from", - "metric":"chrf", - "score":0.4737357708 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"vi", - "task":"translation_to", - "metric":"bleu", - "score":0.2936126961 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"vi", - "task":"translation_to", - "metric":"chrf", - "score":0.5463661979 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"zh", - "task":"mmlu", - "metric":"accuracy", - "score":0.9 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"zh", - "task":"translation_from", - "metric":"bleu", - "score":0.1648676261 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"zh", - "task":"translation_from", - "metric":"chrf", - "score":0.4464998109 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"zh", - "task":"translation_to", - "metric":"bleu", - "score":0.2613931597 - }, - { - "model":"openai\/gpt-4o-mini", - "bcp_47":"zh", - "task":"translation_to", - "metric":"chrf", - "score":0.3184321553 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ar", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ar", - "task":"translation_from", - "metric":"bleu", - "score":0.1426801844 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ar", - "task":"translation_from", - "metric":"chrf", - "score":0.3326263623 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ar", - "task":"translation_to", - "metric":"bleu", - "score":0.3435811596 - }, - { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ar", - "task":"translation_to", - "metric":"chrf", - "score":0.5560665631 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"bn", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"bn", - "task":"translation_from", - "metric":"bleu", - "score":0.2276144574 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"bn", - "task":"translation_from", - "metric":"chrf", - "score":0.4138420365 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"bn", - "task":"translation_to", - "metric":"bleu", - "score":0.23060166 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"bn", - "task":"translation_to", - "metric":"chrf", - "score":0.4010315404 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"de", - "task":"mmlu", - "metric":"accuracy", - "score":0.0 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"de", - "task":"translation_from", - "metric":"bleu", - "score":0.212111905 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"de", "task":"translation_from", "metric":"chrf", - "score":0.3988046786 + "score":0.5708001033 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"de", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"ar", "task":"translation_to", "metric":"bleu", - "score":0.3848855202 + "score":0.3786192599 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"de", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"ar", "task":"translation_to", "metric":"chrf", - "score":0.5719833521 + "score":0.5997968087 + }, + { + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"en", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.6 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"translation_from", "metric":"bleu", - "score":0.344237678 + "score":0.5225428152 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"translation_from", "metric":"chrf", - "score":0.4935151813 + "score":0.6575645999 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"translation_to", "metric":"bleu", - "score":0.4698515104 + "score":0.6797313469 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"translation_to", "metric":"chrf", - "score":0.6371144417 + "score":0.8385625651 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"es", + "task":"classification", + "metric":"accuracy", + "score":0.9 + }, + { + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.7 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"translation_from", "metric":"bleu", - "score":0.125959352 + "score":0.3141844263 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"translation_from", "metric":"chrf", - "score":0.3551503278 + "score":0.5385699999 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"translation_to", "metric":"bleu", - "score":0.3175303108 + "score":0.3178891159 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"translation_to", "metric":"chrf", - "score":0.5762214155 + "score":0.5832816452 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"fr", - "task":"mmlu", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"hi", + "task":"classification", "metric":"accuracy", - "score":0.0 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"fr", - "task":"translation_from", - "metric":"bleu", - "score":0.0902044795 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"fr", - "task":"translation_from", - "metric":"chrf", - "score":0.2605011709 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"fr", - "task":"translation_to", - "metric":"bleu", - "score":0.4870055243 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"fr", - "task":"translation_to", - "metric":"chrf", - "score":0.6791559488 + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":1.0 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", - "score":0.2633758754 + "score":0.3554389812 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", - "score":0.4288846329 + "score":0.6054458339 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", - "score":0.1741929926 + "score":0.4313822923 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", - "score":0.3117418002 + "score":0.6444269335 + }, + { + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"zh", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"id", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"id", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"zh", "task":"translation_from", "metric":"bleu", - "score":0.1385874564 + "score":0.2132360508 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"id", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"zh", "task":"translation_from", "metric":"chrf", - "score":0.3284698328 + "score":0.5251194405 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"id", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"zh", "task":"translation_to", "metric":"bleu", - "score":0.2276816545 + "score":0.2317379616 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"id", + "model":"deepseek\/deepseek-chat-v3-0324", + "bcp_47":"zh", "task":"translation_to", "metric":"chrf", - "score":0.452659811 + "score":0.3176035924 + }, + { + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"ar", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ja", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":1.0 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ja", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"ar", "task":"translation_from", "metric":"bleu", - "score":0.1632177135 + "score":0.2742096156 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ja", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"ar", "task":"translation_from", "metric":"chrf", - "score":0.3531449142 + "score":0.5237396198 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ja", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"ar", "task":"translation_to", "metric":"bleu", - "score":0.2119349185 + "score":0.3878873306 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ja", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"ar", "task":"translation_to", "metric":"chrf", - "score":0.3123591264 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"jv", - "task":"translation_from", - "metric":"bleu", - "score":0.1156136731 - }, - { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"jv", - "task":"translation_from", - "metric":"chrf", - "score":0.2195665146 + "score":0.592059186 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"jv", - "task":"translation_to", - "metric":"bleu", - "score":0.1897770522 + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"en", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"jv", - "task":"translation_to", - "metric":"chrf", - "score":0.4637640103 + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"en", + "task":"mmlu", + "metric":"accuracy", + "score":0.7 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"mr", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"en", "task":"translation_from", "metric":"bleu", - "score":0.1903027055 + "score":0.4617498931 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"mr", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"en", "task":"translation_from", "metric":"chrf", - "score":0.3635453097 + "score":0.632500083 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"mr", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"en", "task":"translation_to", "metric":"bleu", - "score":0.1757074235 + "score":0.5929094835 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"mr", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"en", "task":"translation_to", "metric":"chrf", - "score":0.289700177 + "score":0.7605588552 + }, + { + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"es", + "task":"classification", + "metric":"accuracy", + "score":0.9 + }, + { + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"es", + "task":"mmlu", + "metric":"accuracy", + "score":0.7 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pa", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"es", "task":"translation_from", "metric":"bleu", - "score":0.3436575025 + "score":0.3524793037 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pa", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"es", "task":"translation_from", "metric":"chrf", - "score":0.482656548 + "score":0.5702304962 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pa", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"es", "task":"translation_to", "metric":"bleu", - "score":0.099939613 + "score":0.3535078361 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pa", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"es", "task":"translation_to", "metric":"chrf", - "score":0.1871385498 + "score":0.6042934807 + }, + { + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"hi", + "task":"classification", + "metric":"accuracy", + "score":1.0 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pt", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pt", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"hi", "task":"translation_from", "metric":"bleu", - "score":0.139109335 + "score":0.4240988738 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pt", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"hi", "task":"translation_from", "metric":"chrf", - "score":0.2975718876 + "score":0.6315937478 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pt", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"hi", "task":"translation_to", "metric":"bleu", - "score":0.3775837608 + "score":0.4354841175 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"pt", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"hi", "task":"translation_to", "metric":"chrf", - "score":0.5791655154 + "score":0.6560516689 + }, + { + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"zh", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ru", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.8 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ru", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"zh", "task":"translation_from", "metric":"bleu", - "score":0.1469881314 + "score":0.2579131129 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ru", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"zh", "task":"translation_from", "metric":"chrf", - "score":0.3543525231 + "score":0.5221861909 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ru", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"zh", "task":"translation_to", "metric":"bleu", - "score":0.2601449601 + "score":0.2744015447 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ru", + "model":"google\/gemini-2.0-flash-lite-001", + "bcp_47":"zh", "task":"translation_to", "metric":"chrf", - "score":0.4637758535 + "score":0.314329989 + }, + { + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"ar", + "task":"classification", + "metric":"accuracy", + "score":1.0 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"sw", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"sw", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"ar", "task":"translation_from", "metric":"bleu", - "score":0.1254796913 + "score":0.3508412834 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"sw", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"ar", "task":"translation_from", "metric":"chrf", - "score":0.3282519563 + "score":0.5872252847 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"sw", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"ar", "task":"translation_to", "metric":"bleu", - "score":0.0915983019 + "score":0.4287777143 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"sw", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"ar", "task":"translation_to", "metric":"chrf", - "score":0.3007484482 + "score":0.6233671596 + }, + { + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"en", + "task":"classification", + "metric":"accuracy", + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"te", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"en", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"te", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"en", "task":"translation_from", "metric":"bleu", - "score":0.2589650486 + "score":0.5793531611 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"te", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"en", "task":"translation_from", "metric":"chrf", - "score":0.3965279961 + "score":0.7289340401 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"te", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"en", "task":"translation_to", "metric":"bleu", - "score":0.1905279289 + "score":0.725042796 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"te", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"en", "task":"translation_to", "metric":"chrf", - "score":0.315796915 + "score":0.868453404 + }, + { + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"es", + "task":"classification", + "metric":"accuracy", + "score":1.0 + }, + { + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"es", + "task":"mmlu", + "metric":"accuracy", + "score":0.7 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ur", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"es", "task":"translation_from", "metric":"bleu", - "score":0.1738304371 + "score":0.3216105574 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ur", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"es", "task":"translation_from", "metric":"chrf", - "score":0.402419023 + "score":0.5687312504 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ur", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"es", "task":"translation_to", "metric":"bleu", - "score":0.20023176 + "score":0.4170343863 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"ur", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"es", "task":"translation_to", "metric":"chrf", - "score":0.3992758012 + "score":0.6312350249 + }, + { + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"hi", + "task":"classification", + "metric":"accuracy", + "score":1.0 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"vi", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"vi", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"hi", "task":"translation_from", "metric":"bleu", - "score":0.2127933722 + "score":0.4661333335 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"vi", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"hi", "task":"translation_from", "metric":"chrf", - "score":0.4040178087 + "score":0.6599650679 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"vi", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"hi", "task":"translation_to", "metric":"bleu", - "score":0.1091836451 + "score":0.4241555175 }, { - "model":"qwen\/qwen3-235b-a22b", - "bcp_47":"vi", + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"hi", "task":"translation_to", "metric":"chrf", - "score":0.3365920606 + "score":0.6308064781 + }, + { + "model":"google\/gemini-2.5-flash-preview", + "bcp_47":"zh", + "task":"classification", + "metric":"accuracy", + "score":1.0 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", - "score":0.0 + "score":0.9 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", - "score":0.1277232373 + "score":0.2618499798 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", - "score":0.3405306117 + "score":0.5541922583 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", - "score":0.2524023407 + "score":0.3106583668 }, { - "model":"qwen\/qwen3-235b-a22b", + "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", - "score":0.2849249913 + "score":0.369137454 } ] \ No newline at end of file