diff --git "a/results.json" "b/results.json" --- "a/results.json" +++ "b/results.json" @@ -7,12 +7,14 @@ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.4351349353198866, - "chrf": 54.9504915580248 + "chrf": 54.9504915580248, + "accuracy": 1.0 } ], "bleu": 0.4351349353198866, "chrf": 54.9504915580248, - "commonvoice_hours": 2649.0, + "accuracy": 1.0, + "commonvoice_hours": 2651.0, "commonvoice_locale": "en", "population": { "AC": 931, @@ -180,11 +182,13 @@ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3977775857451761, - "chrf": 57.672913792439125 + "chrf": 57.672913792439125, + "accuracy": 1.0 } ], "bleu": 0.3977775857451761, "chrf": 57.672913792439125, + "accuracy": 1.0, "commonvoice_hours": 422.0, "commonvoice_locale": "zh-TW", "population": { @@ -218,11 +222,13 @@ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.333521621016373, - "chrf": 50.48364584189306 + "chrf": 50.48364584189306, + "accuracy": 0.9333333333333333 } ], "bleu": 0.333521621016373, "chrf": 50.48364584189306, + "accuracy": 0.9333333333333333, "commonvoice_hours": 16.0, "commonvoice_locale": "hi-IN", "population": { @@ -242,11 +248,13 @@ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.29160032861883095, - "chrf": 47.668399832701844 + "chrf": 47.668399832701844, + "accuracy": 0.9666666666666667 } ], "bleu": 0.29160032861883095, "chrf": 47.668399832701844, + "accuracy": 0.9666666666666667, "commonvoice_hours": 446.0, "commonvoice_locale": "es", "population": { @@ -299,11 +307,13 @@ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.277257629790728, - "chrf": 46.62779335380641 + "chrf": 46.62779335380641, + "accuracy": 0.9333333333333333 } ], "bleu": 0.277257629790728, "chrf": 46.62779335380641, + "accuracy": 0.9333333333333333, "commonvoice_hours": 91.0, "commonvoice_locale": "ar", "population": { @@ -355,12 +365,14 @@ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2659144372728079, - "chrf": 44.14831240898717 + "chrf": 44.14831240898717, + "accuracy": 0.8333333333333334 } ], "bleu": 0.2659144372728079, "chrf": 44.14831240898717, - "commonvoice_hours": 76.0, + "accuracy": 0.8333333333333334, + "commonvoice_hours": 77.0, "commonvoice_locale": "ur", "population": { "CA": 286475, @@ -378,11 +390,13 @@ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.315663773358301, - "chrf": 49.253978669350964 + "chrf": 49.253978669350964, + "accuracy": 0.9666666666666667 } ], "bleu": 0.315663773358301, "chrf": 49.253978669350964, + "accuracy": 0.9666666666666667, "commonvoice_hours": 1052.0, "commonvoice_locale": "fr", "population": { @@ -458,11 +472,13 @@ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21265887286151353, - "chrf": 41.501657722373686 + "chrf": 41.501657722373686, + "accuracy": 0.9333333333333333 } ], "bleu": 0.21265887286151353, "chrf": 41.501657722373686, + "accuracy": 0.9333333333333333, "commonvoice_hours": 49.0, "commonvoice_locale": "bn", "population": { @@ -478,15 +494,47 @@ "bcp_47": "pt", "speakers": 237496885, "scores": [ + { + "model": "openai/gpt-4o-mini", + "bleu": 0.37370265193281843, + "chrf": 57.010201314973216, + "accuracy": 0.9666666666666667 + }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.27514792195783394, - "chrf": 45.901248962808694 + "chrf": 45.901248962808694, + "accuracy": 0.9666666666666667 + }, + { + "model": "mistralai/mistral-small-24b-instruct-2501", + "bleu": 0.3691905380990064, + "chrf": 54.842418095352954, + "accuracy": 0.9666666666666667 + }, + { + "model": "google/gemini-2.0-flash-001", + "bleu": 0.4020145367576223, + "chrf": 60.73156386707501, + "accuracy": 0.9 + }, + { + "model": "deepseek/deepseek-chat", + "bleu": 0.39831859400698993, + "chrf": 59.99225659809846, + "accuracy": 0.9666666666666667 + }, + { + "model": "microsoft/phi-4", + "bleu": 0.35576182901107084, + "chrf": 56.05856754270042, + "accuracy": 0.9 } ], - "bleu": 0.27514792195783394, - "chrf": 45.901248962808694, - "commonvoice_hours": 176.0, + "bleu": 0.36235601196089035, + "chrf": 55.756042730168126, + "accuracy": 0.9444444444444445, + "commonvoice_hours": 177.0, "commonvoice_locale": "pt", "population": { "AG": 1571, @@ -512,39 +560,16 @@ "bcp_47": "pa", "speakers": 203571210, "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.32250608979374484, - "chrf": 48.43043110055342 - }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3048037308116852, - "chrf": 48.4304965568793 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.2314793285308931, - "chrf": 37.53667069805556 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.37468960333154994, - "chrf": 53.381398796420086 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.359056244961623, - "chrf": 52.278269268214366 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.27331521881795146, - "chrf": 45.15087304059057 + "chrf": 48.4304965568793, + "accuracy": 0.9666666666666667 } ], - "bleu": 0.31097503604124127, - "chrf": 47.53468991011888, + "bleu": 0.3048037308116852, + "chrf": 48.4304965568793, + "accuracy": 0.9666666666666667, "commonvoice_hours": 2.3, "commonvoice_locale": "pa-IN", "population": { @@ -555,4397 +580,5 @@ "PK": 163450700, "SG": 9314 } - }, - { - "language_name": "Russian", - "bcp_47": "ru", - "speakers": 195841151, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.26108507692625094, - "chrf": 45.063308940468154 - } - ], - "bleu": 0.26108507692625094, - "chrf": 45.063308940468154, - "commonvoice_hours": 241.0, - "commonvoice_locale": "ru", - "population": { - "BG": 1602387, - "BY": 1137350, - "CA": 211087, - "CN": 13940, - "DE": 4809582, - "EE": 688027, - "FI": 45131, - "GE": 359730, - "IL": 954303, - "KG": 2147364, - "KZ": 13746168, - "LT": 2185168, - "LV": 714867, - "MD": 100935, - "MN": 4118, - "PL": 6890814, - "RU": 133218680, - "SJ": 1200, - "TJ": 1064840, - "TM": 663436, - "UA": 20204534, - "US": 798334, - "UZ": 4279156 - } - }, - { - "language_name": "Swahili", - "bcp_47": "sw", - "speakers": 171610296, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2709203338132304, - "chrf": 44.36399636969686 - } - ], - "bleu": 0.2709203338132304, - "chrf": 44.36399636969686, - "commonvoice_hours": 411.0, - "commonvoice_locale": "sw", - "population": { - "BI": 6408, - "CD": 50890000, - "KE": 35328414, - "MZ": 9330, - "SO": 235142, - "TZ": 52697520, - "UG": 32439750, - "YT": 2716, - "ZA": 1016 - } - }, - { - "language_name": "Indonesian", - "bcp_47": "id", - "speakers": 171207687, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.27441353638286026, - "chrf": 46.025445629112156 - } - ], - "bleu": 0.27441353638286026, - "chrf": 46.025445629112156, - "commonvoice_hours": 33.0, - "commonvoice_locale": "id", - "population": { - "ID": 170896640, - "NL": 311047 - } - }, - { - "language_name": "German", - "bcp_47": "de", - "speakers": 136350226, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.3338682761061998, - "chrf": 50.216731068308064 - } - ], - "bleu": 0.3338682761061998, - "chrf": 50.216731068308064, - "commonvoice_hours": 1357.0, - "commonvoice_locale": "de", - "population": { - "AT": 8593666, - "BE": 2578554, - "BG": 557352, - "BR": 1778414, - "CA": 294014, - "CH": 6134913, - "CZ": 1605375, - "DE": 72945327, - "DK": 2758623, - "FI": 1002901, - "FR": 3392410, - "GB": 5918499, - "GR": 530355, - "HU": 1758929, - "IT": 998443, - "KZ": 1221882, - "LI": 39137, - "LT": 382404, - "LU": 395880, - "NA": 23671, - "NL": 12269084, - "PL": 7273637, - "PY": 208559, - "RO": 44736, - "SI": 883126, - "SK": 1196932, - "US": 1563403 - } - }, - { - "language_name": "Japanese", - "bcp_47": "ja", - "speakers": 119729026, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2940100667664714, - "chrf": 46.403097021492236 - } - ], - "bleu": 0.2940100667664714, - "chrf": 46.403097021492236, - "commonvoice_hours": 222.0, - "commonvoice_locale": "ja", - "population": { - "BR": 444604, - "CA": 52772, - "JP": 119231650 - } - }, - { - "language_name": "Telugu", - "bcp_47": "te", - "speakers": 95478480, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.28964452051612244, - "chrf": 45.75290973782886 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2750887189010237, - "chrf": 46.31463752811596 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.1314237858560668, - "chrf": 27.750433857144273 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.35187073123584545, - "chrf": 52.04190885735586 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.2808664068178743, - "chrf": 48.221979700718 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.16468103557770178, - "chrf": 36.43981433605732 - } - ], - "bleu": 0.24892919981743908, - "chrf": 42.75361400287005, - "commonvoice_hours": 0.3, - "commonvoice_locale": "te", - "population": { - "IN": 95478480 - } - }, - { - "language_name": "Marathi", - "bcp_47": "mr", - "speakers": 92826300, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.235200323237626, - "chrf": 39.43789667749676 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2584800238292114, - "chrf": 44.69889855306244 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.1158656438579424, - "chrf": 27.118712681006436 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.3039098126596327, - "chrf": 48.26657289026638 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.23702154369195902, - "chrf": 43.039588362590955 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.14770612974379574, - "chrf": 35.63558588907277 - } - ], - "bleu": 0.21636391283669454, - "chrf": 39.69954250891596, - "commonvoice_hours": 20.0, - "commonvoice_locale": "mr", - "population": { - "IN": 92826300 - } - }, - { - "language_name": "Javanese", - "bcp_47": "jv", - "speakers": 91180665, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.23082586428104943, - "chrf": 41.42591471734489 - } - ], - "bleu": 0.23082586428104943, - "chrf": 41.42591471734489, - "commonvoice_hours": 0.0, - "commonvoice_locale": "jv", - "population": { - "ID": 90788840, - "MY": 391825 - } - }, - { - "language_name": "Vietnamese", - "bcp_47": "vi", - "speakers": 86222962, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.2790022403255029, - "chrf": 44.76577365559692 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.252552287345529, - "chrf": 43.351007120897606 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.17142561681893811, - "chrf": 34.23876708175754 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.32076560886563743, - "chrf": 49.46269756420099 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.269842412561934, - "chrf": 44.17252674642975 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.19118351096728373, - "chrf": 37.642090247027234 - } - ], - "bleu": 0.24746194614747083, - "chrf": 42.272143735985004, - "commonvoice_hours": 5.9, - "commonvoice_locale": "vi", - "population": { - "CA": 184701, - "CN": 6970, - "US": 1130973, - "VN": 84900318 - } - }, - { - "language_name": "Tamil", - "bcp_47": "ta", - "speakers": 85616159, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.2159676106476219, - "chrf": 38.592451568525966 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.24147470924504938, - "chrf": 41.98664320436057 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.11055269618146167, - "chrf": 26.929660798631293 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.2993653070835946, - "chrf": 46.760725728832945 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.22772498517043588, - "chrf": 40.963440857462984 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.14949134449145374, - "chrf": 33.188135588116566 - } - ], - "bleu": 0.2074294421366029, - "chrf": 38.07017629098839, - "commonvoice_hours": 234.0, - "commonvoice_locale": "ta", - "population": { - "CA": 184701, - "GB": 2104355, - "IN": 78239310, - "LK": 3433380, - "MU": 34484, - "MY": 1371388, - "RE": 118138, - "SG": 130403 - } - }, - { - "language_name": "Persian", - "bcp_47": "fa", - "speakers": 84710459, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2543274430452483, - "chrf": 44.798186786819564 - } - ], - "bleu": 0.2543274430452483, - "chrf": 44.798186786819564, - "commonvoice_hours": 370.0, - "commonvoice_locale": "fa", - "population": { - "AE": 189850, - "AF": 18321900, - "CA": 245012, - "IQ": 338192, - "IR": 63692475, - "OM": 43849, - "PK": 1541107, - "QA": 268859, - "TJ": 69215 - } - }, - { - "language_name": "Turkish", - "bcp_47": "tr", - "speakers": 80360704, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2848412390293461, - "chrf": 45.023438128876236 - } - ], - "bleu": 0.2848412390293461, - "chrf": 45.023438128876236, - "commonvoice_hours": 127.0, - "commonvoice_locale": "tr", - "population": { - "BG": 766359, - "CA": 37694, - "CY": 291336, - "DE": 2003992, - "GB": 131522, - "GR": 127285, - "MK": 74409, - "NL": 207365, - "RO": 27694, - "TR": 76276275, - "UA": 184476, - "UZ": 232297 - } - }, - { - "language_name": "Cantonese", - "bcp_47": "yue", - "speakers": 79654759, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2745324223697869, - "chrf": 46.18505717482203 - } - ], - "bleu": 0.2745324223697869, - "chrf": 46.18505717482203, - "commonvoice_hours": 203.0, - "commonvoice_locale": "yue", - "population": { - "CA": 640800, - "CN": 72489040, - "HK": 6524919 - } - }, - { - "language_name": "Korean", - "bcp_47": "ko", - "speakers": 78357046, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.21423807187509414, - "chrf": 42.31955622695572 - } - ], - "bleu": 0.21423807187509414, - "chrf": 42.31955622695572, - "commonvoice_hours": 1.7, - "commonvoice_locale": "ko", - "population": { - "BR": 44460, - "CA": 169623, - "CN": 2091030, - "JP": 652636, - "KP": 22566280, - "KR": 51835100, - "US": 997917 - } - }, - { - "language_name": "Italian", - "bcp_47": "it", - "speakers": 70247060, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.29744196180619636, - "chrf": 46.58884190721562 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2947459489130278, - "chrf": 48.23578446556176 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.26709213193768344, - "chrf": 43.42719841351614 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.34518602347709243, - "chrf": 51.45922256128821 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.3136120219290237, - "chrf": 49.456527591725454 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.26629405288011837, - "chrf": 44.95821657057411 - } - ], - "bleu": 0.29739535682385704, - "chrf": 47.35429858498022, - "commonvoice_hours": 362.0, - "commonvoice_locale": "it", - "population": { - "AT": 797350, - "AU": 483864, - "BR": 592805, - "CA": 343016, - "CH": 361372, - "DE": 5611179, - "FR": 1153419, - "GB": 131522, - "HR": 67644, - "IT": 59282565, - "MT": 256070, - "SI": 3995, - "SM": 30466, - "US": 1130973, - "VA": 820 - } - }, - { - "language_name": "Filipino", - "bcp_47": "fil", - "speakers": 67471096, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.3020075248846111, - "chrf": 45.148692233744825 - } - ], - "bleu": 0.3020075248846111, - "chrf": 45.148692233744825, - "commonvoice_hours": 0.0, - "commonvoice_locale": "tl", - "population": { - "CA": 565412, - "PH": 65508600, - "US": 1397084 - } - }, - { - "language_name": "Egyptian Arabic", - "bcp_47": "arz", - "speakers": 66639360, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.18353324091064518, - "chrf": 37.67362087042454 - } - ], - "bleu": 0.18353324091064518, - "chrf": 37.67362087042454, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "EG": 66639360 - } - }, - { - "language_name": "Gujarati", - "bcp_47": "gu", - "speakers": 61721799, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.27618486571036477, - "chrf": 45.30697533607047 - } - ], - "bleu": 0.27618486571036477, - "chrf": 45.30697533607047, - "commonvoice_hours": 0.0, - "commonvoice_locale": "gu-IN", - "population": { - "CA": 135699, - "GB": 1907072, - "IN": 59674050, - "KE": 4978 - } - }, - { - "language_name": "Thai", - "bcp_47": "th", - "speakers": 55181920, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2461561900137243, - "chrf": 43.886996888191135 - } - ], - "bleu": 0.2461561900137243, - "chrf": 43.886996888191135, - "commonvoice_hours": 172.0, - "commonvoice_locale": "th", - "population": { - "TH": 55181920 - } - }, - { - "language_name": "Kannada", - "bcp_47": "kn", - "speakers": 49065330, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.25650866519447973, - "chrf": 43.53706086839864 - } - ], - "bleu": 0.25650866519447973, - "chrf": 43.53706086839864, - "commonvoice_hours": 0.0, - "commonvoice_locale": "kn", - "population": { - "IN": 49065330 - } - }, - { - "language_name": "Malayalam", - "bcp_47": "ml", - "speakers": 43257484, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.23073727076678055, - "chrf": 41.53402359471923 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.21782657144614825, - "chrf": 41.630569782738704 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.1695641998407403, - "chrf": 33.20242503336964 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.3195014249623395, - "chrf": 50.399254960139395 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.2633223158501049, - "chrf": 45.473592535604965 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.19162873119255258, - "chrf": 38.21485785002488 - } - ], - "bleu": 0.23209675234311103, - "chrf": 41.7424539594328, - "commonvoice_hours": 2.8, - "commonvoice_locale": "ml", - "population": { - "AE": 699446, - "BH": 49665, - "IL": 7981, - "IN": 42434880, - "MY": 48978, - "QA": 6599, - "SG": 9935 - } - }, - { - "language_name": "Odia", - "bcp_47": "or", - "speakers": 42434880, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2616054244059909, - "chrf": 44.81898318236423 - } - ], - "bleu": 0.2616054244059909, - "chrf": 44.81898318236423, - "commonvoice_hours": 2.8, - "commonvoice_locale": "or", - "population": { - "IN": 42434880 - } - }, - { - "language_name": "Polish", - "bcp_47": "pl", - "speakers": 41077399, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.24382878885531348, - "chrf": 43.88802125000008 - } - ], - "bleu": 0.24382878885531348, - "chrf": 43.88802125000008, - "commonvoice_hours": 174.0, - "commonvoice_locale": "pl", - "population": { - "CA": 173393, - "CZ": 52442, - "DE": 232463, - "GB": 2630444, - "IL": 130132, - "PL": 36751008, - "RO": 2769, - "SK": 50598, - "UA": 1054150 - } - }, - { - "language_name": "Hausa", - "bcp_47": "ha", - "speakers": 40411882, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1493358875548207, - "chrf": 31.734228520521885 - } - ], - "bleu": 0.1493358875548207, - "chrf": 31.734228520521885, - "commonvoice_hours": 4.1, - "commonvoice_locale": "ha", - "population": { - "CM": 38843, - "GH": 252326, - "NE": 9336684, - "NG": 29963920, - "SD": 820109 - } - }, - { - "language_name": "Sindhi", - "bcp_47": "sd", - "speakers": 40329510, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.22524971121549384, - "chrf": 41.35718488130492 - } - ], - "bleu": 0.22524971121549384, - "chrf": 41.35718488130492, - "commonvoice_hours": 0.4, - "commonvoice_locale": "sd", - "population": { - "IN": 5304360, - "PK": 35025150 - } - }, - { - "language_name": "Malay", - "bcp_47": "ms", - "speakers": 38097307, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.27545115634664297, - "chrf": 46.5366529956061 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2445459295400275, - "chrf": 43.683725288188164 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.23380594556884363, - "chrf": 38.09288562825641 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.3521510571182875, - "chrf": 51.6107732437805 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.2984537737499322, - "chrf": 47.07139912930323 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.20974647653543713, - "chrf": 40.94603982712764 - } - ], - "bleu": 0.2690257231431951, - "chrf": 44.656912685377016, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ms", - "population": { - "BN": 455189, - "CC": 495, - "ID": 12283196, - "MY": 24489075, - "SG": 869352 - } - }, - { - "language_name": "Burmese", - "bcp_47": "my", - "speakers": 36559231, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.17529594258449108, - "chrf": 38.09396067803311 - } - ], - "bleu": 0.17529594258449108, - "chrf": 38.09396067803311, - "commonvoice_hours": 0.0, - "commonvoice_locale": "my", - "population": { - "BD": 341567, - "MM": 36217664 - } - }, - { - "language_name": "Amharic", - "bcp_47": "am", - "speakers": 35728475, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.16453710162389373, - "chrf": 34.89886094698235 - } - ], - "bleu": 0.16453710162389373, - "chrf": 34.89886094698235, - "commonvoice_hours": 1.8, - "commonvoice_locale": "am", - "population": { - "ET": 35677290, - "IL": 51185 - } - }, - { - "language_name": "Oromo", - "bcp_47": "om", - "speakers": 34897121, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.0691897827221633, - "chrf": 22.166735299684706 - } - ], - "bleu": 0.0691897827221633, - "chrf": 22.166735299684706, - "commonvoice_hours": 0.0, - "commonvoice_locale": "om", - "population": { - "ET": 34596160, - "KE": 251581, - "SO": 49380 - } - }, - { - "language_name": "Bhojpuri", - "bcp_47": "bho", - "speakers": 32934797, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.23176648838308359, - "chrf": 42.06617216025931 - } - ], - "bleu": 0.23176648838308359, - "chrf": 42.06617216025931, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IN": 30500070, - "MU": 372430, - "NP": 2062297 - } - }, - { - "language_name": "Uzbek", - "bcp_47": "uz", - "speakers": 32792780, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2038544554531401, - "chrf": 39.8364082393657 - } - ], - "bleu": 0.2038544554531401, - "chrf": 39.8364082393657, - "commonvoice_hours": 100.0, - "commonvoice_locale": "uz", - "population": { - "AF": 1722259, - "CN": 5576, - "TM": 497577, - "TR": 1968, - "UZ": 30565400 - } - }, - { - "language_name": "Azerbaijani", - "bcp_47": "az", - "speakers": 32446682, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1911853993562902, - "chrf": 38.284553121320506 - } - ], - "bleu": 0.1911853993562902, - "chrf": 38.284553121320506, - "commonvoice_hours": 0.5, - "commonvoice_locale": "az", - "population": { - "AM": 0, - "AZ": 10093536, - "IQ": 699709, - "IR": 20381592, - "RU": 131801, - "TR": 1140044 - } - }, - { - "language_name": "Sundanese", - "bcp_47": "su", - "speakers": 32043120, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.18638464691782505, - "chrf": 36.390067106438586 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.22413489641063433, - "chrf": 39.08607675576149 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.12396443500389862, - "chrf": 27.44605100630464 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.31100961611618916, - "chrf": 48.684591434160616 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.23467068441612768, - "chrf": 40.54860164135514 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.1315217916431994, - "chrf": 32.71394871178255 - } - ], - "bleu": 0.20194767841797903, - "chrf": 37.47822277596717, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "ID": 32043120 - } - }, - { - "language_name": "Dutch", - "bcp_47": "nl", - "speakers": 31765645, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2592795402705898, - "chrf": 44.356739380497835 - } - ], - "bleu": 0.2592795402705898, - "chrf": 44.356739380497835, - "commonvoice_hours": 114.0, - "commonvoice_locale": "nl", - "population": { - "AW": 115845, - "BE": 6446385, - "BQ": 1600, - "CA": 56541, - "CW": 12108, - "DE": 7214373, - "FR": 88203, - "NL": 17280400, - "SR": 548612, - "SX": 1578 - } - }, - { - "language_name": "Moroccan Arabic", - "bcp_47": "ary", - "speakers": 30938679, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1795140543637709, - "chrf": 37.426822543484725 - } - ], - "bleu": 0.1795140543637709, - "chrf": 37.426822543484725, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "MA": 30938679 - } - }, - { - "language_name": "Ukrainian", - "bcp_47": "uk", - "speakers": 29348975, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.2564463888571809, - "chrf": 41.5181715644979 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2922812040972885, - "chrf": 46.82127627710616 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.24225796102393954, - "chrf": 39.42058485499366 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.3452563778145038, - "chrf": 50.96771482708702 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.3292127494173498, - "chrf": 48.580427643849056 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.2452825737163755, - "chrf": 42.05493360532406 - } - ], - "bleu": 0.28512287582110635, - "chrf": 44.893851462142976, - "commonvoice_hours": 98.0, - "commonvoice_locale": "uk", - "population": { - "CA": 75388, - "MD": 471030, - "PL": 149301, - "RS": 0, - "SK": 103371, - "UA": 28549885 - } - }, - { - "language_name": "Yoruba", - "bcp_47": "yo", - "speakers": 28685568, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.09852676389614487, - "chrf": 25.656950226425916 - } - ], - "bleu": 0.09852676389614487, - "chrf": 25.656950226425916, - "commonvoice_hours": 5.9, - "commonvoice_locale": "yo", - "population": { - "BJ": 861928, - "NG": 27823640 - } - }, - { - "language_name": "Igbo", - "bcp_47": "ig", - "speakers": 27823640, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1549827013842116, - "chrf": 35.431035675871804 - } - ], - "bleu": 0.1549827013842116, - "chrf": 35.431035675871804, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ig", - "population": { - "NG": 27823640 - } - }, - { - "language_name": "Cebuano", - "bcp_47": "ceb", - "speakers": 26203440, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.27783470672988303, - "chrf": 43.53549785827671 - } - ], - "bleu": 0.27783470672988303, - "chrf": 43.53549785827671, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "PH": 26203440 - } - }, - { - "language_name": "Awadhi", - "bcp_47": "awa", - "speakers": 25862924, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2554810263222905, - "chrf": 42.35795046006446 - } - ], - "bleu": 0.2554810263222905, - "chrf": 42.35795046006446, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IN": 25195710, - "NP": 667214 - } - }, - { - "language_name": "Malagasy", - "bcp_47": "mg", - "speakers": 24260130, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.15163299980391426, - "chrf": 32.41299083831688 - } - ], - "bleu": 0.15163299980391426, - "chrf": 32.41299083831688, - "commonvoice_hours": 0.0, - "commonvoice_locale": "mg", - "population": { - "MG": 24260130 - } - }, - { - "language_name": "Romanian", - "bcp_47": "ro", - "speakers": 22187408, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.33899025568959984, - "chrf": 49.419389839471826 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.26666997541189236, - "chrf": 44.76525386460237 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.24172488724041316, - "chrf": 37.98045602918644 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.37365302832845815, - "chrf": 52.777299654432 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.332600965807992, - "chrf": 49.14582652136321 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.2510789925018768, - "chrf": 42.27877315264307 - } - ], - "bleu": 0.30078635083003874, - "chrf": 46.06116651028316, - "commonvoice_hours": 21.0, - "commonvoice_locale": "ro", - "population": { - "CA": 101774, - "HU": 96741, - "IL": 320993, - "MD": 2119635, - "RO": 19172610, - "RS": 147256, - "UA": 228399 - } - }, - { - "language_name": "Nepali", - "bcp_47": "ne", - "speakers": 20903374, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.26199282928489126, - "chrf": 44.7560662712792 - } - ], - "bleu": 0.26199282928489126, - "chrf": 44.7560662712792, - "commonvoice_hours": 1.3, - "commonvoice_locale": "ne-NP", - "population": { - "BT": 132994, - "IN": 7426104, - "NP": 13344276 - } - }, - { - "language_name": "Maithili", - "bcp_47": "mai", - "speakers": 19249149, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.23975507119180453, - "chrf": 41.8894484718934 - } - ], - "bleu": 0.23975507119180453, - "chrf": 41.8894484718934, - "commonvoice_hours": 0.0, - "commonvoice_locale": "mai", - "population": { - "IN": 15913080, - "NP": 3336069 - } - }, - { - "language_name": "Assamese", - "bcp_47": "as", - "speakers": 17239170, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.19363225565136952, - "chrf": 38.96237165376663 - } - ], - "bleu": 0.19363225565136952, - "chrf": 38.96237165376663, - "commonvoice_hours": 2.8, - "commonvoice_locale": "as", - "population": { - "IN": 17239170 - } - }, - { - "language_name": "Nyanja", - "bcp_47": "ny", - "speakers": 17026781, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.09504458945778768, - "chrf": 27.576030002164906 - } - ], - "bleu": 0.09504458945778768, - "chrf": 27.576030002164906, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ny", - "population": { - "MW": 13353858, - "MZ": 782553, - "ZM": 2613990, - "ZW": 276380 - } - }, - { - "language_name": "Somali", - "bcp_47": "so", - "speakers": 16911645, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.2024994684991584, - "chrf": 37.37281822856629 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1532133716194419, - "chrf": 32.05620028647162 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.043408496427191995, - "chrf": 15.267715935014895 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.3048371831537258, - "chrf": 46.24092673305936 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.21360545410615966, - "chrf": 38.32357547187653 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.06484340154849859, - "chrf": 22.781181465233722 - } - ], - "bleu": 0.16373456255902938, - "chrf": 32.00706968670374, - "commonvoice_hours": 0.0, - "commonvoice_locale": "so", - "population": { - "CA": 49002, - "DJ": 377940, - "ET": 6486780, - "GB": 131522, - "KE": 695863, - "SO": 9170538 - } - }, - { - "language_name": "Magahi", - "bcp_47": "mag", - "speakers": 15913080, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.29925558767802407, - "chrf": 47.539870710677974 - } - ], - "bleu": 0.29925558767802407, - "chrf": 47.539870710677974, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IN": 15913080 - } - }, - { - "language_name": "Serbian", - "bcp_47": "sr", - "speakers": 15602410, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.26029402164210574, - "chrf": 44.75391848259974 - } - ], - "bleu": 0.26029402164210574, - "chrf": 44.75391848259974, - "commonvoice_hours": 7.4, - "commonvoice_locale": "sr", - "population": { - "BA": 767118, - "CA": 64080, - "ME": 640352, - "RO": 25563, - "RS": 13884096, - "RU": 4960, - "TR": 22965, - "XK": 193276 - } - }, - { - "language_name": "Sinhala", - "bcp_47": "si", - "speakers": 15564656, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.20259734060180434, - "chrf": 39.00188422851495 - } - ], - "bleu": 0.20259734060180434, - "chrf": 39.00188422851495, - "commonvoice_hours": 0.0, - "commonvoice_locale": "si", - "population": { - "LK": 15564656 - } - }, - { - "language_name": "Khmer", - "bcp_47": "km", - "speakers": 15065030, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.21699232146684352, - "chrf": 41.99979148031644 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.21417349432612984, - "chrf": 41.35771196976409 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.03287369352293625, - "chrf": 14.140423341647319 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.34264229339556035, - "chrf": 51.55826045210756 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.24630515818736093, - "chrf": 44.60963216433486 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.11830648687368288, - "chrf": 31.534012306448343 - } - ], - "bleu": 0.1952155746287523, - "chrf": 37.53330528576977, - "commonvoice_hours": 0.0, - "commonvoice_locale": "km", - "population": { - "KH": 15065030 - } - }, - { - "language_name": "Chhattisgarhi", - "bcp_47": "hne", - "speakers": 14586990, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2212498883000727, - "chrf": 39.102114067893005 - } - ], - "bleu": 0.2212498883000727, - "chrf": 39.102114067893005, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IN": 14586990 - } - }, - { - "language_name": "Nigerian Fulfulde", - "bcp_47": "fuv", - "speakers": 14339876, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.08190470208193343, - "chrf": 23.041107899884107 - } - ], - "bleu": 0.08190470208193343, - "chrf": 23.041107899884107, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "NG": 14339876 - } - }, - { - "language_name": "Zulu", - "bcp_47": "zu", - "speakers": 13973830, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.15449146502209737, - "chrf": 33.087212745906356 - } - ], - "bleu": 0.15449146502209737, - "chrf": 33.087212745906356, - "commonvoice_hours": 0.0, - "commonvoice_locale": "zu", - "population": { - "LS": 275706, - "MW": 69949, - "MZ": 1806, - "SZ": 75105, - "ZA": 13551264 - } - }, - { - "language_name": "Kazakh", - "bcp_47": "kk", - "speakers": 13637392, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2392246097188628, - "chrf": 42.67763456362536 - } - ], - "bleu": 0.2392246097188628, - "chrf": 42.67763456362536, - "commonvoice_hours": 2.1, - "commonvoice_locale": "kk", - "population": { - "AF": 2015, - "CN": 1184917, - "IR": 2972, - "KZ": 12218816, - "MN": 228098, - "TR": 574 - } - }, - { - "language_name": "Czech", - "bcp_47": "cs", - "speakers": 13045532, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2844520855192069, - "chrf": 47.327010269160255 - } - ], - "bleu": 0.2844520855192069, - "chrf": 47.327010269160255, - "commonvoice_hours": 74.0, - "commonvoice_locale": "cs", - "population": { - "CZ": 10488450, - "SK": 2557082 - } - }, - { - "language_name": "Swedish", - "bcp_47": "sv", - "speakers": 12932871, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.31838456223051165, - "chrf": 48.08041424667649 - } - ], - "bleu": 0.31838456223051165, - "chrf": 48.08041424667649, - "commonvoice_hours": 47.0, - "commonvoice_locale": "sv-SE", - "population": { - "AX": 25938, - "DK": 763023, - "FI": 2451535, - "SE": 9692375 - } - }, - { - "language_name": "Hungarian", - "bcp_47": "hu", - "speakers": 12443430, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2517614908428288, - "chrf": 42.77520440394925 - } - ], - "bleu": 0.2517614908428288, - "chrf": 42.77520440394925, - "commonvoice_hours": 92.0, - "commonvoice_locale": "hu", - "population": { - "AT": 23035, - "CA": 49002, - "HU": 9771830, - "IL": 86755, - "RO": 1405991, - "RS": 336584, - "SI": 9252, - "SK": 598466, - "UA": 162515 - } - }, - { - "language_name": "Greek", - "bcp_47": "el", - "speakers": 12292242, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.24888370153898132, - "chrf": 42.743109839531535 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.27269196827838943, - "chrf": 45.53483489961114 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.21351544070708506, - "chrf": 39.64425368969459 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.3500489758234636, - "chrf": 51.1729677922222 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.32858632704534785, - "chrf": 49.70348294319061 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.1903000734693107, - "chrf": 39.41911092527115 - } - ], - "bleu": 0.2673377478104297, - "chrf": 44.7029600149202, - "commonvoice_hours": 20.0, - "commonvoice_locale": "el", - "population": { - "AL": 58417, - "CA": 124391, - "CY": 1203346, - "DE": 304607, - "EG": 63516, - "GR": 10501029, - "IT": 21841, - "RO": 4048, - "TR": 4019, - "UA": 7028 - } - }, - { - "language_name": "Shona", - "bcp_47": "sn", - "speakers": 11782503, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.10438047654339373, - "chrf": 28.358507205354343 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1054140213254438, - "chrf": 28.275522446025796 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.049580893458705456, - "chrf": 18.458313551880636 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.20979752102494492, - "chrf": 38.7737276002067 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.14066476436038525, - "chrf": 32.050887533047465 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.060530921002659346, - "chrf": 22.35054590384825 - } - ], - "bleu": 0.11172809961925541, - "chrf": 28.04458404006053, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "ZW": 11782503 - } - }, - { - "language_name": "Central Kurdish", - "bcp_47": "ckb", - "speakers": 11086549, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.20654412682006296, - "chrf": 38.66802600490074 - } - ], - "bleu": 0.20654412682006296, - "chrf": 38.66802600490074, - "commonvoice_hours": 135.0, - "commonvoice_locale": "ckb", - "population": { - "IQ": 7774540, - "IR": 3312009 - } - }, - { - "language_name": "Kinyarwanda", - "bcp_47": "rw", - "speakers": 11083625, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.14006909985587948, - "chrf": 31.739340000779823 - } - ], - "bleu": 0.14006909985587948, - "chrf": 31.739340000779823, - "commonvoice_hours": 2002.0, - "commonvoice_locale": "rw", - "population": { - "CD": 386764, - "RW": 9788548, - "UG": 908313 - } - }, - { - "language_name": "Wolof", - "bcp_47": "wo", - "speakers": 11025494, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.08408628490585719, - "chrf": 23.767561072306325 - } - ], - "bleu": 0.08408628490585719, - "chrf": 23.767561072306325, - "commonvoice_hours": 0.0, - "commonvoice_locale": "wo", - "population": { - "MR": 10014, - "SN": 11015480 - } - }, - { - "language_name": "Tunisian Arabic", - "bcp_47": "aeb", - "speakers": 10549080, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.23738824104522893, - "chrf": 42.85642541970995 - } - ], - "bleu": 0.23738824104522893, - "chrf": 42.85642541970995, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "TN": 10549080 - } - }, - { - "language_name": "Iloko", - "bcp_47": "ilo", - "speakers": 10481376, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.20685666710862224, - "chrf": 37.066073786659494 - } - ], - "bleu": 0.20685666710862224, - "chrf": 37.066073786659494, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "PH": 10481376 - } - }, - { - "language_name": "Xhosa", - "bcp_47": "xh", - "speakers": 10182944, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1143143326923908, - "chrf": 28.93119601652647 - } - ], - "bleu": 0.1143143326923908, - "chrf": 28.93119601652647, - "commonvoice_hours": 0.0, - "commonvoice_locale": "xh", - "population": { - "LS": 19496, - "ZA": 10163448 - } - }, - { - "language_name": "Tigrinya", - "bcp_47": "ti", - "speakers": 10145911, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.08532345270447181, - "chrf": 25.148121650983146 - } - ], - "bleu": 0.08532345270447181, - "chrf": 25.148121650983146, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ti", - "population": { - "ER": 3648720, - "ET": 6486780, - "IL": 10411 - } - }, - { - "language_name": "Belarusian", - "bcp_47": "be", - "speakers": 10064517, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.18341973561481445, - "chrf": 39.828398360117035 - } - ], - "bleu": 0.18341973561481445, - "chrf": 39.828398360117035, - "commonvoice_hours": 1804.0, - "commonvoice_locale": "be", - "population": { - "BY": 9477920, - "PL": 222037, - "UA": 364560 - } - }, - { - "language_name": "Luba-Lulua", - "bcp_47": "lua", - "speakers": 9770880, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.11581604983636683, - "chrf": 28.08252510561598 - } - ], - "bleu": 0.11581604983636683, - "chrf": 28.08252510561598, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "CD": 9770880 - } - }, - { - "language_name": "Tajik", - "bcp_47": "tg", - "speakers": 9644223, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1846236171638531, - "chrf": 39.50004300248175 - } - ], - "bleu": 0.1846236171638531, - "chrf": 39.50004300248175, - "commonvoice_hours": 0.0, - "commonvoice_locale": "tg", - "population": { - "PK": 770553, - "TJ": 8873670 - } - }, - { - "language_name": "Umbundu", - "bcp_47": "umb", - "speakers": 9431467, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.05520341910203098, - "chrf": 20.443280736080066 - } - ], - "bleu": 0.05520341910203098, - "chrf": 20.443280736080066, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "AO": 9431467 - } - }, - { - "language_name": "Bambara", - "bcp_47": "bm", - "speakers": 9385632, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.07227674667013836, - "chrf": 22.136582910055218 - } - ], - "bleu": 0.07227674667013836, - "chrf": 22.136582910055218, - "commonvoice_hours": 0.0, - "commonvoice_locale": "bm", - "population": { - "ML": 9385632 - } - }, - { - "language_name": "Afrikaans", - "bcp_47": "af", - "speakers": 9318845, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.3277177864074156, - "chrf": 48.61873150516328 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.31538459755111, - "chrf": 47.75260784921264 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.26710030799119333, - "chrf": 38.869810330084285 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.4361740814378139, - "chrf": 56.524736028951835 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.37149647257024515, - "chrf": 53.20003536968165 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.2883662842075808, - "chrf": 45.203083386764426 - } - ], - "bleu": 0.3343732550275598, - "chrf": 48.361500744976354, - "commonvoice_hours": 0.5, - "commonvoice_locale": "af", - "population": { - "BW": 6025, - "NA": 1972552, - "ZA": 7340268 - } - }, - { - "language_name": "Kikuyu", - "bcp_47": "ki", - "speakers": 9099743, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.0883556207236924, - "chrf": 24.868277910094278 - } - ], - "bleu": 0.0883556207236924, - "chrf": 24.868277910094278, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ki", - "population": { - "KE": 9099743 - } - }, - { - "language_name": "Haitian Creole", - "bcp_47": "ht", - "speakers": 8964918, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1872609836464467, - "chrf": 37.00166091804026 - } - ], - "bleu": 0.1872609836464467, - "chrf": 37.00166091804026, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ht", - "population": { - "HT": 8964918 - } - }, - { - "language_name": "Catalan", - "bcp_47": "ca", - "speakers": 8679139, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.29445274007068095, - "chrf": 47.46074814103581 - } - ], - "bleu": 0.29445274007068095, - "chrf": 47.46074814103581, - "commonvoice_hours": 2842.0, - "commonvoice_locale": "ca", - "population": { - "AD": 39270, - "ES": 8502686, - "FR": 115342, - "IT": 21841 - } - }, - { - "language_name": "Hebrew", - "bcp_47": "he", - "speakers": 8675480, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2824719214447976, - "chrf": 46.279771845770604 - } - ], - "bleu": 0.2824719214447976, - "chrf": 46.279771845770604, - "commonvoice_hours": 1.1, - "commonvoice_locale": "he", - "population": { - "IL": 8675480 - } - }, - { - "language_name": "Mossi", - "bcp_47": "mos", - "speakers": 8334160, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.08102496244147746, - "chrf": 22.184797797745208 - } - ], - "bleu": 0.08102496244147746, - "chrf": 22.184797797745208, - "commonvoice_hours": 0.0, - "commonvoice_locale": "mos", - "population": { - "BF": 8334160 - } - }, - { - "language_name": "Kimbundu", - "bcp_47": "kmb", - "speakers": 8130575, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.07329500673809967, - "chrf": 22.838222030254723 - } - ], - "bleu": 0.07329500673809967, - "chrf": 22.838222030254723, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "AO": 8130575 - } - }, - { - "language_name": "Uyghur", - "bcp_47": "ug", - "speakers": 8052967, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.18397910035998616, - "chrf": 35.50154017198535 - } - ], - "bleu": 0.18397910035998616, - "chrf": 35.50154017198535, - "commonvoice_hours": 361.0, - "commonvoice_locale": "ug", - "population": { - "AF": 3005, - "CN": 7667110, - "KZ": 381838, - "MN": 1014 - } - }, - { - "language_name": "Minangkabau", - "bcp_47": "min", - "speakers": 8010780, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.22401269807719826, - "chrf": 40.34678123737912 - } - ], - "bleu": 0.22401269807719826, - "chrf": 40.34678123737912, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "ID": 8010780 - } - }, - { - "language_name": "Bulgarian", - "bcp_47": "bg", - "speakers": 7878315, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.24723779163445408, - "chrf": 45.55403997487483 - } - ], - "bleu": 0.24723779163445408, - "chrf": 45.55403997487483, - "commonvoice_hours": 16.0, - "commonvoice_locale": "bg", - "population": { - "BG": 6966900, - "GR": 28639, - "MD": 316263, - "RO": 6817, - "TR": 344474, - "UA": 215222 - } - }, - { - "language_name": "Standard Moroccan Tamazight", - "bcp_47": "zgh", - "speakers": 7823574, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.0366272802298245, - "chrf": 15.463910171449278 - } - ], - "bleu": 0.0366272802298245, - "chrf": 15.463910171449278, - "commonvoice_hours": 1.3, - "commonvoice_locale": "zgh", - "population": { - "MA": 7823574 - } - }, - { - "language_name": "Bosnian", - "bcp_47": "bs", - "speakers": 7594468, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.3051247921441283, - "chrf": 48.3731481201238 - } - ], - "bleu": 0.3051247921441283, - "chrf": 48.3731481201238, - "commonvoice_hours": 0.0, - "commonvoice_locale": "bs", - "population": { - "BA": 7594468 - } - }, - { - "language_name": "Rundi", - "bcp_47": "rn", - "speakers": 7475454, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.0957054530624, - "chrf": 28.269164219007447 - } - ], - "bleu": 0.0957054530624, - "chrf": 28.269164219007447, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "BI": 7475454 - } - }, - { - "language_name": "Santali", - "bcp_47": "sat", - "speakers": 7293495, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.11554870024021023, - "chrf": 29.549934477174254 - } - ], - "bleu": 0.11554870024021023, - "chrf": 29.549934477174254, - "commonvoice_hours": 0.5, - "commonvoice_locale": "sat", - "population": { - "IN": 7293495 - } - }, - { - "language_name": "Danish", - "bcp_47": "da", - "speakers": 7072056, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.3512857581168584, - "chrf": 51.12453594191334 - } - ], - "bleu": 0.3512857581168584, - "chrf": 51.12453594191334, - "commonvoice_hours": 13.0, - "commonvoice_locale": "da", - "population": { - "DE": 1603194, - "DK": 5458551, - "GL": 8066, - "IS": 2245 - } - }, - { - "language_name": "Turkmen", - "bcp_47": "tk", - "speakers": 6870838, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.15410064596625964, - "chrf": 36.3602620147462 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.14820890318014426, - "chrf": 34.78567738931005 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.07938993687949465, - "chrf": 24.385063397578556 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.2761834512123037, - "chrf": 47.41646574422384 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.1718597543270264, - "chrf": 38.84762953076257 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.10118596975980092, - "chrf": 28.31840639855478 - } - ], - "bleu": 0.15515477688750492, - "chrf": 35.018917412529326, - "commonvoice_hours": 2.8, - "commonvoice_locale": "tk", - "population": { - "AF": 622945, - "IR": 2377852, - "TM": 3870041 - } - }, - { - "language_name": "Kurdish", - "bcp_47": "ku", - "speakers": 6866757, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.14440915289810186, - "chrf": 31.3622869278075 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.15987085387022903, - "chrf": 35.18523232721762 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.10987778830152085, - "chrf": 25.91454653951599 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.28985769410441137, - "chrf": 46.64309582634758 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.1865343501300658, - "chrf": 37.28483871763787 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.10000019378200214, - "chrf": 26.406217887319293 - } - ], - "bleu": 0.16509167218105517, - "chrf": 33.79936970430764, - "commonvoice_hours": 69.0, - "commonvoice_locale": "kmr", - "population": { - "AM": 99704, - "AZ": 24494, - "DE": 529054, - "GE": 35573, - "LB": 92983, - "SY": 1551872, - "TM": 22115, - "TR": 4510962 - } - }, - { - "language_name": "Croatian", - "bcp_47": "hr", - "speakers": 6813164, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2290484937313612, - "chrf": 41.79307139114272 - } - ], - "bleu": 0.2290484937313612, - "chrf": 41.79307139114272, - "commonvoice_hours": 0.0, - "commonvoice_locale": "hr", - "population": { - "AT": 106313, - "BA": 460271, - "CA": 45233, - "DE": 633262, - "HR": 4185472, - "HU": 31270, - "IT": 3495, - "RS": 65213, - "SI": 1282635 - } - }, - { - "language_name": "Albanian", - "bcp_47": "sq", - "speakers": 6791906, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.26490711574268994, - "chrf": 44.54318851690701 - } - ], - "bleu": 0.26490711574268994, - "chrf": 44.54318851690701, - "commonvoice_hours": 8.8, - "commonvoice_locale": "sq", - "population": { - "AL": 3074580, - "GR": 9971, - "ME": 48179, - "MK": 531492, - "RS": 1332312, - "TR": 17224, - "XK": 1778148 - } - }, - { - "language_name": "Slovak", - "bcp_47": "sk", - "speakers": 6680269, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2826836020834733, - "chrf": 45.73110764547564 - } - ], - "bleu": 0.2826836020834733, - "chrf": 45.73110764547564, - "commonvoice_hours": 40.0, - "commonvoice_locale": "sk", - "population": { - "CZ": 1712400, - "HU": 11726, - "RS": 59603, - "SK": 4896540 - } - }, - { - "language_name": "Dyula", - "bcp_47": "dyu", - "speakers": 6667328, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.0633017924291756, - "chrf": 21.71247042480193 - } - ], - "bleu": 0.0633017924291756, - "chrf": 21.71247042480193, - "commonvoice_hours": 0.3, - "commonvoice_locale": "dyu", - "population": { - "BF": 6667328 - } - }, - { - "language_name": "Mongolian", - "bcp_47": "mn", - "speakers": 6572846, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1950781841033538, - "chrf": 41.09779803278993 - } - ], - "bleu": 0.1950781841033538, - "chrf": 41.09779803278993, - "commonvoice_hours": 46.0, - "commonvoice_locale": "mn", - "population": { - "CN": 3624452, - "MN": 2946268, - "RU": 2126 - } - }, - { - "language_name": "Southern Sotho", - "bcp_47": "st", - "speakers": 6390567, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.12381413258013083, - "chrf": 28.651371806370722 - } - ], - "bleu": 0.12381413258013083, - "chrf": 28.651371806370722, - "commonvoice_hours": 0.0, - "commonvoice_locale": "st", - "population": { - "LS": 1929943, - "ZA": 4460624 - } - }, - { - "language_name": "Tswana", - "bcp_47": "tn", - "speakers": 6113428, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.09139131060492443, - "chrf": 25.97258334431281 - } - ], - "bleu": 0.09139131060492443, - "chrf": 25.97258334431281, - "commonvoice_hours": 4.2, - "commonvoice_locale": "tn", - "population": { - "BW": 1436683, - "NA": 14728, - "ZA": 4630015, - "ZW": 32002 - } - }, - { - "language_name": "Guarani", - "bcp_47": "gn", - "speakers": 5827107, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.12296923497272805, - "chrf": 29.040238903610113 - } - ], - "bleu": 0.12296923497272805, - "chrf": 29.040238903610113, - "commonvoice_hours": 3.7, - "commonvoice_locale": "gn", - "population": { - "AR": 21375, - "BO": 52380, - "PY": 5753352 - } - }, - { - "language_name": "Finnish", - "bcp_47": "fi", - "speakers": 5736842, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2306868672081301, - "chrf": 42.094559446408105 - } - ], - "bleu": 0.2306868672081301, - "chrf": 42.094559446408105, - "commonvoice_hours": 15.0, - "commonvoice_locale": "fi", - "population": { - "EE": 258010, - "FI": 5237370, - "RU": 17007, - "SE": 224455 - } - }, - { - "language_name": "Ganda", - "bcp_47": "lg", - "speakers": 5622890, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.09865217050437662, - "chrf": 25.926064361947446 - } - ], - "bleu": 0.09865217050437662, - "chrf": 25.926064361947446, - "commonvoice_hours": 437.0, - "commonvoice_locale": "lg", - "population": { - "UG": 5622890 - } - }, - { - "language_name": "Kashmiri", - "bcp_47": "ks", - "speakers": 5598085, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1344939664526747, - "chrf": 32.68903932403509 - } - ], - "bleu": 0.1344939664526747, - "chrf": 32.68903932403509, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IN": 5436969, - "PK": 161116 - } - }, - { - "language_name": "Norwegian Bokmål", - "bcp_47": "nb", - "speakers": 5468932, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.3568538739752233, - "chrf": 50.18863565152686 - } - ], - "bleu": 0.3568538739752233, - "chrf": 50.18863565152686, - "commonvoice_hours": 0.1, - "commonvoice_locale": "nb-NO", - "population": { - "NO": 5467440, - "SJ": 1492 - } - }, - { - "language_name": "Bemba", - "bcp_47": "bem", - "speakers": 5402246, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.07496563614353445, - "chrf": 23.431580504923282 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.10425825663987873, - "chrf": 26.156513526794825 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.048552315311727906, - "chrf": 17.052116746764433 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.20820762589055672, - "chrf": 37.899164894105404 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.13037194233770932, - "chrf": 29.069219847377738 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.07919429950933718, - "chrf": 22.45526755754465 - } - ], - "bleu": 0.10759167930545738, - "chrf": 26.01064384625172, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "ZM": 5402246 - } - }, - { - "language_name": "Armenian", - "bcp_47": "hy", - "speakers": 5317273, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2525113198548088, - "chrf": 44.115149170486895 - } - ], - "bleu": 0.2525113198548088, - "chrf": 44.115149170486895, - "commonvoice_hours": 31.0, - "commonvoice_locale": "hy-AM", - "population": { - "AM": 2960894, - "CY": 2787, - "GE": 279790, - "IR": 203816, - "LB": 284420, - "RU": 1190465, - "SY": 349171, - "TR": 45930 - } - }, - { - "language_name": "Northern Sotho", - "bcp_47": "nso", - "speakers": 5307578, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.1287306186367617, - "chrf": 28.538499437787404 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.11431860079235977, - "chrf": 29.789689541768464 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.048032427671766596, - "chrf": 16.511000736562355 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.277532484522071, - "chrf": 43.725160462861595 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.1559013863573944, - "chrf": 31.344526297642712 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.08683694629684643, - "chrf": 24.176752802670837 - } - ], - "bleu": 0.13522541071286664, - "chrf": 29.014271546548898, - "commonvoice_hours": 0.0, - "commonvoice_locale": "nso", - "population": { - "ZA": 5307578 - } - }, - { - "language_name": "Luo (Kenya and Tanzania)", - "bcp_47": "luo", - "speakers": 5245734, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.07123028733548639, - "chrf": 21.84956735049654 - } - ], - "bleu": 0.07123028733548639, - "chrf": 21.84956735049654, - "commonvoice_hours": 30.0, - "commonvoice_locale": "luo", - "population": { - "KE": 5245734 - } - }, - { - "language_name": "Tok Pisin", - "bcp_47": "tpi", - "speakers": 5154217, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.17665711931817996, - "chrf": 35.46692292627831 - } - ], - "bleu": 0.17665711931817996, - "chrf": 35.46692292627831, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "PG": 5154217 - } - }, - { - "language_name": "Lao", - "bcp_47": "lo", - "speakers": 5138706, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.17291556794348653, - "chrf": 35.37210957380201 - } - ], - "bleu": 0.17291556794348653, - "chrf": 35.37210957380201, - "commonvoice_hours": 0.2, - "commonvoice_locale": "lo", - "population": { - "LA": 5138706 - } - }, - { - "language_name": "Tsonga", - "bcp_47": "ts", - "speakers": 4880932, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.1264498146181144, - "chrf": 26.535888048377064 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.09614725376527729, - "chrf": 26.35974604226704 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.043920591728788254, - "chrf": 15.171343812958735 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.2843690426617385, - "chrf": 43.51952547276378 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.10072669531344912, - "chrf": 24.778318092682298 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.0708900783780892, - "chrf": 24.114324582460767 - } - ], - "bleu": 0.12041724607757613, - "chrf": 26.746524341918285, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ts", - "population": { - "MZ": 2377758, - "SZ": 18776, - "ZA": 2484398 - } - }, - { - "language_name": "Balinese", - "bcp_47": "ban", - "speakers": 4806468, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.20937766416587725, - "chrf": 37.802202729584685 - } - ], - "bleu": 0.20937766416587725, - "chrf": 37.802202729584685, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "ID": 4806468 - } - }, - { - "language_name": "Ewe", - "bcp_47": "ee", - "speakers": 4690857, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.06328122760447334, - "chrf": 21.930232101484705 - } - ], - "bleu": 0.06328122760447334, - "chrf": 21.930232101484705, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ee", - "population": { - "GH": 3227422, - "TG": 1463435 - } - }, - { - "language_name": "Buginese", - "bcp_47": "bug", - "speakers": 4298211, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.11888625287150432, - "chrf": 26.566035803697112 - } - ], - "bleu": 0.11888625287150432, - "chrf": 26.566035803697112, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "ID": 4272416, - "MY": 25795 - } - }, - { - "language_name": "Goan Konkani", - "bcp_47": "gom", - "speakers": 4243488, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.17517997036818814, - "chrf": 35.662782661637515 - } - ], - "bleu": 0.17517997036818814, - "chrf": 35.662782661637515, - "commonvoice_hours": 0.0, - "commonvoice_locale": "gom", - "population": { - "IN": 4243488 - } - }, - { - "language_name": "Kamba", - "bcp_47": "kam", - "speakers": 4068120, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.09766297423802607, - "chrf": 24.328994687372596 - } - ], - "bleu": 0.09766297423802607, - "chrf": 24.328994687372596, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "KE": 4068120 - } - }, - { - "language_name": "Banjar", - "bcp_47": "bjn", - "speakers": 4010288, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.21429523594040997, - "chrf": 37.997802850700054 - } - ], - "bleu": 0.21429523594040997, - "chrf": 37.997802850700054, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "ID": 4005390, - "MY": 4898 - } - }, - { - "language_name": "Lombard", - "bcp_47": "lmo", - "speakers": 3901518, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.19986098660959015, - "chrf": 38.27348032712485 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.21159778572935684, - "chrf": 40.44461475492081 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.1588300738463149, - "chrf": 31.614789846388156 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.30563834118855027, - "chrf": 47.111364119506355 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.22666325208418955, - "chrf": 42.07703156302399 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.18882590620933629, - "chrf": 37.12550972377164 - } - ], - "bleu": 0.215236057611223, - "chrf": 39.44113172245597, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "CH": 344564, - "IT": 3556954 - } - }, - { - "language_name": "Achinese", - "bcp_47": "ace", - "speakers": 3738364, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.16911632683538352, - "chrf": 35.055560798896856 - } - ], - "bleu": 0.16911632683538352, - "chrf": 35.055560798896856, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ace", - "population": { - "ID": 3738364 - } - }, - { - "language_name": "Shan", - "bcp_47": "shn", - "speakers": 3687984, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.03567194702202585, - "chrf": 15.061253284595894 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.14589621017705648, - "chrf": 32.36640913542123 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.011114664716630177, - "chrf": 9.54345606166579 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.24688742301342204, - "chrf": 43.47153430068834 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.11534595629433392, - "chrf": 32.685789872044445 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.06564720827517354, - "chrf": 21.409820368208937 - } - ], - "bleu": 0.10342723491644035, - "chrf": 25.756377170437435, - "commonvoice_hours": 0.0, - "commonvoice_locale": "shn", - "population": { - "MM": 3621766, - "TH": 66218 - } - }, - { - "language_name": "Georgian", - "bcp_47": "ka", - "speakers": 3543646, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.22489436376782782, - "chrf": 42.05355918136569 - } - ], - "bleu": 0.22489436376782782, - "chrf": 42.05355918136569, - "commonvoice_hours": 158.0, - "commonvoice_locale": "ka", - "population": { - "GE": 3437420, - "IR": 60296, - "TR": 45930 - } - }, - { - "language_name": "Galician", - "bcp_47": "gl", - "speakers": 3515530, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2463790593991139, - "chrf": 43.16984633200989 - } - ], - "bleu": 0.2463790593991139, - "chrf": 43.16984633200989, - "commonvoice_hours": 109.0, - "commonvoice_locale": "gl", - "population": { - "ES": 3501106, - "PT": 14424 - } - }, - { - "language_name": "Lingala", - "bcp_47": "ln", - "speakers": 3514491, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.10115588577551943, - "chrf": 26.041173534129616 - } - ], - "bleu": 0.10115588577551943, - "chrf": 26.041173534129616, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ln", - "population": { - "AO": 217899, - "CD": 3155180, - "CF": 14378, - "CG": 127034 - } - }, - { - "language_name": "Kabyle", - "bcp_47": "kab", - "speakers": 3351886, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.03368838568519845, - "chrf": 17.84430440143716 - } - ], - "bleu": 0.03368838568519845, - "chrf": 17.84430440143716, - "commonvoice_hours": 571.0, - "commonvoice_locale": "kab", - "population": { - "DZ": 3351886 - } - }, - { - "language_name": "Kyrgyz", - "bcp_47": "ky", - "speakers": 3338267, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.21002212869070494, - "chrf": 39.3726736717899 - } - ], - "bleu": 0.21002212869070494, - "chrf": 39.3726736717899, - "commonvoice_hours": 39.0, - "commonvoice_locale": "ky", - "population": { - "CN": 473967, - "KG": 2863152, - "TR": 1148 - } - }, - { - "language_name": "Fon", - "bcp_47": "fon", - "speakers": 3216150, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.0832678269022026, - "chrf": 21.627442109362057 - } - ], - "bleu": 0.0832678269022026, - "chrf": 21.627442109362057, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "BJ": 3216150 - } - }, - { - "language_name": "Waray", - "bcp_47": "war", - "speakers": 3166927, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2156899984074879, - "chrf": 38.73293471051519 - } - ], - "bleu": 0.2156899984074879, - "chrf": 38.73293471051519, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "CA": 678, - "PH": 3166249 - } - }, - { - "language_name": "Tibetan", - "bcp_47": "bo", - "speakers": 3006697, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.11883469874802492, - "chrf": 29.188000714511094 - } - ], - "bleu": 0.11883469874802492, - "chrf": 29.188000714511094, - "commonvoice_hours": 0.0, - "commonvoice_locale": "bo", - "population": { - "CN": 2788040, - "IN": 145870, - "NP": 72787 - } - }, - { - "language_name": "Sango", - "bcp_47": "sg", - "speakers": 2935521, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.052708457503892185, - "chrf": 21.077342933150366 - } - ], - "bleu": 0.052708457503892185, - "chrf": 21.077342933150366, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "CF": 2935521 - } - }, - { - "language_name": "Aymara", - "bcp_47": "ay", - "speakers": 2838620, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.07563412710891973, - "chrf": 23.042901816075858 - } - ], - "bleu": 0.07563412710891973, - "chrf": 23.042901816075858, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "BO": 2327980, - "PE": 510640 - } - }, - { - "language_name": "Dogri", - "bcp_47": "doi", - "speakers": 2652180, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.18698274115592, - "chrf": 36.17811444987121 - } - ], - "bleu": 0.18698274115592, - "chrf": 36.17811444987121, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IN": 2652180 - } - }, - { - "language_name": "Lithuanian", - "bcp_47": "lt", - "speakers": 2488617, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.23629191535308328, - "chrf": 41.457632804056466 - } - ], - "bleu": 0.23629191535308328, - "chrf": 41.457632804056466, - "commonvoice_hours": 25.0, - "commonvoice_locale": "lt", - "population": { - "GB": 131522, - "LT": 2349056, - "PL": 8039 - } - }, - { - "language_name": "Swati", - "bcp_47": "ss", - "speakers": 2212379, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.10571792263190831, - "chrf": 26.72182646364123 - } - ], - "bleu": 0.10571792263190831, - "chrf": 26.72182646364123, - "commonvoice_hours": 0.0, - "commonvoice_locale": "ss", - "population": { - "LS": 47264, - "SZ": 640598, - "ZA": 1524517 - } - }, - { - "language_name": "Occitan", - "bcp_47": "oc", - "speakers": 2040398, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.3116700967049491, - "chrf": 49.33859359679982 - } - ], - "bleu": 0.3116700967049491, - "chrf": 49.33859359679982, - "commonvoice_hours": 1.8, - "commonvoice_locale": "oc", - "population": { - "ES": 4952, - "FR": 2035446 - } - }, - { - "language_name": "Tatar", - "bcp_47": "tt", - "speakers": 1984108, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.20199966692246552, - "chrf": 38.969447814905855 - } - ], - "bleu": 0.20199966692246552, - "chrf": 38.969447814905855, - "commonvoice_hours": 32.0, - "commonvoice_locale": "tt", - "population": { - "RU": 1984108 - } - }, - { - "language_name": "Slovenian", - "bcp_47": "sl", - "speakers": 1973181, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.25710751649810404, - "chrf": 43.97419502631293 - } - ], - "bleu": 0.25710751649810404, - "chrf": 43.97419502631293, - "commonvoice_hours": 17.0, - "commonvoice_locale": "sl", - "population": { - "AT": 32780, - "HU": 4984, - "IT": 106085, - "SI": 1829332 - } - }, - { - "language_name": "Bodo", - "bcp_47": "brx", - "speakers": 1856526, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.07193315161893905, - "chrf": 21.198050861825372 - } - ], - "bleu": 0.07193315161893905, - "chrf": 21.198050861825372, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IN": 1856526 - } - }, - { - "language_name": "Chuvash", - "bcp_47": "cv", - "speakers": 1842386, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.16319209573807847, - "chrf": 35.5605244185656 - } - ], - "bleu": 0.16319209573807847, - "chrf": 35.5605244185656, - "commonvoice_hours": 27.0, - "commonvoice_locale": "cv", - "population": { - "RU": 1842386 - } - }, - { - "language_name": "Bashkir", - "bcp_47": "ba", - "speakers": 1842386, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.23494956875272427, - "chrf": 43.58361096793536 - } - ], - "bleu": 0.23494956875272427, - "chrf": 43.58361096793536, - "commonvoice_hours": 259.0, - "commonvoice_locale": "ba", - "population": { - "RU": 1842386 - } - }, - { - "language_name": "Tumbuka", - "bcp_47": "tum", - "speakers": 1780514, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.09211959148198216, - "chrf": 25.078974624106454 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.08953119623294435, - "chrf": 26.60406076382521 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.036353192983993324, - "chrf": 15.247949668242475 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.21852974820220555, - "chrf": 40.790711915222424 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.13308678184347988, - "chrf": 31.89595169117875 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.06252197708878435, - "chrf": 23.417269345311365 - } - ], - "bleu": 0.10535708130556494, - "chrf": 27.172486334647783, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "MW": 1780514 - } - }, - { - "language_name": "Macedonian", - "bcp_47": "mk", - "speakers": 1608565, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2635416107541368, - "chrf": 46.15461303306508 - } - ], - "bleu": 0.2635416107541368, - "chrf": 46.15461303306508, - "commonvoice_hours": 18.0, - "commonvoice_locale": "mk", - "population": { - "AL": 14451, - "GR": 169714, - "MK": 1424400 - } - }, - { - "language_name": "Pangasinan", - "bcp_47": "pag", - "speakers": 1528534, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.14637588345836686, - "chrf": 33.27328635596556 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.17061619096272593, - "chrf": 35.63408741214199 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.11313843155080379, - "chrf": 26.507505375137484 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.27369890360254523, - "chrf": 44.34141738332121 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.19116528491340065, - "chrf": 36.672507303893106 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.11054650956119119, - "chrf": 30.67803234997905 - } - ], - "bleu": 0.16759020067483896, - "chrf": 34.51780603007307, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "PH": 1528534 - } - }, - { - "language_name": "Manipuri", - "bcp_47": "mni", - "speakers": 1476591, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.061702153982680315, - "chrf": 20.627265799751633 - } - ], - "bleu": 0.061702153982680315, - "chrf": 20.627265799751633, - "commonvoice_hours": 0.0, - "commonvoice_locale": "mni", - "population": { - "BD": 17892, - "IN": 1458699 - } - }, - { - "language_name": "Venetian", - "bcp_47": "vec", - "speakers": 1380829, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.24764447442173138, - "chrf": 41.97335861313842 - } - ], - "bleu": 0.24764447442173138, - "chrf": 41.97335861313842, - "commonvoice_hours": 0.0, - "commonvoice_locale": "vec", - "population": { - "BR": 508118, - "HR": 29594, - "IT": 811235, - "MX": 2444, - "SI": 29438 - } - }, - { - "language_name": "Norwegian Nynorsk", - "bcp_47": "nn", - "speakers": 1366860, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.31661912673403325, - "chrf": 48.792392911852595 - } - ], - "bleu": 0.31661912673403325, - "chrf": 48.792392911852595, - "commonvoice_hours": 1.5, - "commonvoice_locale": "nn-NO", - "population": { - "NO": 1366860 - } - }, - { - "language_name": "Irish", - "bcp_47": "ga", - "speakers": 1237487, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2768559181644857, - "chrf": 46.47151564403362 - } - ], - "bleu": 0.2768559181644857, - "chrf": 46.47151564403362, - "commonvoice_hours": 5.8, - "commonvoice_locale": "ga-IE", - "population": { - "GB": 98642, - "IE": 1138845 - } - }, - { - "language_name": "Latvian", - "bcp_47": "lv", - "speakers": 1147550, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.23200427142275887, - "chrf": 44.00679383253316 - } - ], - "bleu": 0.23200427142275887, - "chrf": 44.00679383253316, - "commonvoice_hours": 260.0, - "commonvoice_locale": "lv", - "population": { - "LV": 1147550 - } - }, - { - "language_name": "Basque", - "bcp_47": "eu", - "speakers": 1088519, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1907277513380933, - "chrf": 40.633564870804214 - } - ], - "bleu": 0.1907277513380933, - "chrf": 40.633564870804214, - "commonvoice_hours": 335.0, - "commonvoice_locale": "eu", - "population": { - "ES": 1000316, - "FR": 88203 - } - }, - { - "language_name": "Sardinian", - "bcp_47": "sc", - "speakers": 1060846, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2003666163856343, - "chrf": 39.97859549386356 - } - ], - "bleu": 0.2003666163856343, - "chrf": 39.97859549386356, - "commonvoice_hours": 2.9, - "commonvoice_locale": "sc", - "population": { - "IT": 1060846 - } - }, - { - "language_name": "Najdi Arabic", - "bcp_47": "ars", - "speakers": 1025205, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.26325866988203733, - "chrf": 45.79452460253912 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.25411630061861235, - "chrf": 45.68081123321704 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.19634428413472024, - "chrf": 37.402576382999925 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.3267312117229826, - "chrf": 50.07524798517934 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.27947088689796734, - "chrf": 47.70370329275568 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.23043700347741075, - "chrf": 40.64509062227617 - } - ], - "bleu": 0.2583930594556218, - "chrf": 44.550325686494546, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "SA": 1025205 - } - }, - { - "language_name": "Yiddish", - "bcp_47": "yi", - "speakers": 997214, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2748989006328114, - "chrf": 44.151717001541904 - } - ], - "bleu": 0.2748989006328114, - "chrf": 44.151717001541904, - "commonvoice_hours": 0.5, - "commonvoice_locale": "yi", - "population": { - "IL": 260264, - "SE": 2959, - "UA": 570998, - "US": 162993 - } - }, - { - "language_name": "Kachin", - "bcp_47": "kac", - "speakers": 962032, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.06343642810657522, - "chrf": 21.96116119019238 - } - ], - "bleu": 0.06343642810657522, - "chrf": 21.96116119019238, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "MM": 962032 - } - }, - { - "language_name": "Limburgish", - "bcp_47": "li", - "speakers": 950422, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.28017358847160223, - "chrf": 45.82764538788154 - } - ], - "bleu": 0.28017358847160223, - "chrf": 45.82764538788154, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "NL": 950422 - } - }, - { - "language_name": "Welsh", - "bcp_47": "cy", - "speakers": 884910, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.31667961925197524, - "chrf": 48.49344578612579 - } - ], - "bleu": 0.31667961925197524, - "chrf": 48.49344578612579, - "commonvoice_hours": 124.0, - "commonvoice_locale": "cy", - "population": { - "AR": 30016, - "GB": 854894 - } - }, - { - "language_name": "Estonian", - "bcp_47": "et", - "speakers": 878449, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.23762161272302187, - "chrf": 42.90409268311042 - } - ], - "bleu": 0.23762161272302187, - "chrf": 42.90409268311042, - "commonvoice_hours": 58.0, - "commonvoice_locale": "et", - "population": { - "EE": 872320, - "FI": 6129 - } - }, - { - "language_name": "Asturian", - "bcp_47": "ast", - "speakers": 650205, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.3066166431048003, - "chrf": 47.792858053998366 - } - ], - "bleu": 0.3066166431048003, - "chrf": 47.792858053998366, - "commonvoice_hours": 0.7, - "commonvoice_locale": "ast", - "population": { - "ES": 650205 - } - }, - { - "language_name": "N’Ko", - "bcp_47": "nqo", - "speakers": 626370, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.0026829540009563496, - "chrf": 8.366238228343608 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.0028810767141941676, - "chrf": 11.711522538883516 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.002244809403558117, - "chrf": 6.87154254960649 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.1373860974763018, - "chrf": 29.63649387292498 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.005449384832055512, - "chrf": 13.63303465097306 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.001220306675003964, - "chrf": 11.614232149229839 - } - ], - "bleu": 0.02531077151701165, - "chrf": 13.638843998326914, - "commonvoice_hours": 0.0, - "commonvoice_locale": "nqo", - "population": { - "GN": 626370 - } - }, - { - "language_name": "Nuer", - "bcp_47": "nus", - "speakers": 591427, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.04422064781985695, - "chrf": 18.101658717442856 - } - ], - "bleu": 0.04422064781985695, - "chrf": 18.101658717442856, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "SS": 591427 - } - }, - { - "language_name": "Ligurian", - "bcp_47": "lij", - "speakers": 536663, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.2433180508520944, - "chrf": 42.16628456571689 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2730358021257564, - "chrf": 43.59770121161605 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.1659569541464764, - "chrf": 32.04882604989477 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.3868854055493315, - "chrf": 52.982923067584665 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.30131335750773747, - "chrf": 47.80059076162273 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.22953082347299453, - "chrf": 39.89195612874355 - } - ], - "bleu": 0.26667339894239844, - "chrf": 43.08138029752978, - "commonvoice_hours": 5.1, - "commonvoice_locale": "lij", - "population": { - "IT": 536663 - } - }, - { - "language_name": "Kabuverdianu", - "bcp_47": "kea", - "speakers": 530762, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2049568393036302, - "chrf": 39.12735936405683 - } - ], - "bleu": 0.2049568393036302, - "chrf": 39.12735936405683, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "CV": 530762 - } - }, - { - "language_name": "Mari", - "bcp_47": "chm", - "speakers": 524371, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.1477332953533076, - "chrf": 33.28969144955911 - } - ], - "bleu": 0.1477332953533076, - "chrf": 33.28969144955911, - "commonvoice_hours": 282.0, - "commonvoice_locale": "mhr", - "population": { - "RU": 524371 - } - }, - { - "language_name": "Sicilian", - "bcp_47": "scn", - "speakers": 511702, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.26991391704955275, - "chrf": 44.10988575231252 - } - ], - "bleu": 0.26991391704955275, - "chrf": 44.10988575231252, - "commonvoice_hours": 0.0, - "commonvoice_locale": "scn", - "population": { - "IT": 511702 - } - }, - { - "language_name": "Silesian", - "bcp_47": "szl", - "speakers": 497670, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2478859256009672, - "chrf": 42.936549641246145 - } - ], - "bleu": 0.2478859256009672, - "chrf": 42.936549641246145, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "PL": 497670 - } - }, - { - "language_name": "Maltese", - "bcp_47": "mt", - "speakers": 457267, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.27335683193570975, - "chrf": 45.920903610737895 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.28654849898846085, - "chrf": 46.936205424540766 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.15248109554681186, - "chrf": 30.516627088468166 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.38954095833662916, - "chrf": 54.945196672005885 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.3175303995061197, - "chrf": 51.04064318379729 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.11179045198515461, - "chrf": 30.191257026189298 - } - ], - "bleu": 0.2552080393831477, - "chrf": 43.25847216762322, - "commonvoice_hours": 8.7, - "commonvoice_locale": "mt", - "population": { - "MT": 457267 - } - }, - { - "language_name": "Luxembourgish", - "bcp_47": "lb", - "speakers": 421015, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2835765541228824, - "chrf": 44.80618475111259 - } - ], - "bleu": 0.2835765541228824, - "chrf": 44.80618475111259, - "commonvoice_hours": 0.0, - "commonvoice_locale": "lb", - "population": { - "LU": 421015 - } - }, - { - "language_name": "Dzongkha", - "bcp_47": "dz", - "speakers": 370341, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.05723633975042216, - "chrf": 23.461346449303786 - } - ], - "bleu": 0.05723633975042216, - "chrf": 23.461346449303786, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "BT": 367689, - "IN": 2652 - } - }, - { - "language_name": "Fijian", - "bcp_47": "fj", - "speakers": 365030, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.10451524271139898, - "chrf": 24.713018515709646 - } - ], - "bleu": 0.10451524271139898, - "chrf": 24.713018515709646, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "FJ": 365030 - } - }, - { - "language_name": "Icelandic", - "bcp_47": "is", - "speakers": 350734, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2142644347281729, - "chrf": 40.04812999157868 - } - ], - "bleu": 0.2142644347281729, - "chrf": 40.04812999157868, - "commonvoice_hours": 0.1, - "commonvoice_locale": "is", - "population": { - "IS": 350734 - } - }, - { - "language_name": "Samoan", - "bcp_47": "sm", - "speakers": 252717, - "scores": [ - { - "model": "openai/gpt-4o-mini", - "bleu": 0.13327372905795537, - "chrf": 28.9583532166856 - }, - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.11602460228698847, - "chrf": 29.156949243212015 - }, - { - "model": "mistralai/mistral-small-24b-instruct-2501", - "bleu": 0.05771585788755527, - "chrf": 16.752178728973906 - }, - { - "model": "google/gemini-2.0-flash-001", - "bleu": 0.2585259997356889, - "chrf": 45.18784342425295 - }, - { - "model": "deepseek/deepseek-chat", - "bleu": 0.22221137013078898, - "chrf": 39.7496231353589 - }, - { - "model": "microsoft/phi-4", - "bleu": 0.04756300118196289, - "chrf": 21.060764155029197 - } - ], - "bleu": 0.13921909338015664, - "chrf": 30.144285317252095, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "AS": 48943, - "WS": 203774 - } - }, - { - "language_name": "Crimean Tatar", - "bcp_47": "crh", - "speakers": 245968, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.22725256040555009, - "chrf": 42.07005703444819 - } - ], - "bleu": 0.22725256040555009, - "chrf": 42.07005703444819, - "commonvoice_hours": 0.0, - "commonvoice_locale": "crh", - "population": { - "UA": 245968 - } - }, - { - "language_name": "Papiamento", - "bcp_47": "pap", - "speakers": 211640, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.21648792499796674, - "chrf": 41.57061175824069 - } - ], - "bleu": 0.21648792499796674, - "chrf": 41.57061175824069, - "commonvoice_hours": 0.0, - "commonvoice_locale": "pap-AW", - "population": { - "AW": 72851, - "BQ": 16200, - "CW": 122589 - } - }, - { - "language_name": "Latgalian", - "bcp_47": "ltg", - "speakers": 167429, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.13475273241606922, - "chrf": 32.044009672423776 - } - ], - "bleu": 0.13475273241606922, - "chrf": 32.044009672423776, - "commonvoice_hours": 28.0, - "commonvoice_locale": "ltg", - "population": { - "LV": 167429 - } - }, - { - "language_name": "Māori", - "bcp_47": "mi", - "speakers": 137913, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.17610729049259877, - "chrf": 35.01961886760811 - } - ], - "bleu": 0.17610729049259877, - "chrf": 35.01961886760811, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "NZ": 137913 - } - }, - { - "language_name": "Scottish Gaelic", - "bcp_47": "gd", - "speakers": 72337, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2154279041570466, - "chrf": 37.40630914857015 - } - ], - "bleu": 0.2154279041570466, - "chrf": 37.40630914857015, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "GB": 72337 - } - }, - { - "language_name": "Faroese", - "bcp_47": "fo", - "speakers": 71351, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.21180629663838063, - "chrf": 39.28673819745006 - } - ], - "bleu": 0.21180629663838063, - "chrf": 39.28673819745006, - "commonvoice_hours": 0.0, - "commonvoice_locale": "fo", - "population": { - "DK": 22304, - "FO": 49047 - } - }, - { - "language_name": "Friulian", - "bcp_47": "fur", - "speakers": 37442, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.2255876860328074, - "chrf": 40.6191779521821 - } - ], - "bleu": 0.2255876860328074, - "chrf": 40.6191779521821, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IT": 37442 - } - }, - { - "language_name": "Sanskrit", - "bcp_47": "sa", - "speakers": 15913, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.14313294345831834, - "chrf": 32.44676491537583 - } - ], - "bleu": 0.14313294345831834, - "chrf": 32.44676491537583, - "commonvoice_hours": null, - "commonvoice_locale": null, - "population": { - "IN": 15913 - } - }, - { - "language_name": "Esperanto", - "bcp_47": "eo", - "speakers": 301, - "scores": [ - { - "model": "meta-llama/llama-3.3-70b-instruct", - "bleu": 0.27440987441620224, - "chrf": 44.40715599582661 - } - ], - "bleu": 0.27440987441620224, - "chrf": 44.40715599582661, - "commonvoice_hours": 1436.0, - "commonvoice_locale": "eo", - "population": { - "SM": 301 - } } ] \ No newline at end of file