More evals

app.py

@@ -286,7 +286,7 @@ def create_language_stats_df(metric):
 
     df = pd.DataFrame(flat_data)
     return gr.DataFrame(
-        value=df,
+        value=df, 
         label="Language Results",
         show_search="search",
         datatype=[
@@ -297,6 +297,7 @@ def create_language_stats_df(metric):
             "number",  # Translation
             "number",  # Classification
             "number",  # MLM
+            "number",  # ASR
             "markdown",  # Best Model
             "markdown",  # CommonVoice Hours
         ],

evals.py

@@ -42,11 +42,11 @@ models = [
 model_fast = "meta-llama/llama-3.3-70b-instruct"
 transcription_models = [
     "elevenlabs/scribe_v1",
-    "openai/whisper-large-v3-turbo",
+    "openai/whisper-large-v3",
     # "openai/whisper-small",
     # "facebook/seamless-m4t-v2-large",
 ]
-transcription_model_fast = "openai/whisper-large-v3-turbo"
+transcription_model_fast = "elevenlabs/scribe_v1"
 n_sentences = 30
 
 # ===== setup =====
@@ -203,14 +203,14 @@ languages = pd.merge(
 )  # "left" because keep it simple for now
 languages["in_benchmark"] = languages["bcp_47"].isin(benchmark_languages["bcp_47"])
 
-languages = languages.sort_values(by="speakers", ascending=False).iloc[:5]
+languages = languages.sort_values(by="speakers", ascending=False).iloc[:10]
 
 # sample languages to translate to
 target_languages = languages[languages["in_benchmark"]].sample(
     n=n_sentences, weights="speakers", replace=True, random_state=42
 )
 # sample languages to analyze with all models
-detailed_languages = languages[languages["in_benchmark"]].iloc[:2]
+detailed_languages = languages[languages["in_benchmark"]].iloc[:5]
 
 
 # ===== define tasks and metrics =====

results.json

@@ -56,10 +56,10 @@
         "overall_score": 0.34916319968417603
       },
       {
-        "model": "openai/whisper-large-v3-turbo",
+        "model": "openai/whisper-large-v3",
         "model_type": "speech-to-text",
-        "asr_wer": 0.2670470041301468,
-        "overall_score": 0.2670470041301468
+        "asr_wer": 0.25418986127300397,
+        "overall_score": 0.25418986127300397
       }
     ],
     "commonvoice_hours": 2651.0,
@@ -226,8 +226,8 @@
     "mt_chrf": 65.91917919929946,
     "cls_acc": 0.6533333333333333,
     "mlm_chrf": 93.48244773503015,
-    "asr_wer": 0.30810510190716145,
-    "overall_score": 0.6231132678488628
+    "asr_wer": 0.30167653047859,
+    "overall_score": 0.6212765331549852
   },
   {
     "language_name": "Chinese",
@@ -286,7 +286,7 @@
         "overall_score": 1.0
       },
       {
-        "model": "openai/whisper-large-v3-turbo",
+        "model": "openai/whisper-large-v3",
         "model_type": "speech-to-text",
         "asr_wer": 1.0,
         "overall_score": 1.0
@@ -329,6 +329,15 @@
     "bcp_47": "hi",
     "speakers": 546882144,
     "scores": [
+      {
+        "model": "openai/gpt-4o-mini",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.3647010036099328,
+        "mt_chrf": 55.294100726869324,
+        "cls_acc": 0.5,
+        "mlm_chrf": 93.01665261992896,
+        "overall_score": 0.6610358444893277
+      },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
         "model_type": "text-to-text",
@@ -339,10 +348,43 @@
         "overall_score": 0.6657108425749162
       },
       {
-        "model": "openai/whisper-large-v3-turbo",
+        "model": "mistralai/mistral-small-24b-instruct-2501",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.346304931512649,
+        "mt_chrf": 52.59738987149503,
+        "cls_acc": 0.5,
+        "mlm_chrf": 90.59820426203353,
+        "overall_score": 0.6439853137784286
+      },
+      {
+        "model": "google/gemini-2.0-flash-001",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.43820082761650603,
+        "mt_chrf": 61.68241787594198,
+        "cls_acc": 0.8666666666666667,
+        "mlm_chrf": 96.5534261573122,
+        "overall_score": 0.8163417023330695
+      },
+      {
+        "model": "microsoft/phi-4",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.388038924689894,
+        "mt_chrf": 56.13147190849697,
+        "cls_acc": 0.8,
+        "mlm_chrf": 94.47772196307584,
+        "overall_score": 0.7686973129052427
+      },
+      {
+        "model": "elevenlabs/scribe_v1",
         "model_type": "speech-to-text",
-        "asr_wer": 0.40237814140706235,
-        "overall_score": 0.40237814140706235
+        "asr_wer": 0.2338948365728121,
+        "overall_score": 0.2338948365728121
+      },
+      {
+        "model": "openai/whisper-large-v3",
+        "model_type": "speech-to-text",
+        "asr_wer": 0.43522263872986894,
+        "overall_score": 0.43522263872986894
       }
     ],
     "commonvoice_hours": 16.0,
@@ -356,18 +398,27 @@
       "ZA": 1129272
     },
     "language_family": "Indo-European",
-    "mt_bleu": 0.32404902340686065,
-    "mt_chrf": 53.54085104449268,
-    "cls_acc": 0.5,
-    "mlm_chrf": 96.17240172798218,
-    "asr_wer": 0.40237814140706235,
-    "overall_score": 0.5340444919909892
+    "mt_bleu": 0.3722589421671684,
+    "mt_chrf": 55.849246285459195,
+    "cls_acc": 0.6333333333333334,
+    "mlm_chrf": 94.16368134606655,
+    "asr_wer": 0.33455873765134053,
+    "overall_score": 0.6035554987690951
   },
   {
     "language_name": "Spanish",
     "bcp_47": "es",
     "speakers": 493528077,
     "scores": [
+      {
+        "model": "openai/gpt-4o-mini",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.3313892289629975,
+        "mt_chrf": 52.582336167000754,
+        "cls_acc": 0.6,
+        "mlm_chrf": 96.34068599793908,
+        "overall_score": 0.6964100738831327
+      },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
         "model_type": "text-to-text",
@@ -378,10 +429,43 @@
         "overall_score": 0.6635684659512185
       },
       {
-        "model": "openai/whisper-large-v3-turbo",
+        "model": "mistralai/mistral-small-24b-instruct-2501",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.3120516523386294,
+        "mt_chrf": 50.14036662376262,
+        "cls_acc": 0.6333333333333333,
+        "mlm_chrf": 93.9463761308063,
+        "overall_score": 0.6914002536263407
+      },
+      {
+        "model": "google/gemini-2.0-flash-001",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.3318101035886167,
+        "mt_chrf": 53.80973313056067,
+        "cls_acc": 0.8666666666666667,
+        "mlm_chrf": 96.46210398041012,
+        "overall_score": 0.7897950125921249
+      },
+      {
+        "model": "microsoft/phi-4",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.2808051746473726,
+        "mt_chrf": 49.858821063770044,
+        "cls_acc": 0.7666666666666667,
+        "mlm_chrf": 96.28984542933327,
+        "overall_score": 0.7427177771992333
+      },
+      {
+        "model": "elevenlabs/scribe_v1",
         "model_type": "speech-to-text",
-        "asr_wer": 0.16641846898275137,
-        "overall_score": 0.16641846898275137
+        "asr_wer": 0.19653905528613333,
+        "overall_score": 0.19653905528613333
+      },
+      {
+        "model": "openai/whisper-large-v3",
+        "model_type": "speech-to-text",
+        "asr_wer": 0.17561491933862197,
+        "overall_score": 0.17561491933862197
       }
     ],
     "commonvoice_hours": 446.0,
@@ -428,18 +512,27 @@
       "VE": 23488572
     },
     "language_family": "Indo-European",
-    "mt_bleu": 0.31587937116142056,
-    "mt_chrf": 52.142851262301726,
-    "cls_acc": 0.5,
-    "mlm_chrf": 96.92768852306384,
-    "asr_wer": 0.16641846898275137,
-    "overall_score": 0.41499346746698496
+    "mt_bleu": 0.31438710613980736,
+    "mt_chrf": 51.70682164947916,
+    "cls_acc": 0.6733333333333333,
+    "mlm_chrf": 95.99334001231053,
+    "asr_wer": 0.18607698731237765,
+    "overall_score": 0.5651493654109723
   },
   {
     "language_name": "Arabic",
     "bcp_47": "ar",
     "speakers": 351664197,
     "scores": [
+      {
+        "model": "openai/gpt-4o-mini",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.312307215788044,
+        "mt_chrf": 51.50028782321851,
+        "cls_acc": 0.6,
+        "mlm_chrf": 96.57677413527875,
+        "overall_score": 0.6935902065283241
+      },
       {
         "model": "meta-llama/llama-3.3-70b-instruct",
         "model_type": "text-to-text",
@@ -450,10 +543,43 @@
         "overall_score": 0.6638448614180232
       },
       {
-        "model": "openai/whisper-large-v3-turbo",
+        "model": "mistralai/mistral-small-24b-instruct-2501",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.26940670420361396,
+        "mt_chrf": 46.07654794208784,
+        "cls_acc": 0.7,
+        "mlm_chrf": 90.92876728332028,
+        "overall_score": 0.690017717418027
+      },
+      {
+        "model": "google/gemini-2.0-flash-001",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.3928034519711188,
+        "mt_chrf": 57.47507639595937,
+        "cls_acc": 0.9,
+        "mlm_chrf": 96.82551362297947,
+        "overall_score": 0.8143353000631294
+      },
+      {
+        "model": "microsoft/phi-4",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.2700754695807907,
+        "mt_chrf": 45.896533062231306,
+        "cls_acc": 0.7,
+        "mlm_chrf": 93.60903960225609,
+        "overall_score": 0.6983519088816247
+      },
+      {
+        "model": "elevenlabs/scribe_v1",
         "model_type": "speech-to-text",
-        "asr_wer": 0.17249214841393012,
-        "overall_score": 0.17249214841393012
+        "asr_wer": 0.2685436379713873,
+        "overall_score": 0.2685436379713873
+      },
+      {
+        "model": "openai/whisper-large-v3",
+        "model_type": "speech-to-text",
+        "asr_wer": 0.17370718156523782,
+        "overall_score": 0.17370718156523782
       }
     ],
     "commonvoice_hours": 91.0,
@@ -499,11 +625,270 @@
       "YE": 22114456
     },
     "language_family": "Afro-Asiatic",
-    "mt_bleu": 0.39547934933771334,
-    "mt_chrf": 57.51652731936118,
-    "cls_acc": 0.4666666666666667,
-    "mlm_chrf": 94.97026443937914,
-    "asr_wer": 0.17249214841393012,
-    "overall_score": 0.4181685049159767
+    "mt_bleu": 0.32801443817625614,
+    "mt_chrf": 51.692994508571644,
+    "cls_acc": 0.6733333333333332,
+    "mlm_chrf": 94.58207181664275,
+    "asr_wer": 0.22112540976831258,
+    "overall_score": 0.5717701162636791
+  },
+  {
+    "language_name": "Urdu",
+    "bcp_47": "ur",
+    "speakers": 290790290,
+    "scores": [
+      {
+        "model": "meta-llama/llama-3.3-70b-instruct",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.26585004461425726,
+        "mt_chrf": 47.37157150967947,
+        "cls_acc": 0.43333333333333335,
+        "mlm_chrf": 94.38802161979918,
+        "overall_score": 0.6169764215427066
+      },
+      {
+        "model": "elevenlabs/scribe_v1",
+        "model_type": "speech-to-text",
+        "asr_wer": 0.2982973325975355,
+        "overall_score": 0.2982973325975355
+      }
+    ],
+    "commonvoice_hours": 77.0,
+    "commonvoice_locale": "ur",
+    "population": {
+      "CA": 286475,
+      "GB": 2301638,
+      "IN": 66304500,
+      "MU": 71727,
+      "PK": 221825950
+    },
+    "language_family": "Indo-European",
+    "mt_bleu": 0.26585004461425726,
+    "mt_chrf": 47.37157150967947,
+    "cls_acc": 0.43333333333333335,
+    "mlm_chrf": 94.38802161979918,
+    "asr_wer": 0.2982973325975355,
+    "overall_score": 0.457636877070121
+  },
+  {
+    "language_name": "French",
+    "bcp_47": "fr",
+    "speakers": 278611507,
+    "scores": [
+      {
+        "model": "meta-llama/llama-3.3-70b-instruct",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.3510210872150948,
+        "mt_chrf": 55.795595938804894,
+        "cls_acc": 0.5666666666666667,
+        "mlm_chrf": 97.12318847922649,
+        "overall_score": 0.6986181702823268
+      },
+      {
+        "model": "elevenlabs/scribe_v1",
+        "model_type": "speech-to-text",
+        "asr_wer": 0.2610754929736961,
+        "overall_score": 0.2610754929736961
+      }
+    ],
+    "commonvoice_hours": 1052.0,
+    "commonvoice_locale": "fr",
+    "population": {
+      "AD": 5775,
+      "AT": 974540,
+      "BE": 4453866,
+      "BF": 4583788,
+      "BI": 7000822,
+      "BJ": 4502610,
+      "BL": 6837,
+      "CA": 11308230,
+      "CD": 3867640,
+      "CF": 2935521,
+      "CG": 4446179,
+      "CH": 1764838,
+      "CI": 13465739,
+      "CM": 18866600,
+      "CY": 88668,
+      "DE": 14428746,
+      "DJ": 19358,
+      "DZ": 8594580,
+      "FR": 67169718,
+      "GA": 1405473,
+      "GB": 15125053,
+      "GF": 153622,
+      "GN": 3632946,
+      "GP": 407498,
+      "GQ": 73584,
+      "GR": 954639,
+      "HT": 520187,
+      "HU": 293155,
+      "IE": 880017,
+      "IT": 3931370,
+      "KM": 473917,
+      "LB": 20238,
+      "LU": 546691,
+      "MA": 7112340,
+      "MC": 38610,
+      "MF": 32556,
+      "MG": 18599433,
+      "ML": 8994564,
+      "MQ": 427408,
+      "MR": 680932,
+      "MT": 50299,
+      "MU": 41381,
+      "NC": 278409,
+      "NE": 6603996,
+      "NL": 5011316,
+      "PF": 180024,
+      "PM": 5133,
+      "PT": 1545405,
+      "RE": 700950,
+      "RO": 3621493,
+      "RW": 2288,
+      "SC": 57589,
+      "SN": 6137196,
+      "SY": 1144506,
+      "TD": 4388124,
+      "TF": 140,
+      "TG": 5251148,
+      "TN": 8673688,
+      "US": 1862778,
+      "VU": 149166,
+      "WF": 7610,
+      "YT": 110580
+    },
+    "language_family": "Indo-European",
+    "mt_bleu": 0.3510210872150948,
+    "mt_chrf": 55.795595938804894,
+    "cls_acc": 0.5666666666666667,
+    "mlm_chrf": 97.12318847922649,
+    "asr_wer": 0.2610754929736961,
+    "overall_score": 0.47984683162801145
+  },
+  {
+    "language_name": "Bangla",
+    "bcp_47": "bn",
+    "speakers": 267193288,
+    "scores": [
+      {
+        "model": "meta-llama/llama-3.3-70b-instruct",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.2874920154082786,
+        "mt_chrf": 51.49279116112809,
+        "cls_acc": 0.4,
+        "mlm_chrf": 90.6067262108039,
+        "overall_score": 0.6069983912397733
+      },
+      {
+        "model": "elevenlabs/scribe_v1",
+        "model_type": "speech-to-text",
+        "asr_wer": 0.26686188207927336,
+        "overall_score": 0.26686188207927336
+      }
+    ],
+    "commonvoice_hours": 49.0,
+    "commonvoice_locale": "bn",
+    "population": {
+      "BD": 159397980,
+      "CA": 90466,
+      "GB": 263044,
+      "IN": 107413290,
+      "NP": 28508
+    },
+    "language_family": "Indo-European",
+    "mt_bleu": 0.2874920154082786,
+    "mt_chrf": 51.49279116112809,
+    "cls_acc": 0.4,
+    "mlm_chrf": 90.6067262108039,
+    "asr_wer": 0.26686188207927336,
+    "overall_score": 0.4369301366595233
+  },
+  {
+    "language_name": "Portuguese",
+    "bcp_47": "pt",
+    "speakers": 237496885,
+    "scores": [
+      {
+        "model": "meta-llama/llama-3.3-70b-instruct",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.33491649454450034,
+        "mt_chrf": 54.60211868234021,
+        "cls_acc": 0.5666666666666667,
+        "mlm_chrf": 96.52676764996336,
+        "overall_score": 0.6926518433299008
+      },
+      {
+        "model": "elevenlabs/scribe_v1",
+        "model_type": "speech-to-text",
+        "asr_wer": 0.22967756370402836,
+        "overall_score": 0.22967756370402836
+      }
+    ],
+    "commonvoice_hours": 177.0,
+    "commonvoice_locale": "pt",
+    "population": {
+      "AG": 1571,
+      "AO": 21789941,
+      "BR": 192661560,
+      "CA": 229934,
+      "CH": 285736,
+      "CV": 443274,
+      "FR": 882027,
+      "GB": 131522,
+      "GQ": 1,
+      "GW": 1927100,
+      "LU": 100541,
+      "MO": 30723,
+      "MZ": 8126514,
+      "PT": 9890592,
+      "ST": 179454,
+      "TL": 816395
+    },
+    "language_family": "Indo-European",
+    "mt_bleu": 0.33491649454450034,
+    "mt_chrf": 54.60211868234021,
+    "cls_acc": 0.5666666666666667,
+    "mlm_chrf": 96.52676764996336,
+    "asr_wer": 0.22967756370402836,
+    "overall_score": 0.4611647035169646
+  },
+  {
+    "language_name": "Punjabi",
+    "bcp_47": "pa",
+    "speakers": 203571210,
+    "scores": [
+      {
+        "model": "meta-llama/llama-3.3-70b-instruct",
+        "model_type": "text-to-text",
+        "mt_bleu": 0.3078917767345886,
+        "mt_chrf": 50.505686987696365,
+        "cls_acc": 0.5333333333333333,
+        "mlm_chrf": 90.10119297923285,
+        "overall_score": 0.6464673776675418
+      },
+      {
+        "model": "elevenlabs/scribe_v1",
+        "model_type": "speech-to-text",
+        "asr_wer": 0.20953788908863977,
+        "overall_score": 0.20953788908863977
+      }
+    ],
+    "commonvoice_hours": 2.3,
+    "commonvoice_locale": "pa-IN",
+    "population": {
+      "CA": 603106,
+      "GB": 2367400,
+      "IN": 37130520,
+      "KE": 10170,
+      "PK": 163450700,
+      "SG": 9314
+    },
+    "language_family": "Indo-European",
+    "mt_bleu": 0.3078917767345886,
+    "mt_chrf": 50.505686987696365,
+    "cls_acc": 0.5333333333333333,
+    "mlm_chrf": 90.10119297923285,
+    "asr_wer": 0.20953788908863977,
+    "overall_score": 0.4280026333780908
   }
 ]

uv.lock

@@ -1245,7 +1245,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "gradio", specifier = ">=5.16.2" },
-    { name = "language-data" },
+    { name = "language-data", specifier = ">=1.3.0" },
     { name = "pandas", specifier = ">=2.2.3" },
     { name = "plotly", specifier = ">=6.0.0" },
     { name = "pycountry", specifier = ">=24.6.1" },