Spaces:

fair-forward
/

evals-for-every-language

Running

App Files Files Community

David Pomerenke commited on Oct 31, 2024

Commit

d178010

1 Parent(s): 160ce91

Improve language chart

Browse files

Files changed (4) hide show

index.html +27 -5
languagebench.py +1 -1
results.json +40 -0
results_summary.json +5 -0

index.html CHANGED Viewed

@@ -22,6 +22,8 @@
             margin: 0;
         }
     </style>
 </head>
 <body>
@@ -37,24 +39,44 @@
             const summary = await fetch('results_summary.json');
             const summaryData = await summary.json();
             // Create summary plot
             const summaryPlot = Plot.plot({
                 width: 800,
                 height: 400,
-                x: { label: "Number of speakers" },
                 y: { label: "BLEU Score (average across models)" },
                 marks: [
                     Plot.rectY(summaryData, Plot.stackX({
                         x: "speakers",
                         order: "bleu",
                         reverse: true,
                         y2: "bleu", // y2 to avoid stacking by y
-                        title: (d) => `${d.target_language_name}\n${d.bleu.toFixed(1)}`,
-                        insetLeft: 0.2,
-                        insetRight: 0.2
                     })),
-                    Plot.ruleY([0])
                 ]
             });

             margin: 0;
         }
     </style>
+    <link rel="icon"
+        href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22 fill=%22black%22>🌍</text></svg>">
 </head>
 <body>
             const summary = await fetch('results_summary.json');
             const summaryData = await summary.json();
+            const formatTitle = d => d.target_language_name + "\n" + parseInt(d.speakers / 1_000_00) / 10 + "M speakers\nBLEU score: " + d.bleu.toFixed(1)
             // Create summary plot
             const summaryPlot = Plot.plot({
                 width: 800,
                 height: 400,
+                marginBottom: 100,
+                x: { label: "Number of speakers", axis: null },
                 y: { label: "BLEU Score (average across models)" },
                 marks: [
                     Plot.rectY(summaryData, Plot.stackX({
                         x: "speakers",
                         order: "bleu",
                         reverse: true,
                         y2: "bleu", // y2 to avoid stacking by y
+                        title: formatTitle,
+                        tip: true,
                     })),
+                    Plot.rectY(summaryData, Plot.pointerX(Plot.stackX({
+                        x: "speakers",
+                        order: "bleu",
+                        reverse: true,
+                        y2: "bleu", // y2 to avoid stacking by y
+                        fill: "grey",
+                    }))),
+                    Plot.text(summaryData, Plot.stackX({
+                        filter: (d) => d.speakers > 1_000_000,
+                        x: "speakers",
+                        y2: "bleu",
+                        order: "bleu",
+                        reverse: true,
+                        text: "target_language_name",
+                        frameAnchor: "bottom",
+                        textAnchor: "end",
+                        dy: 10,
+                        rotate: 270
+                    }))
                 ]
             });

languagebench.py CHANGED Viewed

@@ -25,7 +25,7 @@ original_language = "eng_Latn"
 dataset = "floresp-v2.0-rc.3/dev"
 random.seed(42)
 target_languages = [f.split(".")[1] for f in os.listdir(dataset)]
-target_languages = random.choices(target_languages, k=8)
 # target_languages = [
 #     "eng_Latn",
 #     "deu_Latn",

 dataset = "floresp-v2.0-rc.3/dev"
 random.seed(42)
 target_languages = [f.split(".")[1] for f in os.listdir(dataset)]
+target_languages = random.choices(target_languages, k=9)
 # target_languages = [
 #     "eng_Latn",
 #     "deu_Latn",

results.json CHANGED Viewed

@@ -318,5 +318,45 @@
     "target_language_name": "Czech",
     "speakers": 10700000,
     "bleu": 60.25088578142904
   }
 ]

     "target_language_name": "Czech",
     "speakers": 10700000,
     "bleu": 60.25088578142904
+  },
+  {
+    "model": "openai/gpt-4o-mini",
+    "original_language": "eng_Latn",
+    "target_language": "sag_Latn",
+    "target_language_name": "Sango",
+    "speakers": 4600000,
+    "bleu": 2.2745290486034833
+  },
+  {
+    "model": "google/gemini-flash-1.5",
+    "original_language": "eng_Latn",
+    "target_language": "sag_Latn",
+    "target_language_name": "Sango",
+    "speakers": 4600000,
+    "bleu": 5.131617554505083
+  },
+  {
+    "model": "anthropic/claude-3.5-sonnet",
+    "original_language": "eng_Latn",
+    "target_language": "sag_Latn",
+    "target_language_name": "Sango",
+    "speakers": 4600000,
+    "bleu": 22.265544703760973
+  },
+  {
+    "model": "qwen/qwen-2.5-72b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "sag_Latn",
+    "target_language_name": "Sango",
+    "speakers": 4600000,
+    "bleu": 1.1524444505654738
+  },
+  {
+    "model": "meta-llama/llama-3.1-8b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "sag_Latn",
+    "target_language_name": "Sango",
+    "speakers": 4600000,
+    "bleu": 1.437953401517244
   }
 ]

results_summary.json CHANGED Viewed

@@ -38,5 +38,10 @@
     "target_language_name":"Polish",
     "bleu":59.3540779188,
     "speakers":40200000.0
   }
 ]

     "target_language_name":"Polish",
     "bleu":59.3540779188,
     "speakers":40200000.0
+  },
+  {
+    "target_language_name":"Sango",
+    "bleu":6.4524178318,
+    "speakers":4600000.0
   }
 ]