Init

.env.example

@@ -0,0 +1 @@
+OPENROUTER_API_KEY=

.gitignore

@@ -0,0 +1,14 @@
+floresp-*
+.cache
+.env
+
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

.python-version

@@ -0,0 +1 @@
+3.10

README.md

@@ -0,0 +1,9 @@
+# languagebench 🌍
+
+Benchmarking all big AI models on all benchmarkable languages.
+
+Sources:
+
+1. For AI models: [OpenRouter](https://openrouter.ai/)
+2. For language benchmarks: [FLORES+](https://github.com/openlanguagedata/flores)
+3. For language statistics: [Wikidata](https://gist.github.com/unhammer/3e8f2e0f79972bf5008a4c970081502d) (Potential alternative: [Ethnologue](https://www.ethnologue.com/browse/names/))

dashboard.py

@@ -0,0 +1,22 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+
+st.title("Language Bench")
+
+st.write("## Results")
+
+
+results = pd.read_json("results.json")
+
+st.dataframe(results)
+
+for language in results["target_language"].unique():
+    st.write(f"## {language}")
+    fig = px.bar(
+        results[results["target_language"] == language],
+        x="model",
+        y="bleu",
+        range_y=[0, 1],
+    )
+    st.plotly_chart(fig)

languagebench.py

@@ -0,0 +1,86 @@
+import asyncio
+import json
+import os
+from os import getenv
+
+import evaluate
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+from tqdm.asyncio import tqdm_asyncio
+from joblib.memory import Memory
+
+# config
+models = [
+    "openai/gpt-4o-mini",
+    "google/gemini-flash-1.5",
+    "anthropic/claude-3.5-sonnet",
+    "qwen/qwen-2.5-72b-instruct",
+    "meta-llama/llama-3.1-8b-instruct",
+]
+original_language = "eng_Latn"
+dataset = "floresp-v2.0-rc.3/dev"
+# target_languages = [f.split(".")[1] for f in os.listdir(dataset)]
+target_languages = [
+    "eng_Latn",
+    "deu_Latn",
+    "fra_Latn",
+    "spa_Latn",
+    "cmn_Hans",
+]
+
+# setup
+client = AsyncOpenAI(
+    base_url="https://openrouter.ai/api/v1",
+    api_key=getenv("OPENROUTER_API_KEY"),
+)
+load_dotenv()
+cache = Memory(location=".cache", verbose=0).cache
+bleu = evaluate.load("bleu")
+
+
+@cache
+async def translate(model, target_language, sentence):
+    reply = await client.chat.completions.create(
+        model=model,
+        messages=[
+            {
+                "role": "user",
+                "content": f"Translate the following text from {original_language} to {target_language}:\n\n{sentence}",
+            }
+        ],
+        temperature=0,
+    )
+    return reply.choices[0].message.content
+
+
+async def main():
+    n = 30
+    results = []
+    original_sentences = open(f"{dataset}/dev.{original_language}").readlines()
+    for target_language in target_languages:
+        target_sentences = open(f"{dataset}/dev.{target_language}").readlines()
+        for model in models:
+            print(f"{model} -> {target_language}")
+            predictions = await tqdm_asyncio.gather(
+                *[
+                    translate(model, target_language, sentence)
+                    for sentence in original_sentences[:n]
+                ],
+            )
+            metrics = bleu.compute(
+                predictions=predictions, references=target_sentences[:n]
+            )
+            results.append(
+                {
+                    "model": model,
+                    "original_language": original_language,
+                    "target_language": target_language,
+                    "bleu": metrics["bleu"],
+                }
+            )
+            with open("results.json", "w") as f:
+                json.dump(results, f, indent=2)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "languagebench"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "evaluate>=0.4.3",
+    "joblib>=1.4.2",
+    "openai>=1.52.2",
+    "pandas>=2.2.3",
+    "plotly>=5.24.1",
+    "python-dotenv>=1.0.1",
+    "streamlit>=1.39.0",
+    "tqdm>=4.66.6",
+]

results.json

@@ -0,0 +1,152 @@
+[
+  {
+    "model": "openai/gpt-4o-mini",
+    "original_language": "eng_Latn",
+    "target_language": "eng_Latn",
+    "bleu": 0.9601875101934466
+  },
+  {
+    "model": "google/gemini-flash-1.5",
+    "original_language": "eng_Latn",
+    "target_language": "eng_Latn",
+    "bleu": 0.796483772261889
+  },
+  {
+    "model": "anthropic/claude-3.5-sonnet",
+    "original_language": "eng_Latn",
+    "target_language": "eng_Latn",
+    "bleu": 0.4789694173473208
+  },
+  {
+    "model": "qwen/qwen-2.5-72b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "eng_Latn",
+    "bleu": 0.5708253125905761
+  },
+  {
+    "model": "meta-llama/llama-3.1-8b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "eng_Latn",
+    "bleu": 0.7139866196167579
+  },
+  {
+    "model": "openai/gpt-4o-mini",
+    "original_language": "eng_Latn",
+    "target_language": "deu_Latn",
+    "bleu": 0.42769123869791453
+  },
+  {
+    "model": "google/gemini-flash-1.5",
+    "original_language": "eng_Latn",
+    "target_language": "deu_Latn",
+    "bleu": 0.481667025275085
+  },
+  {
+    "model": "anthropic/claude-3.5-sonnet",
+    "original_language": "eng_Latn",
+    "target_language": "deu_Latn",
+    "bleu": 0.47566381880734276
+  },
+  {
+    "model": "qwen/qwen-2.5-72b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "deu_Latn",
+    "bleu": 0.3886704151083369
+  },
+  {
+    "model": "meta-llama/llama-3.1-8b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "deu_Latn",
+    "bleu": 0.3229429355718441
+  },
+  {
+    "model": "openai/gpt-4o-mini",
+    "original_language": "eng_Latn",
+    "target_language": "fra_Latn",
+    "bleu": 0.4770220301445618
+  },
+  {
+    "model": "google/gemini-flash-1.5",
+    "original_language": "eng_Latn",
+    "target_language": "fra_Latn",
+    "bleu": 0.4950529382461408
+  },
+  {
+    "model": "anthropic/claude-3.5-sonnet",
+    "original_language": "eng_Latn",
+    "target_language": "fra_Latn",
+    "bleu": 0.505571990673057
+  },
+  {
+    "model": "qwen/qwen-2.5-72b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "fra_Latn",
+    "bleu": 0.4343766704709354
+  },
+  {
+    "model": "meta-llama/llama-3.1-8b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "fra_Latn",
+    "bleu": 0.3738013101452592
+  },
+  {
+    "model": "openai/gpt-4o-mini",
+    "original_language": "eng_Latn",
+    "target_language": "spa_Latn",
+    "bleu": 0.34656060748435535
+  },
+  {
+    "model": "google/gemini-flash-1.5",
+    "original_language": "eng_Latn",
+    "target_language": "spa_Latn",
+    "bleu": 0.3449205632717461
+  },
+  {
+    "model": "anthropic/claude-3.5-sonnet",
+    "original_language": "eng_Latn",
+    "target_language": "spa_Latn",
+    "bleu": 0.34586378905270954
+  },
+  {
+    "model": "qwen/qwen-2.5-72b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "spa_Latn",
+    "bleu": 0.3341419407814188
+  },
+  {
+    "model": "meta-llama/llama-3.1-8b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "spa_Latn",
+    "bleu": 0.29470460185415065
+  },
+  {
+    "model": "openai/gpt-4o-mini",
+    "original_language": "eng_Latn",
+    "target_language": "cmn_Hans",
+    "bleu": 0.0
+  },
+  {
+    "model": "google/gemini-flash-1.5",
+    "original_language": "eng_Latn",
+    "target_language": "cmn_Hans",
+    "bleu": 0.0
+  },
+  {
+    "model": "anthropic/claude-3.5-sonnet",
+    "original_language": "eng_Latn",
+    "target_language": "cmn_Hans",
+    "bleu": 0.0
+  },
+  {
+    "model": "qwen/qwen-2.5-72b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "cmn_Hans",
+    "bleu": 0.0
+  },
+  {
+    "model": "meta-llama/llama-3.1-8b-instruct",
+    "original_language": "eng_Latn",
+    "target_language": "cmn_Hans",
+    "bleu": 0.0
+  }
+]