David Pomerenke
commited on
Commit
·
c29b8da
1
Parent(s):
2eeba23
Fix: don't cache model metadata forever
Browse files- evals/models.py +6 -13
evals/models.py
CHANGED
@@ -121,12 +121,13 @@ async def transcribe(path, model="elevenlabs/scribe_v1"):
|
|
121 |
|
122 |
models = pd.DataFrame(models, columns=["id"])
|
123 |
|
124 |
-
|
125 |
@cache
|
|
|
|
|
|
|
126 |
def get_or_metadata(id):
|
127 |
# get metadata from OpenRouter
|
128 |
-
|
129 |
-
models = response.json()["data"]
|
130 |
metadata = next((m for m in models if m["slug"] == id), None)
|
131 |
return metadata
|
132 |
|
@@ -163,21 +164,13 @@ def get_hf_metadata(row):
|
|
163 |
return empty
|
164 |
|
165 |
|
166 |
-
or_metadata = models["id"].apply(get_or_metadata)
|
167 |
-
hf_metadata = or_metadata.apply(get_hf_metadata)
|
168 |
-
|
169 |
-
|
170 |
def get_cost(row):
|
171 |
cost = float(row["endpoint"]["pricing"]["completion"])
|
172 |
return round(cost * 1_000_000, 2)
|
173 |
|
174 |
|
175 |
-
|
176 |
-
|
177 |
-
models[exists],
|
178 |
-
or_metadata[exists],
|
179 |
-
hf_metadata[exists],
|
180 |
-
)
|
181 |
creation_date_hf = pd.to_datetime(hf_metadata.str["creation_date"]).dt.date
|
182 |
creation_date_or = pd.to_datetime(
|
183 |
or_metadata.str["created_at"].str.split("T").str[0]
|
|
|
121 |
|
122 |
models = pd.DataFrame(models, columns=["id"])
|
123 |
|
|
|
124 |
@cache
|
125 |
+
def get_models(date):
|
126 |
+
return get("https://openrouter.ai/api/frontend/models/").json()["data"]
|
127 |
+
|
128 |
def get_or_metadata(id):
|
129 |
# get metadata from OpenRouter
|
130 |
+
models = get_models(date.today())
|
|
|
131 |
metadata = next((m for m in models if m["slug"] == id), None)
|
132 |
return metadata
|
133 |
|
|
|
164 |
return empty
|
165 |
|
166 |
|
|
|
|
|
|
|
|
|
167 |
def get_cost(row):
|
168 |
cost = float(row["endpoint"]["pricing"]["completion"])
|
169 |
return round(cost * 1_000_000, 2)
|
170 |
|
171 |
|
172 |
+
or_metadata = models["id"].apply(get_or_metadata)
|
173 |
+
hf_metadata = or_metadata.apply(get_hf_metadata)
|
|
|
|
|
|
|
|
|
174 |
creation_date_hf = pd.to_datetime(hf_metadata.str["creation_date"]).dt.date
|
175 |
creation_date_or = pd.to_datetime(
|
176 |
or_metadata.str["created_at"].str.split("T").str[0]
|