David Pomerenke commited on
Commit
c29b8da
·
1 Parent(s): 2eeba23

Fix: don't cache model metadata forever

Browse files
Files changed (1) hide show
  1. evals/models.py +6 -13
evals/models.py CHANGED
@@ -121,12 +121,13 @@ async def transcribe(path, model="elevenlabs/scribe_v1"):
121
 
122
  models = pd.DataFrame(models, columns=["id"])
123
 
124
-
125
  @cache
 
 
 
126
  def get_or_metadata(id):
127
  # get metadata from OpenRouter
128
- response = cache(get)("https://openrouter.ai/api/frontend/models/")
129
- models = response.json()["data"]
130
  metadata = next((m for m in models if m["slug"] == id), None)
131
  return metadata
132
 
@@ -163,21 +164,13 @@ def get_hf_metadata(row):
163
  return empty
164
 
165
 
166
- or_metadata = models["id"].apply(get_or_metadata)
167
- hf_metadata = or_metadata.apply(get_hf_metadata)
168
-
169
-
170
  def get_cost(row):
171
  cost = float(row["endpoint"]["pricing"]["completion"])
172
  return round(cost * 1_000_000, 2)
173
 
174
 
175
- exists = or_metadata.apply(lambda x: x is not None)
176
- models, or_metadata, hf_metadata = (
177
- models[exists],
178
- or_metadata[exists],
179
- hf_metadata[exists],
180
- )
181
  creation_date_hf = pd.to_datetime(hf_metadata.str["creation_date"]).dt.date
182
  creation_date_or = pd.to_datetime(
183
  or_metadata.str["created_at"].str.split("T").str[0]
 
121
 
122
  models = pd.DataFrame(models, columns=["id"])
123
 
 
124
  @cache
125
+ def get_models(date):
126
+ return get("https://openrouter.ai/api/frontend/models/").json()["data"]
127
+
128
  def get_or_metadata(id):
129
  # get metadata from OpenRouter
130
+ models = get_models(date.today())
 
131
  metadata = next((m for m in models if m["slug"] == id), None)
132
  return metadata
133
 
 
164
  return empty
165
 
166
 
 
 
 
 
167
  def get_cost(row):
168
  cost = float(row["endpoint"]["pricing"]["completion"])
169
  return round(cost * 1_000_000, 2)
170
 
171
 
172
+ or_metadata = models["id"].apply(get_or_metadata)
173
+ hf_metadata = or_metadata.apply(get_hf_metadata)
 
 
 
 
174
  creation_date_hf = pd.to_datetime(hf_metadata.str["creation_date"]).dt.date
175
  creation_date_or = pd.to_datetime(
176
  or_metadata.str["created_at"].str.split("T").str[0]