group results by max eval length

app.py

@@ -11,11 +11,13 @@ def draw_leaderboard():
     df = load_results()
 
     tasks = ['avg'] + [f"qa{i}" for i in range(1, 11)]
-    columns = ["model_name", "avg(32k)", "avg(128k)"] + LENGTHS
+    columns = ["model_name", "<=32k", "<=128k"] + LENGTHS
 
     st.title("🔎📚🪡📚❓ BABILong Leaderboard 🏆")
     st.markdown(PAGE_INFO)
-    st.subheader("Average Accuracy")
+    st.subheader("Evaluation results:")
+    st.text('Each tab corresponds to a task, avg - averaged scores over qa1-5 tasks.')
+    st.markdown('All models predictions: [BABILong evals](https://huggingface.co/datasets/RMT-team/babilong_evals)')
     search_term = st.text_input("Search models:", "")
 
     tabs = st.tabs([str(task) for task in tasks])

draw_utils.py

@@ -13,7 +13,7 @@ PAGE_MARKDOWN = """
  </style>
 """
 
-PAGE_INFO = """[![Dataset on HF](https://huggingface.co/datasets/huggingface/badges/resolve/main/dataset-on-hf-lg.svg)](https://huggingface.co/datasets/booydar/babilong) | [GitHub](https://github.com/booydar/babilong) | [Paper](https://arxiv.org/abs/2406.10149) | [HF Dataset](https://huggingface.co/datasets/booydar/babilong) | [HF Dataset 1k samples per task](https://huggingface.co/datasets/RMT-team/babilong-1k-samples) |"""
+PAGE_INFO = """[![Dataset on HF](https://huggingface.co/datasets/huggingface/badges/resolve/main/dataset-on-hf-lg.svg)](https://huggingface.co/datasets/RMT-team/babilong) | [GitHub](https://github.com/booydar/babilong) | [Paper](https://arxiv.org/abs/2406.10149) | [HF Dataset](https://huggingface.co/datasets/RMT-team/babilong) | [HF Dataset 1k samples per task](https://huggingface.co/datasets/RMT-team/babilong-1k-samples) |"""
 
 LENGTHS = ['0k', '1k', '2k', '4k', '8k', '16k', '32k', '64k', '128k', '512k', '1M', '2M']
 LENGTHS_32k = ['0k', '1k', '2k', '4k', '8k', '16k', '32k']
@@ -28,9 +28,16 @@ def load_results():
 
     res = pd.concat([old_results, new_results])
     res.replace(-1, np.nan, inplace=True)
-    res['avg(32k)'] = res[LENGTHS_32k].mean(axis=1)
-    res['avg(128k)'] = res[LENGTHS_128k].mean(axis=1)
-    res.sort_values(['avg(128k)'], ascending=False, inplace=True)
+    res['<=32k'] = res[LENGTHS_32k].mean(axis=1)
+    res['<=128k'] = res[LENGTHS_128k].mean(axis=1)
+
+    # Calculate the maximum length with non-NaN values for each model
+    res['max_eval_length_idx'] = res.apply(
+        lambda row: max([LENGTHS.index(col) for col in LENGTHS if not pd.isna(row[col])], default=-1), axis=1)
+    res['max_eval_length'] = res['max_eval_length_idx'].apply(lambda x: LENGTHS[x])
+
+    # Sort first by max length (descending) and then by average score (descending)
+    res.sort_values(['max_eval_length_idx', '<=128k'], ascending=[False, False], inplace=True)
 
     return res