import gradio as gr
import pandas as pd
import numpy as np
import os
import re
from datetime import datetime
import json
import torch
from tqdm import tqdm
from concurrent.futures import ProcessPoolExecutor, as_completed
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from huggingface_hub import HfApi
import shutil
import tempfile
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from queue import Queue
import threading
from threading import Lock
from stark_qa import load_qa
from stark_qa.evaluator import Evaluator
from utils.hub_storage import HubStorage
from utils.token_handler import TokenHandler
# Initialize storage once at startup
try:
REPO_ID = "snap-stanford/stark-leaderboard" # Replace with your space name
hub_storage = HubStorage(REPO_ID)
except Exception as e:
raise RuntimeError(f"Failed to initialize HuggingFace Hub storage: {e}")
# Global lock for thread-safe operations
result_lock = Lock()
def process_single_instance(args):
idx, eval_dict, qa_dataset, evaluator, eval_metrics = args
query, query_id, answer_ids, meta_info = qa_dataset[idx]
try:
# Access prediction using dictionary instead of DataFrame
if query_id not in eval_dict:
raise IndexError(f'Error when processing query_id={query_id}, please make sure the predicted results exist for this query.')
pred_rank = eval_dict[query_id]
if isinstance(pred_rank, str):
try:
pred_rank = eval(pred_rank)
except SyntaxError as e:
raise ValueError(f'Failed to parse pred_rank as a list for query_id={query_id}: {e}')
if not isinstance(pred_rank, list):
raise TypeError(f'Error when processing query_id={query_id}, expected pred_rank to be a list but got {type(pred_rank)}.')
pred_dict = {pred_rank[i]: -i for i in range(min(100, len(pred_rank)))}
answer_ids = torch.LongTensor(answer_ids)
# Evaluate metrics
result = evaluator.evaluate(pred_dict, answer_ids, metrics=eval_metrics)
result["idx"], result["query_id"] = idx, query_id
return result
except Exception as e:
raise RuntimeError(f'Error processing query_id={query_id}: {str(e)}')
def compute_metrics(csv_path: str, dataset: str, split: str, num_threads: int = 4):
candidate_ids_dict = {
'amazon': [i for i in range(957192)],
'mag': [i for i in range(1172724, 1872968)],
'prime': [i for i in range(129375)]
}
try:
# Read and validate CSV
eval_csv = pd.read_csv(csv_path)
if 'query_id' not in eval_csv.columns:
raise ValueError('No `query_id` column found in the submitted csv.')
if 'pred_rank' not in eval_csv.columns:
raise ValueError('No `pred_rank` column found in the submitted csv.')
# Convert DataFrame to dictionary for thread-safe access
eval_dict = dict(zip(eval_csv['query_id'], eval_csv['pred_rank']))
# Validate input parameters
if dataset not in candidate_ids_dict:
raise ValueError(f"Invalid dataset '{dataset}', expected one of {list(candidate_ids_dict.keys())}.")
if split not in ['test', 'test-0.1', 'human_generated_eval']:
raise ValueError(f"Invalid split '{split}', expected one of ['test', 'test-0.1', 'human_generated_eval'].")
# Initialize evaluator and metrics
evaluator = Evaluator(candidate_ids_dict[dataset])
eval_metrics = ['hit@1', 'hit@5', 'recall@20', 'mrr']
# Load dataset and get split indices
qa_dataset = load_qa(dataset, human_generated_eval=split == 'human_generated_eval')
split_idx = qa_dataset.get_idx_split()
all_indices = split_idx[split].tolist()
# Thread-safe containers for results
results_list = []
results_lock = Lock()
# Prepare args for each thread
args = [(idx, eval_dict, qa_dataset, evaluator, eval_metrics) for idx in all_indices]
failed_queries = [] # Track failed queries
# Process using threads
with ThreadPoolExecutor(max_workers=num_threads) as executor:
futures = [executor.submit(process_single_instance, arg) for arg in args]
for future in tqdm(as_completed(futures), total=len(futures)):
try:
result = future.result()
with results_lock:
results_list.append(result)
except Exception as e:
query_id = str(e).split('query_id=')[-1].split(':')[0]
failed_queries.append(query_id)
print(f"Error processing instance: {str(e)}")
if failed_queries:
print(f"\nFailed to process {len(failed_queries)} queries.")
print(f"First few failed query_ids: {failed_queries[:5]}")
if not results_list:
raise ValueError("No results were successfully processed")
# Compute final metrics
results_df = pd.DataFrame(results_list)
final_results = {
metric: np.mean(results_df[metric]) for metric in eval_metrics
}
return final_results
except pd.errors.EmptyDataError:
return "Error: The CSV file is empty or could not be read. Please check the file and try again."
except FileNotFoundError:
return f"Error: The file {csv_path} could not be found. Please check the file path and try again."
except Exception as error:
return f"{error}"
# Data dictionaries for leaderboard
data_synthesized_full = {
'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2'],
'STARK-AMAZON_Hit@1': [44.94, 15.29, 30.96, 26.56, 39.16, 40.93, 21.74, 42.08, 40.07, 46.10],
'STARK-AMAZON_Hit@5': [67.42, 47.93, 51.06, 50.01, 62.73, 64.37, 41.65, 66.87, 64.98, 66.02],
'STARK-AMAZON_R@20': [53.77, 44.49, 41.95, 52.05, 53.29, 54.28, 33.22, 56.52, 55.12, 53.44],
'STARK-AMAZON_MRR': [55.30, 30.20, 40.66, 37.75, 50.35, 51.60, 31.47, 53.46, 51.55, 55.51],
'STARK-MAG_Hit@1': [25.85, 10.51, 21.96, 12.88, 29.08, 30.06, 18.01, 37.90, 25.92, 31.18],
'STARK-MAG_Hit@5': [45.25, 35.23, 36.50, 39.01, 49.61, 50.58, 34.85, 56.74, 50.43, 46.42],
'STARK-MAG_R@20': [45.69, 42.11, 35.32, 46.97, 48.36, 50.49, 35.46, 46.40, 50.80, 43.94],
'STARK-MAG_MRR': [34.91, 21.34, 29.14, 29.12, 38.62, 39.66, 26.10, 47.25, 36.94, 38.39],
'STARK-PRIME_Hit@1': [12.75, 4.46, 6.53, 8.85, 12.63, 10.85, 10.10, 15.57, 15.10, 11.75],
'STARK-PRIME_Hit@5': [27.92, 21.85, 15.67, 21.35, 31.49, 30.23, 22.49, 33.42, 33.56, 23.85],
'STARK-PRIME_R@20': [31.25, 30.13, 16.52, 29.63, 36.00, 37.83, 26.34, 39.09, 38.05, 25.04],
'STARK-PRIME_MRR': [19.84, 12.38, 11.05, 14.73, 21.41, 19.99, 16.12, 24.11, 23.49, 17.39]
}
data_synthesized_10 = {
'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2', 'Claude3 Reranker', 'GPT4 Reranker'],
'STARK-AMAZON_Hit@1': [42.68, 16.46, 30.09, 25.00, 39.02, 43.29, 18.90, 43.29, 40.85, 44.31, 45.49, 44.79],
'STARK-AMAZON_Hit@5': [67.07, 50.00, 49.27, 48.17, 64.02, 67.68, 37.80, 71.34, 62.80, 65.24, 71.13, 71.17],
'STARK-AMAZON_R@20': [54.48, 42.15, 41.91, 51.65, 49.30, 56.04, 34.73, 56.14, 52.47, 51.00, 53.77, 55.35],
'STARK-AMAZON_MRR': [54.02, 30.20, 39.30, 36.87, 50.32, 54.20, 28.76, 55.07, 51.54, 55.07, 55.91, 55.69],
'STARK-MAG_Hit@1': [27.81, 11.65, 22.89, 12.03, 28.20, 34.59, 19.17, 38.35, 25.56, 31.58, 36.54, 40.90],
'STARK-MAG_Hit@5': [45.48, 36.84, 37.26, 37.97, 52.63, 50.75, 33.46, 58.64, 50.37, 47.36, 53.17, 58.18],
'STARK-MAG_R@20': [44.59, 42.30, 44.16, 47.98, 49.25, 50.75, 29.85, 46.38, 53.03, 45.72, 48.36, 48.60],
'STARK-MAG_MRR': [35.97, 21.82, 30.00, 28.70, 38.55, 42.90, 26.06, 48.25, 36.82, 38.98, 44.15, 49.00],
'STARK-PRIME_Hit@1': [13.93, 5.00, 6.78, 7.14, 15.36, 12.14, 9.29, 16.79, 15.36, 15.00, 17.79, 18.28],
'STARK-PRIME_Hit@5': [31.07, 23.57, 16.15, 17.14, 31.07, 31.42, 20.7, 34.29, 32.86, 26.07, 36.90, 37.28],
'STARK-PRIME_R@20': [32.84, 30.50, 17.07, 32.95, 37.88, 37.34, 25.54, 41.11, 40.99, 27.78, 35.57, 34.05],
'STARK-PRIME_MRR': [21.68, 13.50, 11.42, 16.27, 23.50, 21.23, 15.00, 24.99, 23.70, 19.98, 26.27, 26.55]
}
data_human_generated = {
'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2', 'Claude3 Reranker', 'GPT4 Reranker'],
'STARK-AMAZON_Hit@1': [27.16, 16.05, 25.93, 22.22, 39.50, 35.80, 29.63, 40.74, 46.91, 33.33, 53.09, 50.62],
'STARK-AMAZON_Hit@5': [51.85, 39.51, 54.32, 49.38, 64.19, 62.96, 46.91, 71.60, 72.84, 55.56, 74.07, 75.31],
'STARK-AMAZON_R@20': [29.23, 15.23, 23.69, 21.54, 35.46, 33.01, 21.21, 36.30, 40.22, 29.03, 35.46, 35.46],
'STARK-AMAZON_MRR': [18.79, 27.21, 37.12, 31.33, 52.65, 47.84, 38.61, 53.21, 58.74, 43.77, 62.11, 61.06],
'STARK-MAG_Hit@1': [32.14, 4.72, 25.00, 20.24, 28.57, 22.62, 16.67, 34.52, 23.81, 33.33, 38.10, 36.90],
'STARK-MAG_Hit@5': [41.67, 9.52, 30.95, 26.19, 41.67, 36.90, 28.57, 44.04, 41.67, 36.90, 45.24, 46.43],
'STARK-MAG_R@20': [32.46, 25.00, 27.24, 28.76, 35.95, 32.44, 21.74, 34.57, 39.85, 30.50, 35.95, 35.95],
'STARK-MAG_MRR': [37.42, 7.90, 27.98, 25.53, 35.81, 29.68, 21.59, 38.72, 31.43, 35.97, 42.00, 40.65],
'STARK-PRIME_Hit@1': [22.45, 2.04, 7.14, 6.12, 17.35, 16.33, 9.18, 25.51, 24.49, 15.31, 28.57, 28.57],
'STARK-PRIME_Hit@5': [41.84, 9.18, 13.27, 13.27, 34.69, 32.65, 21.43, 41.84, 39.80, 26.53, 46.94, 44.90],
'STARK-PRIME_R@20': [42.32, 10.69, 11.72, 17.62, 41.09, 39.01, 26.77, 48.10, 47.21, 25.56, 41.61, 41.61],
'STARK-PRIME_MRR': [30.37, 7.05, 10.07, 9.39, 26.35, 24.33, 15.24, 34.28, 32.98, 19.67, 36.32, 34.82]
}
# Initialize DataFrames
df_synthesized_full = pd.DataFrame(data_synthesized_full)
df_synthesized_10 = pd.DataFrame(data_synthesized_10)
df_human_generated = pd.DataFrame(data_human_generated)
# Model type definitions
model_types = {
'Sparse Retriever': ['BM25'],
'Small Dense Retrievers': ['DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)'],
'LLM-based Dense Retrievers': ['ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b'],
'Multivector Retrievers': ['multi-ada-002', 'ColBERTv2'],
'LLM Rerankers': ['Claude3 Reranker', 'GPT4 Reranker'],
'Others': [] # Will be populated dynamically with submitted models
}
# Submission form validation functions
def validate_email(email_str):
"""Validate email format(s)"""
emails = [e.strip() for e in email_str.split(';')]
email_pattern = re.compile(r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$')
return all(email_pattern.match(email) for email in emails)
def validate_github_url(url):
"""Validate GitHub URL format"""
github_pattern = re.compile(
r'^https?:\/\/(?:www\.)?github\.com\/[\w-]+\/[\w.-]+\/?$'
)
return bool(github_pattern.match(url))
def validate_csv(file_path):
"""Validate CSV file format and content with better error handling"""
try:
df = pd.read_csv(file_path)
required_cols = ['query_id', 'pred_rank']
# Check for required columns
missing_cols = [col for col in required_cols if col not in df.columns]
if missing_cols:
return False, f"Missing required columns: {', '.join(missing_cols)}"
# Validate first few rows to ensure proper format
for idx, row in df.head().iterrows():
try:
rank_list = eval(row['pred_rank']) if isinstance(row['pred_rank'], str) else row['pred_rank']
if not isinstance(rank_list, list):
return False, f"pred_rank must be a list (row {idx})"
if len(rank_list) < 20:
return False, f"pred_rank must contain at least 20 candidates (row {idx})"
except Exception as e:
return False, f"Invalid pred_rank format in row {idx}: {str(e)}"
return True, "Valid CSV file"
except pd.errors.EmptyDataError:
return False, "CSV file is empty"
except Exception as e:
return False, f"Error processing CSV: {str(e)}"
def sanitize_name(name):
"""Sanitize name for file system use"""
return re.sub(r'[^a-zA-Z0-9]', '_', name)
def read_json_from_hub(api: HfApi, repo_id: str, file_path: str) -> dict:
"""
Read and parse JSON file from HuggingFace Hub.
Args:
api: HuggingFace API instance
repo_id: Repository ID
file_path: Path to file in repository
Returns:
dict: Parsed JSON content
"""
try:
# Download the file content as bytes
content = api.hf_hub_download(
repo_id=repo_id,
filename=file_path,
repo_type="space"
)
# Read and parse JSON
with open(content, 'r') as f:
return json.load(f)
except Exception as e:
print(f"Error reading JSON file {file_path}: {str(e)}")
return None
def scan_submissions_directory():
"""
Scans the submissions directory and updates the model types dictionary
with submitted models.
"""
try:
# Initialize HuggingFace API
api = HfApi()
# Track submissions for each split
submissions_by_split = {
'test': [],
'test-0.1': [],
'human_generated_eval': []
}
# Get all files from repository
try:
all_files = api.list_repo_files(
repo_id=REPO_ID,
repo_type="space"
)
# Filter for files in submissions directory
repo_files = [f for f in all_files if f.startswith('submissions/')]
except Exception as e:
print(f"Error listing repository contents: {str(e)}")
return submissions_by_split
# Group files by team folders
folder_files = {}
for filepath in repo_files:
parts = filepath.split('/')
if len(parts) < 3: # Need at least submissions/team_folder/file
continue
folder_name = parts[1] # team_folder name
if folder_name not in folder_files:
folder_files[folder_name] = []
folder_files[folder_name].append(filepath)
# Process each team folder
for folder_name, files in folder_files.items():
try:
# Find latest.json in this folder
latest_file = next((f for f in files if f.endswith('latest.json')), None)
if not latest_file:
print(f"No latest.json found in {folder_name}")
continue
# Read latest.json
latest_info = read_json_from_hub(api, REPO_ID, latest_file)
if not latest_info:
print(f"Failed to read latest.json for {folder_name}")
continue
timestamp = latest_info.get('latest_submission')
if not timestamp:
print(f"No timestamp found in latest.json for {folder_name}")
continue
# Find metadata file for latest submission
metadata_file = next(
(f for f in files if f.endswith(f'metadata_{timestamp}.json')),
None
)
if not metadata_file:
print(f"No matching metadata file found for {folder_name} timestamp {timestamp}")
continue
# Read metadata file
submission_data = read_json_from_hub(api, REPO_ID, metadata_file)
if not submission_data:
print(f"Failed to read metadata for {folder_name}")
continue
if latest_info.get('status') != 'approved':
print(f"Skipping unapproved submission in {folder_name}")
continue
# Add to submissions by split
split = submission_data.get('Split')
if split in submissions_by_split:
submissions_by_split[split].append(submission_data)
# Update model types if necessary
method_name = submission_data.get('Method Name')
model_type = submission_data.get('Model Type', 'Others')
# Add to model type if it's a new method
method_exists = any(method_name in methods for methods in model_types.values())
if not method_exists and model_type in model_types:
model_types[model_type].append(method_name)
except Exception as e:
print(f"Error processing folder {folder_name}: {str(e)}")
continue
return submissions_by_split
except Exception as e:
print(f"Error scanning submissions directory: {str(e)}")
return None
def initialize_leaderboard():
"""
Initialize the leaderboard with baseline results and submitted results.
"""
global df_synthesized_full, df_synthesized_10, df_human_generated
try:
# First, initialize with baseline results
df_synthesized_full = pd.DataFrame(data_synthesized_full)
df_synthesized_10 = pd.DataFrame(data_synthesized_10)
df_human_generated = pd.DataFrame(data_human_generated)
print("Initialized with baseline results")
# Then scan and add submitted results
submissions = scan_submissions_directory()
if submissions:
for split, split_submissions in submissions.items():
for submission in split_submissions:
if submission.get('results'): # Make sure we have results
# Update appropriate DataFrame based on split
if split == 'test':
df_to_update = df_synthesized_full
elif split == 'test-0.1':
df_to_update = df_synthesized_10
else: # human_generated_eval
df_to_update = df_human_generated
# Prepare new row data
new_row = {
'Method': submission['Method Name'],
f'STARK-{submission["Dataset"].upper()}_Hit@1': submission['results']['hit@1'],
f'STARK-{submission["Dataset"].upper()}_Hit@5': submission['results']['hit@5'],
f'STARK-{submission["Dataset"].upper()}_R@20': submission['results']['recall@20'],
f'STARK-{submission["Dataset"].upper()}_MRR': submission['results']['mrr']
}
# Update existing row or add new one
method_mask = df_to_update['Method'] == submission['Method Name']
if method_mask.any():
for col in new_row:
df_to_update.loc[method_mask, col] = new_row[col]
else:
df_to_update.loc[len(df_to_update)] = new_row
print("Leaderboard initialization complete")
except Exception as e:
print(f"Error initializing leaderboard: {str(e)}")
def get_file_content(file_path):
"""
Helper function to safely read file content from HuggingFace repository
"""
try:
api = HfApi()
content_path = api.hf_hub_download(
repo_id=REPO_ID,
filename=file_path,
repo_type="space"
)
with open(content_path, 'r') as f:
return f.read()
except Exception as e:
print(f"Error reading file {file_path}: {str(e)}")
return None
def save_submission(submission_data, csv_file):
"""
Save submission data and CSV file using model_name_team_name format
Args:
submission_data (dict): Metadata and results for the submission
csv_file: The uploaded CSV file object
"""
# Create folder name from model name and team name
model_name_clean = sanitize_name(submission_data['Method Name'])
team_name_clean = sanitize_name(submission_data['Team Name'])
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
# Create folder name: model_name_team_name
folder_name = f"{model_name_clean}_{team_name_clean}"
submission_id = f"{folder_name}_{timestamp}"
# Create submission directory structure
base_dir = "submissions"
submission_dir = os.path.join(base_dir, folder_name)
os.makedirs(submission_dir, exist_ok=True)
# Save CSV file with timestamp to allow multiple submissions
csv_filename = f"predictions_{timestamp}.csv"
csv_path = os.path.join(submission_dir, csv_filename)
if hasattr(csv_file, 'name'):
with open(csv_file.name, 'rb') as source, open(csv_path, 'wb') as target:
target.write(source.read())
# Add file paths to submission data
submission_data.update({
"csv_path": csv_path,
"submission_id": submission_id,
"folder_name": folder_name
})
# Save metadata as JSON with timestamp
metadata_path = os.path.join(submission_dir, f"metadata_{timestamp}.json")
with open(metadata_path, 'w') as f:
json.dump(submission_data, f, indent=4)
# Update latest.json to track most recent submission
latest_path = os.path.join(submission_dir, "latest.json")
with open(latest_path, 'w') as f:
json.dump({
"latest_submission": timestamp,
"status": "pending_review",
"method_name": submission_data['Method Name']
}, f, indent=4)
return submission_id
def update_leaderboard_data(submission_data):
"""
Update leaderboard data with new submission results
Only updates the specific dataset submitted, preventing empty rows
"""
global df_synthesized_full, df_synthesized_10, df_human_generated
# Determine which DataFrame to update based on split
split_to_df = {
'test': df_synthesized_full,
'test-0.1': df_synthesized_10,
'human_generated_eval': df_human_generated
}
df_to_update = split_to_df[submission_data['Split']]
dataset = submission_data['Dataset'].upper()
# Prepare new row data with only the relevant dataset columns
new_row = {
'Method': submission_data['Method Name']
}
# Only add metrics for the submitted dataset
new_row.update({
f'STARK-{dataset}_Hit@1': submission_data['results']['hit@1'],
f'STARK-{dataset}_Hit@5': submission_data['results']['hit@5'],
f'STARK-{dataset}_R@20': submission_data['results']['recall@20'],
f'STARK-{dataset}_MRR': submission_data['results']['mrr']
})
# Check if method already exists
method_mask = df_to_update['Method'] == submission_data['Method Name']
if method_mask.any():
# Update only the columns for the submitted dataset
for col in new_row:
df_to_update.loc[method_mask, col] = new_row[col]
else:
# For new methods, create a row with only the submitted dataset's values
df_to_update.loc[len(df_to_update)] = new_row
# Function to get emails from meta_data
def get_emails_from_metadata(meta_data):
"""
Extracts emails from the meta_data dictionary.
Args:
meta_data (dict): The metadata dictionary that contains the 'Contact Email(s)' field.
Returns:
list: A list of email addresses.
"""
return [email.strip() for email in meta_data.get("Contact Email(s)", "").split(";")]
# Function to format meta_data as an HTML table (without Prediction CSV)
def format_metadata_as_table(meta_data):
"""
Formats metadata dictionary into an HTML table for the email.
Handles multiple contact emails separated by a semicolon.
Args:
meta_data (dict): Dictionary containing submission metadata.
Returns:
str: HTML string representing the metadata table.
"""
table_rows = ""
for key, value in meta_data.items():
if key == "Contact Email(s)":
# Ensure that contact emails are split by semicolon
emails = value.split(';')
formatted_emails = "; ".join([email.strip() for email in emails])
table_rows += f"| {key} | {formatted_emails} |
"
elif key != "Prediction CSV": # Exclude the Prediction CSV field
table_rows += f"| {key} | {value} |
"
table_html = f"""
"""
return table_html
# Function to get emails from meta_data
def get_emails_from_metadata(meta_data):
"""
Extracts emails from the meta_data dictionary.
Args:
meta_data (dict): The metadata dictionary that contains the 'Contact Email(s)' field.
Returns:
list: A list of email addresses.
"""
return [email.strip() for email in meta_data.get("Contact Email(s)", "").split(";")]
def format_evaluation_results(results):
"""
Formats the evaluation results dictionary into a readable string.
Args:
results (dict): Dictionary containing evaluation metrics and their values.
Returns:
str: Formatted string of evaluation results.
"""
result_lines = [f"{metric}: {value}" for metric, value in results.items()]
return "\n".join(result_lines)
def get_model_type_for_method(method_name):
"""
Find the model type category for a given method name.
Returns 'Others' if not found in predefined categories.
"""
for type_name, methods in model_types.items():
if method_name in methods:
return type_name
return 'Others'
def validate_model_type(method_name, selected_type):
"""
Validate if the selected model type is appropriate for the method name.
Returns (is_valid, message).
"""
# Check if method exists in any category
existing_type = None
for type_name, methods in model_types.items():
if method_name in methods:
existing_type = type_name
break
# If method exists, it must be submitted under its predefined category
if existing_type:
if existing_type != selected_type:
return False, f"This method name is already registered under '{existing_type}'. Please use the correct category."
return True, "Valid model type"
# For new methods, any category is valid
return True, "Valid model type"
def process_submission(
method_name, team_name, dataset, split, contact_email,
code_repo, csv_file, model_description, hardware, paper_link, model_type
):
"""Process submission with progress updates"""
temp_files = []
start_time = time.time()
try:
# 1. Initial validation
yield "Validating submission details..."
if not all([method_name, team_name, dataset, split, contact_email, code_repo, csv_file, model_type]):
return "Error: Please fill in all required fields"
# 2. Process CSV
yield "Processing CSV file..."
temp_csv_path = None
if isinstance(csv_file, str):
temp_csv_path = csv_file
else:
try:
temp_fd, temp_csv_path = tempfile.mkstemp(suffix='.csv')
temp_files.append(temp_csv_path)
os.close(temp_fd)
shutil.copy2(csv_file.name, temp_csv_path)
except Exception as e:
return f"Error processing CSV file: {str(e)}"
# 3. Validate CSV format
yield "Validating CSV format..."
try:
df = pd.read_csv(temp_csv_path)
if 'query_id' not in df.columns or 'pred_rank' not in df.columns:
return "Error: CSV must contain 'query_id' and 'pred_rank' columns"
except Exception as e:
return f"Error reading CSV: {str(e)}"
# 4. Compute metrics with reduced workers
yield f"Computing metrics for {dataset}..."
results = compute_metrics(
csv_path=temp_csv_path,
dataset=dataset.lower(),
split=split,
num_threads=4
)
if isinstance(results, str):
return f"Evaluation error: {results}"
# 5. Process results
yield "Processing results..."
processed_results = {
"hit@1": round(results['hit@1'] * 100, 2),
"hit@5": round(results['hit@5'] * 100, 2),
"recall@20": round(results['recall@20'] * 100, 2),
"mrr": round(results['mrr'] * 100, 2)
}
# 6. Save submission
yield "Saving submission..."
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
folder_name = f"{sanitize_name(method_name)}_{sanitize_name(team_name)}"
submission_data = {
"Method Name": method_name,
"Team Name": team_name,
"Dataset": dataset,
"Split": split,
"Contact Email(s)": contact_email,
"Code Repository": code_repo,
"Model Description": model_description,
"Hardware": hardware,
"Paper link": paper_link,
"Model Type": model_type,
"results": processed_results
}
try:
# Save to HuggingFace Hub
csv_path_in_repo = f"submissions/{folder_name}/predictions_{timestamp}.csv"
hub_storage.save_to_hub(
file_content=temp_csv_path,
path_in_repo=csv_path_in_repo,
commit_message=f"Add submission: {method_name}"
)
except Exception as e:
return f"Error saving to HuggingFace Hub: {str(e)}"
# 7. Update leaderboard
yield "Updating leaderboard..."
update_leaderboard_data(submission_data)
return f"""
Submission successful!
Evaluation Results:
Hit@1: {processed_results['hit@1']:.2f}%
Hit@5: {processed_results['hit@5']:.2f}%
Recall@20: {processed_results['recall@20']:.2f}%
MRR: {processed_results['mrr']:.2f}%
Your submission will appear in the leaderboard after review.
"""
except Exception as e:
total_time = time.time() - start_time
return f"Error ({total_time:.1f}s): {str(e)}"
finally:
for temp_file in temp_files:
try:
if os.path.exists(temp_file):
os.unlink(temp_file)
except Exception as e:
print(f"Warning: Failed to delete {temp_file}: {str(e)}")
def filter_by_model_type(df, selected_types):
"""
Filter DataFrame by selected model types, including submitted models.
"""
if not selected_types:
return df.head(0)
# Get all models from selected types
selected_models = []
for type_name in selected_types:
selected_models.extend(model_types[type_name])
# Filter DataFrame to include only selected models
return df[df['Method'].isin(selected_models)]
def format_dataframe(df, dataset):
"""
Format DataFrame for display, removing rows with no data for the selected dataset
"""
# Select relevant columns
columns = ['Method'] + [col for col in df.columns if dataset in col]
filtered_df = df[columns].copy()
# Remove rows where all metric columns are empty/NaN for this dataset
metric_columns = [col for col in filtered_df.columns if col != 'Method']
filtered_df = filtered_df.dropna(subset=metric_columns, how='all')
# Rename columns to remove dataset prefix
filtered_df.columns = [col.split('_')[-1] if '_' in col else col for col in filtered_df.columns]
# Sort by MRR
if 'MRR' in filtered_df.columns:
filtered_df = filtered_df.sort_values('MRR', ascending=False)
return filtered_df
def update_tables(selected_types):
"""
Update tables based on selected model types.
Include all models from selected categories.
"""
if not selected_types:
return [df.head(0) for df in [df_synthesized_full, df_synthesized_10, df_human_generated]]
filtered_df_full = filter_by_model_type(df_synthesized_full, selected_types)
filtered_df_10 = filter_by_model_type(df_synthesized_10, selected_types)
filtered_df_human = filter_by_model_type(df_human_generated, selected_types)
outputs = []
for df in [filtered_df_full, filtered_df_10, filtered_df_human]:
for dataset in ['AMAZON', 'MAG', 'PRIME']:
outputs.append(format_dataframe(df, f"STARK-{dataset}"))
return outputs
css = """
table > thead {
white-space: normal
}
table {
--cell-width-1: 250px
}
table > tbody > tr > td:nth-child(2) > div {
overflow-x: auto
}
.tab-nav {
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
margin-bottom: 1rem;
}
"""
# Main application
with gr.Blocks(css=css) as demo:
gr.Markdown("# Semi-structured Retrieval Benchmark (STaRK) Leaderboard")
gr.Markdown("Refer to the [STaRK paper](https://arxiv.org/pdf/2404.13207) for details on metrics, tasks and models.")
# Initialize leaderboard at startup
print("Starting leaderboard initialization...")
initialize_leaderboard()
print("Leaderboard initialization finished")
# Model type filter
model_type_filter = gr.CheckboxGroup(
choices=list(model_types.keys()),
value=list(model_types.keys()),
label="Model types",
interactive=True
)
# Initialize dataframes list
all_dfs = []
# Create nested tabs structure
with gr.Tabs() as outer_tabs:
with gr.TabItem("Synthesized (full)"):
with gr.Tabs() as inner_tabs1:
for dataset in ['AMAZON', 'MAG', 'PRIME']:
with gr.TabItem(dataset):
all_dfs.append(gr.DataFrame(interactive=False))
with gr.TabItem("Synthesized (10%)"):
with gr.Tabs() as inner_tabs2:
for dataset in ['AMAZON', 'MAG', 'PRIME']:
with gr.TabItem(dataset):
all_dfs.append(gr.DataFrame(interactive=False))
with gr.TabItem("Human-Generated"):
with gr.Tabs() as inner_tabs3:
for dataset in ['AMAZON', 'MAG', 'PRIME']:
with gr.TabItem(dataset):
all_dfs.append(gr.DataFrame(interactive=False))
# Submission section
gr.Markdown("---")
gr.Markdown("## Submit Your Results")
gr.Markdown("""
Submit your results to be included in the leaderboard. Please ensure your submission meets all requirements.
For questions, contact stark-qa@cs.stanford.edu. Detailed instructions can be referred at [submission instructions](https://docs.google.com/document/d/11coGjTmOEi9p9-PUq1oy0eTOj8f_8CVQhDl5_0FKT14/edit?usp=sharing).
""")
with gr.Row():
with gr.Column():
method_name = gr.Textbox(
label="Method Name (max 25 chars)*",
placeholder="e.g., MyRetrievalModel-v1"
)
dataset = gr.Dropdown(
choices=["amazon", "mag", "prime"],
label="Dataset*",
value="amazon"
)
split = gr.Dropdown(
choices=["test", "test-0.1", "human_generated_eval"],
label="Split*",
value="test"
)
team_name = gr.Textbox(
label="Team Name (max 25 chars)*",
placeholder="e.g., Stanford NLP"
)
contact_email = gr.Textbox(
label="Contact Email(s)*",
placeholder="email@example.com; another@example.com"
)
model_type = gr.Dropdown(
choices=list(model_types.keys()),
label="Model Type*",
value="Others",
info="Select the appropriate category for your model"
)
with gr.Column():
model_description = gr.Textbox(
label="Model Description*",
lines=3,
placeholder="Briefly describe how your retriever model works..."
)
code_repo = gr.Textbox(
label="Code Repository*",
placeholder="https://github.com/snap-stanford/stark-leaderboard"
)
hardware = gr.Textbox(
label="Hardware Specifications*",
placeholder="e.g., 4x NVIDIA A100 80GB"
)
csv_file = gr.File(
label="Prediction CSV*",
file_types=[".csv"],
type="filepath"
)
paper_link = gr.Textbox(
label="Paper Link (Optional)",
placeholder="https://arxiv.org/abs/..."
)
submit_btn = gr.Button("Submit", variant="primary")
result = gr.Textbox(label="Submission Status", interactive=False)
# Set up event handlers
model_type_filter.change(
update_tables,
inputs=[model_type_filter],
outputs=all_dfs
)
# Event handler for submission button
submit_btn.click(
fn=process_submission,
inputs=[
method_name, team_name, dataset, split, contact_email,
code_repo, csv_file, model_description, hardware, paper_link, model_type
],
outputs=result,
api_name="submit"
).success( # Add success handler to update tables
fn=update_tables,
inputs=[model_type_filter],
outputs=all_dfs
)
# Initial table update
demo.load(
update_tables,
inputs=[model_type_filter],
outputs=all_dfs
)
# Launch the application
demo.launch()