Update space

app.py

@@ -189,23 +189,32 @@ def validate_github_url(url):
     )
     return bool(github_pattern.match(url))
 
-def validate_csv(file_obj):
-    """Validate CSV file format and content"""
+def validate_csv(file_path):
+    """Validate CSV file format and content with better error handling"""
     try:
-        df = pd.read_csv(file_obj.name)
+        df = pd.read_csv(file_path)
         required_cols = ['query_id', 'pred_rank']
         
-        if not all(col in df.columns for col in required_cols):
-            return False, "CSV must contain 'query_id' and 'pred_rank' columns"
-            
-        try:
-            first_rank = eval(df['pred_rank'].iloc[0]) if isinstance(df['pred_rank'].iloc[0], str) else df['pred_rank'].iloc[0]
-            if not isinstance(first_rank, list) or len(first_rank) < 20:
-                return False, "pred_rank must be a list with at least 20 candidates"
-        except:
-            return False, "Invalid pred_rank format"
-            
+        # Check for required columns
+        missing_cols = [col for col in required_cols if col not in df.columns]
+        if missing_cols:
+            return False, f"Missing required columns: {', '.join(missing_cols)}"
+        
+        # Validate first few rows to ensure proper format
+        for idx, row in df.head().iterrows():
+            try:
+                rank_list = eval(row['pred_rank']) if isinstance(row['pred_rank'], str) else row['pred_rank']
+                if not isinstance(rank_list, list):
+                    return False, f"pred_rank must be a list (row {idx})"
+                if len(rank_list) < 20:
+                    return False, f"pred_rank must contain at least 20 candidates (row {idx})"
+            except Exception as e:
+                return False, f"Invalid pred_rank format in row {idx}: {str(e)}"
+                
         return True, "Valid CSV file"
+        
+    except pd.errors.EmptyDataError:
+        return False, "CSV file is empty"
     except Exception as e:
         return False, f"Error processing CSV: {str(e)}"
 
@@ -465,7 +474,7 @@ def save_submission(submission_data, csv_file):
 def update_leaderboard_data(submission_data):
     """
     Update leaderboard data with new submission results
-    Only uses model name in the displayed table
+    Only updates the specific dataset submitted, preventing empty rows
     """
     global df_synthesized_full, df_synthesized_10, df_human_generated
     
@@ -477,26 +486,32 @@ def update_leaderboard_data(submission_data):
     }
     
     df_to_update = split_to_df[submission_data['Split']]
+    dataset = submission_data['Dataset'].upper()
     
-    # Prepare new row data
+    # Prepare new row data with only the relevant dataset columns
     new_row = {
-        'Method': submission_data['Method Name'],  # Only use method name in table
-        f'STARK-{submission_data["Dataset"].upper()}_Hit@1': submission_data['results']['hit@1'],
-        f'STARK-{submission_data["Dataset"].upper()}_Hit@5': submission_data['results']['hit@5'],
-        f'STARK-{submission_data["Dataset"].upper()}_R@20': submission_data['results']['recall@20'],
-        f'STARK-{submission_data["Dataset"].upper()}_MRR': submission_data['results']['mrr']
+        'Method': submission_data['Method Name']
     }
+    # Only add metrics for the submitted dataset
+    new_row.update({
+        f'STARK-{dataset}_Hit@1': submission_data['results']['hit@1'],
+        f'STARK-{dataset}_Hit@5': submission_data['results']['hit@5'],
+        f'STARK-{dataset}_R@20': submission_data['results']['recall@20'],
+        f'STARK-{dataset}_MRR': submission_data['results']['mrr']
+    })
     
     # Check if method already exists
     method_mask = df_to_update['Method'] == submission_data['Method Name']
     if method_mask.any():
-        # Update existing row
+        # Update only the columns for the submitted dataset
         for col in new_row:
             df_to_update.loc[method_mask, col] = new_row[col]
     else:
-        # Add new row
+        # For new methods, create a row with only the submitted dataset's values
         df_to_update.loc[len(df_to_update)] = new_row
 
+
+
 # Function to get emails from meta_data
 def get_emails_from_metadata(meta_data):
     """
@@ -601,47 +616,29 @@ def process_submission(
     method_name, team_name, dataset, split, contact_email,
     code_repo, csv_file, model_description, hardware, paper_link, model_type
 ):
-    """Process and validate submission"""
-    temp_files = []
+    """Process and validate submission with better error handling and progress updates"""
     try:
-        # Input validation
+        # 1. Initial validation with early returns
         if not all([method_name, team_name, dataset, split, contact_email, code_repo, csv_file, model_type]):
             return "Error: Please fill in all required fields"
         
-        # Validate model type
+        if len(method_name) > 25:
+            return "Error: Method name must be 25 characters or less"
+        
+        if len(team_name) > 25:
+            return "Error: Team name must be 25 characters or less"
+            
+        # 2. Validate model type
         is_valid, message = validate_model_type(method_name, model_type)
         if not is_valid:
             return f"Error: {message}"
 
-        # Create metadata
-        meta_data = {
-            "Method Name": method_name,
-            "Team Name": team_name,
-            "Dataset": dataset,
-            "Split": split,
-            "Contact Email(s)": contact_email,
-            "Code Repository": code_repo,
-            "Model Description": model_description,
-            "Hardware": hardware,
-            "(Optional) Paper link": paper_link,
-            "Model Type": model_type
-        }
-        
-        # Generate folder name and timestamp
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        folder_name = f"{sanitize_name(method_name)}_{sanitize_name(team_name)}"
-        
-        # Process CSV file
-        temp_csv_path = None
-        if isinstance(csv_file, str):
-            temp_csv_path = csv_file
-        else:
-            temp_fd, temp_csv_path = tempfile.mkstemp(suffix='.csv')
-            temp_files.append(temp_csv_path)
-            os.close(temp_fd)
-            
-            if hasattr(csv_file, 'name'):
-                shutil.copy2(csv_file.name, temp_csv_path)
+        # 3. Create temporary directory for processing
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Copy CSV file to temp directory
+            temp_csv_path = os.path.join(temp_dir, "submission.csv")
+            if isinstance(csv_file, str):
+                shutil.copy2(csv_file, temp_csv_path)
             else:
                 with open(temp_csv_path, 'wb') as temp_file:
                     if hasattr(csv_file, 'seek'):
@@ -651,125 +648,113 @@ def process_submission(
                     else:
                         temp_file.write(csv_file)
 
-        if not os.path.exists(temp_csv_path):
-            raise FileNotFoundError(f"Failed to create temporary CSV file at {temp_csv_path}")
-
-        # Compute metrics
-        results = compute_metrics(
-            csv_path=temp_csv_path,
-            dataset=dataset.lower(),
-            split=split,
-            num_workers=4
-        )
-        
-        if isinstance(results, str):
-            # send_error_notification(meta_data, results)
-            return f"Evaluation error: {results}"
-
-        # Process results
-        processed_results = {
-            "hit@1": round(results['hit@1'] * 100, 2),
-            "hit@5": round(results['hit@5'] * 100, 2),
-            "recall@20": round(results['recall@20'] * 100, 2),
-            "mrr": round(results['mrr'] * 100, 2)
-        }
-        
-        # Save files to HuggingFace Hub
-        try:
-            # 1. Save CSV file
-            csv_filename = f"predictions_{timestamp}.csv"
-            csv_path_in_repo = f"submissions/{folder_name}/{csv_filename}"
-            hub_storage.save_to_hub(
-                file_content=temp_csv_path,
-                path_in_repo=csv_path_in_repo,
-                commit_message=f"Add submission: {method_name} by {team_name}"
+            # 4. Validate CSV format
+            is_valid_csv, csv_message = validate_csv(temp_csv_path)
+            if not is_valid_csv:
+                return f"Error validating CSV: {csv_message}"
+
+            # 5. Compute metrics with progress indication
+            print(f"Computing metrics for {dataset.lower()} dataset...")
+            results = compute_metrics(
+                csv_path=temp_csv_path,
+                dataset=dataset.lower(),
+                split=split,
+                num_workers=4
             )
+            
+            if isinstance(results, str):
+                return f"Evaluation error: {results}"
+
+            # 6. Process results
+            processed_results = {
+                metric: round(value * 100, 2)
+                for metric, value in results.items()
+            }
 
-            # 2. Save metadata
+            # 7. Prepare submission data
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            folder_name = f"{sanitize_name(method_name)}_{sanitize_name(team_name)}"
+            
             submission_data = {
-                **meta_data,
+                "Method Name": method_name,
+                "Team Name": team_name,
+                "Dataset": dataset,
+                "Split": split,
+                "Contact Email(s)": contact_email,
+                "Code Repository": code_repo,
+                "Model Description": model_description,
+                "Hardware": hardware,
+                "(Optional) Paper link": paper_link,
+                "Model Type": model_type,
                 "results": processed_results,
-                "status": "approved",  # or "pending_review"
-                "submission_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-                "csv_path": csv_path_in_repo
+                "status": "pending_review",
+                "submission_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
             }
-            
-            metadata_fd, temp_metadata_path = tempfile.mkstemp(suffix='.json')
-            temp_files.append(temp_metadata_path)
-            os.close(metadata_fd)
-            
-            with open(temp_metadata_path, 'w') as f:
-                json.dump(submission_data, f, indent=4)
-            
-            metadata_path = f"submissions/{folder_name}/metadata_{timestamp}.json"
-            hub_storage.save_to_hub(
-                file_content=temp_metadata_path,
-                path_in_repo=metadata_path,
-                commit_message=f"Add metadata: {method_name} by {team_name}"
-            )
 
-            # 3. Create or update latest.json
-            latest_info = {
-                "latest_submission": timestamp,
-                "status": "approved",  # or "pending_review"
-                "method_name": method_name,
-                "team_name": team_name
-            }
+            # 8. Save to HuggingFace Hub with error handling
+            try:
+                # Save CSV
+                csv_path_in_repo = f"submissions/{folder_name}/predictions_{timestamp}.csv"
+                hub_storage.save_to_hub(
+                    file_content=temp_csv_path,
+                    path_in_repo=csv_path_in_repo,
+                    commit_message=f"Add submission CSV: {method_name} by {team_name}"
+                )
+                submission_data["csv_path"] = csv_path_in_repo
+
+                # Save metadata
+                metadata_path = f"submissions/{folder_name}/metadata_{timestamp}.json"
+                with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as tmp:
+                    json.dump(submission_data, tmp, indent=4)
+                    tmp.flush()
+                    hub_storage.save_to_hub(
+                        file_content=tmp.name,
+                        path_in_repo=metadata_path,
+                        commit_message=f"Add metadata: {method_name} by {team_name}"
+                    )
+
+                # Update latest.json
+                latest_path = f"submissions/{folder_name}/latest.json"
+                latest_info = {
+                    "latest_submission": timestamp,
+                    "status": "pending_review",
+                    "method_name": method_name
+                }
+                with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as tmp:
+                    json.dump(latest_info, tmp, indent=4)
+                    tmp.flush()
+                    hub_storage.save_to_hub(
+                        file_content=tmp.name,
+                        path_in_repo=latest_path,
+                        commit_message=f"Update latest submission info for {method_name}"
+                    )
+
+            except Exception as e:
+                return f"Failed to save to HuggingFace Hub: {str(e)}"
+
+            # 9. Update leaderboard
+            update_leaderboard_data(submission_data)
+
+            # 10. Return success message
+            return f"""
+            Submission successful! 
             
-            latest_fd, temp_latest_path = tempfile.mkstemp(suffix='.json')
-            temp_files.append(temp_latest_path)
-            os.close(latest_fd)
+            Evaluation Results:
+            Hit@1: {processed_results['hit@1']:.2f}%
+            Hit@5: {processed_results['hit@5']:.2f}%
+            Recall@20: {processed_results['recall@20']:.2f}%
+            MRR: {processed_results['mrr']:.2f}%
             
-            with open(temp_latest_path, 'w') as f:
-                json.dump(latest_info, f, indent=4)
+            Your submission has been saved and will be reviewed.
+            Once approved, your results will appear in the leaderboard as: {method_name}
             
-            latest_path = f"submissions/{folder_name}/latest.json"
-            hub_storage.save_to_hub(
-                file_content=temp_latest_path,
-                path_in_repo=latest_path,
-                commit_message=f"Update latest submission info for {method_name}"
-            )
+            You can find your submission at:
+            https://huggingface.co/spaces/{REPO_ID}/tree/main/submissions/{folder_name}
+            """
 
-        except Exception as e:
-            raise RuntimeError(f"Failed to save files to HuggingFace Hub: {str(e)}")
-        
-        # Send confirmation email and update leaderboard data
-        # send_submission_confirmation(meta_data, processed_results)
-        update_leaderboard_data(submission_data)
-        
-        # Return success message
-        return f"""
-        Submission successful! 
-        
-        Evaluation Results:
-        Hit@1: {processed_results['hit@1']:.2f}%
-        Hit@5: {processed_results['hit@5']:.2f}%
-        Recall@20: {processed_results['recall@20']:.2f}%
-        MRR: {processed_results['mrr']:.2f}%
-        
-        Your submission has been saved and a confirmation email has been sent to {contact_email}.
-        Once approved, your results will appear in the leaderboard under: {method_name}
-        
-        You can find your submission at:
-        https://huggingface.co/spaces/{REPO_ID}/tree/main/submissions/{folder_name}
-        
-        Please refresh the page to see your submission in the leaderboard.
-        """
-        
     except Exception as e:
-        error_message = f"Error processing submission: {str(e)}"
-        # send_error_notification(meta_data, error_message)
-        return error_message
-    finally:
-        # Clean up temporary files
-        for temp_file in temp_files:
-            try:
-                if os.path.exists(temp_file):
-                    os.unlink(temp_file)
-            except Exception as e:
-                print(f"Warning: Failed to delete temporary file {temp_file}: {str(e)}")
-
-
+        return f"Error processing submission: {str(e)}"
+    
 def filter_by_model_type(df, selected_types):
     """
     Filter DataFrame by selected model types, including submitted models.
@@ -786,10 +771,24 @@ def filter_by_model_type(df, selected_types):
     return df[df['Method'].isin(selected_models)]
 
 def format_dataframe(df, dataset):
+    """
+    Format DataFrame for display, removing rows with no data for the selected dataset
+    """
+    # Select relevant columns
     columns = ['Method'] + [col for col in df.columns if dataset in col]
     filtered_df = df[columns].copy()
+    
+    # Remove rows where all metric columns are empty/NaN for this dataset
+    metric_columns = [col for col in filtered_df.columns if col != 'Method']
+    filtered_df = filtered_df.dropna(subset=metric_columns, how='all')
+    
+    # Rename columns to remove dataset prefix
     filtered_df.columns = [col.split('_')[-1] if '_' in col else col for col in filtered_df.columns]
-    filtered_df = filtered_df.sort_values('MRR', ascending=False)
+    
+    # Sort by MRR
+    if 'MRR' in filtered_df.columns:
+        filtered_df = filtered_df.sort_values('MRR', ascending=False)
+    
     return filtered_df
 
 def update_tables(selected_types):

utils/hub_storage.py

@@ -6,36 +6,31 @@ class HubStorage:
     def __init__(self, repo_id):
         self.repo_id = repo_id
         self.api = HfApi()
-
-    def get_file_content(self, file_path):
-        """
-        Get content of a file from the repository
-        """
+        
+        # Validate repository access
         try:
-            content = self.api.hf_hub_download(
-                repo_id=self.repo_id,
-                repo_type="space",
-                filename=file_path,
-                text=True
-            )
-            return content
+            self.api.repo_info(repo_id=repo_id, repo_type="space")
         except Exception as e:
-            print(f"Error reading file {file_path}: {str(e)}")
-            return None
+            raise RuntimeError(f"Failed to access repository {repo_id}: {e}")
 
     def save_to_hub(self, file_content, path_in_repo, commit_message):
-        """
-        Save a file to the hub
-        """
-        try:
-            self.api.upload_file(
-                path_or_fileobj=file_content,
-                path_in_repo=path_in_repo,
-                repo_id=self.repo_id,
-                repo_type="space",
-                commit_message=commit_message
-            )
-            return True
-        except Exception as e:
-            print(f"Error saving file to hub: {str(e)}")
-            return False
+        """Save file to HuggingFace Hub with retries and better error handling"""
+        max_retries = 3
+        retry_delay = 1  # seconds
+        
+        for attempt in range(max_retries):
+            try:
+                self.api.upload_file(
+                    repo_id=self.repo_id,
+                    repo_type="space",
+                    path_or_fileobj=file_content,
+                    path_in_repo=path_in_repo,
+                    commit_message=commit_message
+                )
+                return True
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    raise RuntimeError(f"Failed to save file after {max_retries} attempts: {e}")
+                print(f"Attempt {attempt + 1} failed, retrying in {retry_delay} seconds...")
+                time.sleep(retry_delay)
+                retry_delay *= 2  # Exponential backoff