Update space

app.py

@@ -84,23 +84,44 @@ def compute_metrics(csv_path: str, dataset: str, split: str, num_workers: int =
         all_indices = split_idx[split].tolist()
 
         results_list = []
-        query_ids = []
-
-        # Prepare args for each worker
-        args = [(idx, eval_csv, qa_dataset, evaluator, eval_metrics) for idx in all_indices]
-
-        with ProcessPoolExecutor(max_workers=num_workers) as executor:
-            futures = [executor.submit(process_single_instance, arg) for arg in args]
-            for future in tqdm(as_completed(futures), total=len(futures)):
-                result = future.result()  # This will raise an error if the worker encountered one
-                results_list.append(result)
-                query_ids.append(result['query_id'])
-
-        # Concatenate results and compute final metrics
-        eval_csv = pd.concat([eval_csv, pd.DataFrame(results_list)], ignore_index=True)
+        # query_ids = []
+
+        # # Prepare args for each worker
+        # args = [(idx, eval_csv, qa_dataset, evaluator, eval_metrics) for idx in all_indices]
+
+        # with ProcessPoolExecutor(max_workers=num_workers) as executor:
+        #     futures = [executor.submit(process_single_instance, arg) for arg in args]
+        #     for future in tqdm(as_completed(futures), total=len(futures)):
+        #         result = future.result()  # This will raise an error if the worker encountered one
+        #         results_list.append(result)
+        #         query_ids.append(result['query_id'])
+
+        # # Concatenate results and compute final metrics
+        # eval_csv = pd.concat([eval_csv, pd.DataFrame(results_list)], ignore_index=True)
+        # final_results = {
+        #     metric: np.mean(eval_csv[eval_csv['query_id'].isin(query_ids)][metric]) for metric in eval_metrics
+        # }
+        # return final_result
+        batch_size = 100
+        for i in range(0, len(all_indices), batch_size):
+            max_ind = min(i+batch_size, len(all_indices))
+            batch_indices = all_indices[i:max_ind]
+            args = [(idx, eval_csv, qa_dataset, evaluator, eval_metrics) 
+                   for idx in batch_indices]
+            
+            with ProcessPoolExecutor(max_workers=num_workers) as executor:
+                futures = [executor.submit(process_single_instance, arg) 
+                          for arg in args]
+                for future in as_completed(futures):
+                    results_list.append(future.result())
+
+        # Compute final metrics
+        results_df = pd.DataFrame(results_list)
         final_results = {
-            metric: np.mean(eval_csv[eval_csv['query_id'].isin(query_ids)][metric]) for metric in eval_metrics
+            metric: results_df[metric].mean() 
+            for metric in eval_metrics
         }
+        
         return final_results
 
     except pd.errors.EmptyDataError:
@@ -616,144 +637,108 @@ def process_submission(
     method_name, team_name, dataset, split, contact_email,
     code_repo, csv_file, model_description, hardware, paper_link, model_type
 ):
-    """Process and validate submission with better error handling and progress updates"""
+    """Process submission with progress updates"""
     try:
-        # 1. Initial validation with early returns
+        # 1. Initial validation
+        yield "Validating submission details..."
         if not all([method_name, team_name, dataset, split, contact_email, code_repo, csv_file, model_type]):
             return "Error: Please fill in all required fields"
         
-        if len(method_name) > 25:
-            return "Error: Method name must be 25 characters or less"
-        
-        if len(team_name) > 25:
-            return "Error: Team name must be 25 characters or less"
-            
-        # 2. Validate model type
-        is_valid, message = validate_model_type(method_name, model_type)
-        if not is_valid:
-            return f"Error: {message}"
-
-        # 3. Create temporary directory for processing
-        with tempfile.TemporaryDirectory() as temp_dir:
-            # Copy CSV file to temp directory
-            temp_csv_path = os.path.join(temp_dir, "submission.csv")
-            if isinstance(csv_file, str):
-                shutil.copy2(csv_file, temp_csv_path)
-            else:
-                with open(temp_csv_path, 'wb') as temp_file:
-                    if hasattr(csv_file, 'seek'):
-                        csv_file.seek(0)
-                    if hasattr(csv_file, 'read'):
-                        shutil.copyfileobj(csv_file, temp_file)
-                    else:
-                        temp_file.write(csv_file)
-
-            # 4. Validate CSV format
-            is_valid_csv, csv_message = validate_csv(temp_csv_path)
-            if not is_valid_csv:
-                return f"Error validating CSV: {csv_message}"
-
-            # 5. Compute metrics with progress indication
-            print(f"Computing metrics for {dataset.lower()} dataset...")
-            results = compute_metrics(
-                csv_path=temp_csv_path,
-                dataset=dataset.lower(),
-                split=split,
-                num_workers=4
-            )
-            
-            if isinstance(results, str):
-                return f"Evaluation error: {results}"
-
-            # 6. Process results
-            processed_results = {
-                metric: round(value * 100, 2)
-                for metric, value in results.items()
-            }
-
-            # 7. Prepare submission data
-            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-            folder_name = f"{sanitize_name(method_name)}_{sanitize_name(team_name)}"
-            
-            submission_data = {
-                "Method Name": method_name,
-                "Team Name": team_name,
-                "Dataset": dataset,
-                "Split": split,
-                "Contact Email(s)": contact_email,
-                "Code Repository": code_repo,
-                "Model Description": model_description,
-                "Hardware": hardware,
-                "(Optional) Paper link": paper_link,
-                "Model Type": model_type,
-                "results": processed_results,
-                "status": "pending_review",
-                "submission_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            }
-
-            # 8. Save to HuggingFace Hub with error handling
+        # 2. Process CSV
+        yield "Processing CSV file..."
+        temp_csv_path = None
+        if isinstance(csv_file, str):
+            temp_csv_path = csv_file
+        else:
             try:
-                # Save CSV
-                csv_path_in_repo = f"submissions/{folder_name}/predictions_{timestamp}.csv"
-                hub_storage.save_to_hub(
-                    file_content=temp_csv_path,
-                    path_in_repo=csv_path_in_repo,
-                    commit_message=f"Add submission CSV: {method_name} by {team_name}"
-                )
-                submission_data["csv_path"] = csv_path_in_repo
-
-                # Save metadata
-                metadata_path = f"submissions/{folder_name}/metadata_{timestamp}.json"
-                with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as tmp:
-                    json.dump(submission_data, tmp, indent=4)
-                    tmp.flush()
-                    hub_storage.save_to_hub(
-                        file_content=tmp.name,
-                        path_in_repo=metadata_path,
-                        commit_message=f"Add metadata: {method_name} by {team_name}"
-                    )
-
-                # Update latest.json
-                latest_path = f"submissions/{folder_name}/latest.json"
-                latest_info = {
-                    "latest_submission": timestamp,
-                    "status": "pending_review",
-                    "method_name": method_name
-                }
-                with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as tmp:
-                    json.dump(latest_info, tmp, indent=4)
-                    tmp.flush()
-                    hub_storage.save_to_hub(
-                        file_content=tmp.name,
-                        path_in_repo=latest_path,
-                        commit_message=f"Update latest submission info for {method_name}"
-                    )
-
+                temp_fd, temp_csv_path = tempfile.mkstemp(suffix='.csv')
+                os.close(temp_fd)
+                shutil.copy2(csv_file.name, temp_csv_path)
             except Exception as e:
-                return f"Failed to save to HuggingFace Hub: {str(e)}"
+                return f"Error processing CSV file: {str(e)}"
+
+        # 3. Validate CSV format
+        yield "Validating CSV format..."
+        try:
+            df = pd.read_csv(temp_csv_path)
+            if 'query_id' not in df.columns or 'pred_rank' not in df.columns:
+                return "Error: CSV must contain 'query_id' and 'pred_rank' columns"
+        except Exception as e:
+            return f"Error reading CSV: {str(e)}"
+
+        # 4. Compute metrics with reduced workers
+        yield f"Computing metrics for {dataset}..."
+        results = compute_metrics(
+            csv_path=temp_csv_path,
+            dataset=dataset.lower(),
+            split=split,
+            num_workers=2  # Reduced from 4 to 2
+        )
+        
+        if isinstance(results, str):
+            return f"Evaluation error: {results}"
+
+        # 5. Process results
+        yield "Processing results..."
+        processed_results = {
+            "hit@1": round(results['hit@1'] * 100, 2),
+            "hit@5": round(results['hit@5'] * 100, 2),
+            "recall@20": round(results['recall@20'] * 100, 2),
+            "mrr": round(results['mrr'] * 100, 2)
+        }
 
-            # 9. Update leaderboard
-            update_leaderboard_data(submission_data)
+        # 6. Save submission
+        yield "Saving submission..."
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        folder_name = f"{sanitize_name(method_name)}_{sanitize_name(team_name)}"
+        
+        submission_data = {
+            "Method Name": method_name,
+            "Team Name": team_name,
+            "Dataset": dataset,
+            "Split": split,
+            "Contact Email(s)": contact_email,
+            "Code Repository": code_repo,
+            "Model Description": model_description,
+            "Hardware": hardware,
+            "Paper link": paper_link,
+            "Model Type": model_type,
+            "results": processed_results
+        }
 
-            # 10. Return success message
-            return f"""
-            Submission successful! 
-            
-            Evaluation Results:
-            Hit@1: {processed_results['hit@1']:.2f}%
-            Hit@5: {processed_results['hit@5']:.2f}%
-            Recall@20: {processed_results['recall@20']:.2f}%
-            MRR: {processed_results['mrr']:.2f}%
-            
-            Your submission has been saved and will be reviewed.
-            Once approved, your results will appear in the leaderboard as: {method_name}
-            
-            You can find your submission at:
-            https://huggingface.co/spaces/{REPO_ID}/tree/main/submissions/{folder_name}
-            """
+        try:
+            # Save to HuggingFace Hub
+            csv_path_in_repo = f"submissions/{folder_name}/predictions_{timestamp}.csv"
+            hub_storage.save_to_hub(
+                file_content=temp_csv_path,
+                path_in_repo=csv_path_in_repo,
+                commit_message=f"Add submission: {method_name}"
+            )
+        except Exception as e:
+            return f"Error saving to HuggingFace Hub: {str(e)}"
+
+        # 7. Update leaderboard
+        yield "Updating leaderboard..."
+        update_leaderboard_data(submission_data)
+
+        return f"""
+        Submission successful! 
+        
+        Evaluation Results:
+        Hit@1: {processed_results['hit@1']:.2f}%
+        Hit@5: {processed_results['hit@5']:.2f}%
+        Recall@20: {processed_results['recall@20']:.2f}%
+        MRR: {processed_results['mrr']:.2f}%
+        
+        Your submission will appear in the leaderboard after review.
+        """
 
     except Exception as e:
-        return f"Error processing submission: {str(e)}"
+        return f"Error: {str(e)}"
+    finally:
+        # Cleanup
+        if temp_csv_path and os.path.exists(temp_csv_path):
+            os.unlink(temp_csv_path)
     
 def filter_by_model_type(df, selected_types):
     """
@@ -952,8 +937,9 @@ with gr.Blocks(css=css) as demo:
             method_name, team_name, dataset, split, contact_email,
             code_repo, csv_file, model_description, hardware, paper_link, model_type
         ],
-        outputs=result
-    ).success(  # Add a success handler to update tables after successful submission
+        outputs=result,
+        api_name="submit"
+    ).success(  # Add success handler to update tables
         fn=update_tables,
         inputs=[model_type_filter],
         outputs=all_dfs