Update space

app.py

@@ -42,7 +42,7 @@ except Exception as e:
 
 def process_single_instance(args):
     """Process a single instance with improved validation and error handling"""
-    idx, eval_csv, qa_dataset, evaluator, eval_metrics, max_candidate_id = args
+    idx, eval_csv, qa_dataset, evaluator, eval_metrics = args
     try:
         # Get query data
         query, query_id, answer_ids, meta_info = qa_dataset[idx]
@@ -70,26 +70,22 @@ def process_single_instance(args):
             print(f"Warning: pred_rank is not a list for query_id {query_id}")
             return None
             
-        # Validate and filter prediction values
-        valid_pred_rank = []
-        for rank in pred_rank[:100]:  # Only use top 100 predictions
-            if isinstance(rank, (int, np.integer)) and 0 <= rank < max_candidate_id:
-                valid_pred_rank.append(rank)
-            else:
-                print(f"Warning: Invalid prediction {rank} for query_id {query_id}")
+        # # Validate and filter prediction values
+        # valid_pred_rank = []
+        # for rank in pred_rank[:100]:  # Only use top 100 predictions
+        #     if isinstance(rank, (int, np.integer)) and 0 <= rank < max_candidate_id:
+        #         valid_pred_rank.append(rank)
+        #     else:
+        #         print(f"Warning: Invalid prediction {rank} for query_id {query_id}")
                 
-        if not valid_pred_rank:
-            print(f"Warning: No valid predictions for query_id {query_id}")
-            return None
+        # if not valid_pred_rank:
+        #     print(f"Warning: No valid predictions for query_id {query_id}")
+        #     return None
             
-        # Create prediction dictionary with valid predictions only
-        pred_dict = {rank: -i for i, rank in enumerate(valid_pred_rank)}
-        
-        # Convert answer_ids to tensor
+        pred_dict = {pred_rank[i]: -i for i in range(min(100, len(pred_rank)))}
         answer_ids = torch.LongTensor(answer_ids)
-        
-        # Evaluate
         result = evaluator.evaluate(pred_dict, answer_ids, metrics=eval_metrics)
+
         result["idx"], result["query_id"] = idx, query_id
         return result
         
@@ -108,12 +104,6 @@ def compute_metrics(csv_path: str, dataset: str, split: str, num_workers: int =
         'prime': [i for i in range(129375)]
     }
     
-    candidate_size_dict = {
-        'amazon': 957192,
-        'mag': 700244,  # 1872968 - 1172724
-        'prime': 129375
-    }
-    
     try:
         # Input validation
         if dataset not in candidate_ids_dict:
@@ -129,7 +119,6 @@ def compute_metrics(csv_path: str, dataset: str, split: str, num_workers: int =
             raise ValueError(f"CSV must contain columns: {required_columns}")
         
         eval_csv = eval_csv[required_columns]
-        max_candidate_id = candidate_size_dict[dataset]
         
         # Initialize components
         evaluator = Evaluator(candidate_ids_dict[dataset])
@@ -149,7 +138,7 @@ def compute_metrics(csv_path: str, dataset: str, split: str, num_workers: int =
             futures = [
                 executor.submit(
                     process_single_instance, 
-                    (idx, eval_csv, qa_dataset, evaluator, eval_metrics, max_candidate_id)
+                    (idx, eval_csv, qa_dataset, evaluator, eval_metrics)
                 ) 
                 for idx in all_indices
             ]

submissions/abc_abc/latest.json

@@ -1,6 +0,0 @@
-{
-    "latest_submission": "20241115_004044",
-    "status": "approved",
-    "method_name": "abc",
-    "team_name": "abc"
-}

submissions/abc_abc/metadata_20241115_004044.json

@@ -1,21 +0,0 @@
-{
-    "Method Name": "abc",
-    "Team Name": "abc",
-    "Dataset": "prime",
-    "Split": "human_generated_eval",
-    "Contact Email(s)": "a@s.edu",
-    "Code Repository": "https://github.com/",
-    "Model Description": "abc",
-    "Hardware": "abc",
-    "(Optional) Paper link": "",
-    "Model Type": "Others",
-    "results": {
-        "hit@1": 0.0,
-        "hit@5": 0.0,
-        "recall@20": 0.0,
-        "mrr": 0.03
-    },
-    "status": "approved",
-    "submission_date": "2024-11-15 00:40:49",
-    "csv_path": "submissions/abc_abc/predictions_20241115_004044.csv"
-}