Update space

review_submission.py

@@ -2,41 +2,59 @@ import json
 import os
 from datetime import datetime
 import glob
-from app import SubmissionForum
-from utils.hub_storage import HubStorage
+from huggingface_hub import HfApi
 
+REPO_ID = "snap-stanford/stark-leaderboard"
 
+class ForumPost:
+    def __init__(self, message: str, timestamp: str, post_type: str):
+        self.message = message
+        self.timestamp = timestamp
+        self.post_type = post_type
 
-def read_json_file(file_path: str) -> dict:
-    """Read and parse local JSON file."""
+def load_forum_posts(forum_file="submissions/forum_posts.json"):
+    """Load forum posts from local file"""
     try:
-        with open(file_path, 'r') as f:
-            return json.load(f)
+        if os.path.exists(forum_file):
+            with open(forum_file, 'r') as f:
+                posts_data = json.load(f)
+                return [ForumPost(**post) for post in posts_data]
+        return []
     except Exception as e:
-        print(f"Error reading {file_path}: {str(e)}")
-        return None
+        print(f"Error loading forum posts: {e}")
+        return []
 
-def update_json_file(file_path: str, content: dict, method_name: str = None, new_status: str = None) -> bool:
-    """Update local JSON file and add forum post if status changed"""
+def save_forum_posts(posts, forum_file="submissions/forum_posts.json"):
+    """Save forum posts to local file"""
     try:
-        with open(file_path, 'w') as f:
-            json.dump(content, f, indent=4)
-            
-        # Add forum post if this is a status update
-        if method_name and new_status:
-            forum.add_status_update(method_name, new_status)
-            
-        return True
+        posts_data = [
+            {
+                "message": post.message,
+                "timestamp": post.timestamp,
+                "post_type": post.post_type
+            }
+            for post in posts
+        ]
+        os.makedirs(os.path.dirname(forum_file), exist_ok=True)
+        with open(forum_file, 'w') as f:
+            json.dump(posts_data, f, indent=4)
     except Exception as e:
-        print(f"Error updating {file_path}: {str(e)}")
-        return False
+        print(f"Error saving forum posts: {e}")
+
+def add_status_update(posts, method_name: str, new_status: str):
+    """Add a status update post"""
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    emoji = "✅" if new_status == "approved" else "❌"
+    message = f"{emoji} Status update: {method_name} has been {new_status}"
+    posts.append(ForumPost(message, timestamp, "status_update"))
+    save_forum_posts(posts)
 
 def review_pending_submissions(submissions_dir: str = "submissions"):
     """Review all pending submissions and allow admin to approve/reject them."""
     try:
-        REPO_ID = "snap-stanford/stark-leaderboard" 
-        hub_storage = HubStorage(REPO_ID)
-        forum = SubmissionForum(hub_storage=hub_storage)
+        # Load forum posts
+        forum_posts = load_forum_posts()
+        
         # Find all latest.json files
         latest_files = glob.glob(os.path.join(submissions_dir, "**", "latest.json"), recursive=True)
         
@@ -44,101 +62,101 @@ def review_pending_submissions(submissions_dir: str = "submissions"):
             print("No submissions found.")
             return
         
-        changes_made = False
+        changes_made = []
         
         # Process each submission
         for latest_file in latest_files:
-            folder_path = os.path.dirname(latest_file)
-            latest_info = read_json_file(latest_file)
-            
-            if not latest_info:
-                print(f"Could not read {latest_file}, skipping...")
-                continue
+            try:
+                with open(latest_file, 'r') as f:
+                    latest_info = json.load(f)
                 
-            if latest_info.get('status') != 'pending_review':
-                continue
+                if latest_info.get('status') != 'pending_review':
+                    continue
+                    
+                folder_path = os.path.dirname(latest_file)
+                timestamp = latest_info.get('latest_submission')
+                if not timestamp:
+                    print(f"No timestamp found in {latest_file}, skipping...")
+                    continue
                 
-            # Find corresponding metadata file
-            timestamp = latest_info.get('latest_submission')
-            if not timestamp:
-                print(f"No timestamp found in {latest_file}, skipping...")
-                continue
+                metadata_file = os.path.join(folder_path, f"metadata_{timestamp}.json")
+                try:
+                    with open(metadata_file, 'r') as f:
+                        metadata = json.load(f)
+                except Exception as e:
+                    print(f"Could not read metadata for {latest_file}, skipping...")
+                    continue
                 
-            metadata_file = os.path.join(folder_path, f"metadata_{timestamp}.json")
-            metadata = read_json_file(metadata_file)
-            
-            if not metadata:
-                print(f"Could not read metadata for {latest_file}, skipping...")
-                continue
-            
-            # Display submission details
-            print("\n" + "="*80)
-            print(f"Submission Details:")
-            print(f"Method Name: {metadata.get('Method Name')}")
-            print(f"Team Name: {metadata.get('Team Name')}")
-            print(f"Dataset: {metadata.get('Dataset')}")
-            print(f"Split: {metadata.get('Split')}")
-            print(f"Contact Email(s): {metadata.get('Contact Email(s)')}")
-            print(f"Code Repository: {metadata.get('Code Repository')}")
-            print("\nModel Description:")
-            print(metadata.get('Model Description', 'No description provided'))
-            print("\nResults:")
-            if 'results' in metadata:
-                for metric, value in metadata['results'].items():
-                    print(f"{metric}: {value}")
-            print("\nSubmission Date:", metadata.get('submission_date', 'Unknown'))
-            print("="*80)
-            
-            # Get admin decision
-            while True:
-                decision = input("\nApprove this submission? (y/n/skip/quit): ").lower()
-                if decision in ['y', 'n', 'skip', 'quit']:
+                # Display submission details
+                print("\n" + "="*80)
+                print(f"Submission Details:")
+                print(f"Method Name: {metadata.get('Method Name')}")
+                print(f"Team Name: {metadata.get('Team Name')}")
+                print(f"Dataset: {metadata.get('Dataset')}")
+                print(f"Split: {metadata.get('Split')}")
+                print(f"Contact Email(s): {metadata.get('Contact Email(s)')}")
+                print(f"Code Repository: {metadata.get('Code Repository')}")
+                print("\nModel Description:")
+                print(metadata.get('Model Description', 'No description provided'))
+                print("\nResults:")
+                if 'results' in metadata:
+                    for metric, value in metadata['results'].items():
+                        print(f"{metric}: {value}")
+                print("\nSubmission Date:", metadata.get('submission_date', 'Unknown'))
+                print("="*80)
+                
+                # Get admin decision
+                while True:
+                    decision = input("\nApprove this submission? (y/n/skip/quit): ").lower()
+                    if decision in ['y', 'n', 'skip', 'quit']:
+                        break
+                    print("Invalid input. Please enter 'y', 'n', 'skip', or 'quit'")
+                
+                if decision == 'quit':
+                    print("Exiting review process...")
                     break
-                print("Invalid input. Please enter 'y', 'n', 'skip', or 'quit'")
-            
-            if decision == 'quit':
-                print("Exiting review process...")
-                break
+                    
+                if decision == 'skip':
+                    continue
+                    
+                # Update status
+                new_status = 'approved' if decision == 'y' else 'rejected'
+                review_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                 
-            if decision == 'skip':
-                continue
+                # Update latest.json
+                latest_info['status'] = new_status
+                latest_info['reviewed_at'] = review_timestamp
+                with open(latest_file, 'w') as f:
+                    json.dump(latest_info, f, indent=4)
+                changes_made.append(latest_file)
                 
-            # Update status
-            new_status = 'approved' if decision == 'y' else 'rejected'
-            review_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            
-            # Update latest.json
-            latest_info['status'] = new_status
-            latest_info['reviewed_at'] = review_timestamp
-            if not update_json_file(
-                latest_file, 
-                latest_info, 
-                method_name=metadata.get('Method Name'),
-                new_status=new_status
-            ):
-                print("Failed to update latest.json")
-                continue
-            
-            # Update metadata
-            metadata['status'] = new_status
-            metadata['reviewed_at'] = review_timestamp
-            if not update_json_file(metadata_file, metadata):
-                print("Failed to update metadata file")
+                # Update metadata
+                metadata['status'] = new_status
+                metadata['reviewed_at'] = review_timestamp
+                with open(metadata_file, 'w') as f:
+                    json.dump(metadata, f, indent=4)
+                changes_made.append(metadata_file)
+                
+                # Add forum post
+                add_status_update(forum_posts, metadata.get('Method Name'), new_status)
+                changes_made.append("submissions/forum_posts.json")
+                
+                print(f"Submission {new_status}")
+                
+            except Exception as e:
+                print(f"Error processing {latest_file}: {e}")
                 continue
-            
-            forum.add_status_update(metadata.get('Method Name'), new_status)
-            print(f"Submission {new_status}")
-            changes_made = True
-            
+        
         if changes_made:
-            print("\nChanges have been made to the following files:")
-            print("Please push these changes to HuggingFace manually.")
+            print("\nThe following files have been modified:")
+            for file in sorted(set(changes_made)):
+                print(f"- {file}")
+            print("\nPlease push these changes to HuggingFace manually.")
             
     except Exception as e:
         print(f"Error during review process: {str(e)}")
 
 if __name__ == "__main__":
     print("Starting submission review process...")
-    # You can specify a different submissions directory if needed
-    review_pending_submissions("submissions")
+    review_pending_submissions()
     print("\nReview process completed.")

submissions/abc_a/latest.json

@@ -1,6 +1,7 @@
 {
     "latest_submission": "20241121_015958",
-    "status": "pending_review",
+    "status": "approved",
     "method_name": "abc",
-    "team_name": "a"
+    "team_name": "a",
+    "reviewed_at": "2024-11-20 17:09:16"
 }

submissions/abc_a/metadata_20241121_015958.json

@@ -15,7 +15,8 @@
         "recall@20": 35.95,
         "mrr": 35.94
     },
-    "status": "pending_review",
+    "status": "approved",
     "submission_date": "2024-11-21 02:00:15",
-    "csv_path": "submissions/abc_a/predictions_20241121_015958.csv"
+    "csv_path": "submissions/abc_a/predictions_20241121_015958.csv",
+    "reviewed_at": "2024-11-20 17:09:16"
 }

submissions/debug_test_abc/latest.json

@@ -1,6 +1,7 @@
 {
     "latest_submission": "20241121_011636",
-    "status": "pending_review",
+    "status": "rejected",
     "method_name": "debug_test",
-    "team_name": "abc"
+    "team_name": "abc",
+    "reviewed_at": "2024-11-20 17:09:52"
 }

submissions/debug_test_abc/metadata_20241121_011636.json

@@ -15,7 +15,8 @@
         "recall@20": 35.95,
         "mrr": 35.94
     },
-    "status": "pending_review",
+    "status": "rejected",
     "submission_date": "2024-11-21 01:16:52",
-    "csv_path": "submissions/debug_test_abc/predictions_20241121_011636.csv"
+    "csv_path": "submissions/debug_test_abc/predictions_20241121_011636.csv",
+    "reviewed_at": "2024-11-20 17:09:52"
 }

submissions/forum_posts.json

@@ -8,5 +8,15 @@
         "message": "\ud83d\udce5 New submission: abc on human_generated_eval/mag",
         "timestamp": "2024-11-21 02:00:17",
         "post_type": "submission"
+    },
+    {
+        "message": "\u2705 Status update: abc has been approved",
+        "timestamp": "2024-11-20 17:09:16",
+        "post_type": "status_update"
+    },
+    {
+        "message": "\u274c Status update: debug_test has been rejected",
+        "timestamp": "2024-11-20 17:09:52",
+        "post_type": "status_update"
     }
 ]