Spaces:
Runtime error
Runtime error
| import json | |
| import os | |
| from datetime import datetime, timezone | |
| from src.display.formatting import styled_error, styled_message, styled_warning | |
| from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO | |
| from src.submission.check_validity import already_submitted_models | |
| from src.about import Training_Dataset | |
| REQUESTED_MODELS = None | |
| def add_new_eval( | |
| model_name : str = None, | |
| model_link : str = None, | |
| model_backbone : str = "Unknown", | |
| precision : str = None, | |
| model_parameters: float = 0, | |
| paper_name: str = "None", | |
| paper_link: str = "None", | |
| training_dataset: str = "", | |
| testing_type : str = None, | |
| cmap_value : float = 0, | |
| auroc_value : float = 0, | |
| t1acc_value : float = 0, | |
| ): | |
| global REQUESTED_MODELS | |
| if not REQUESTED_MODELS: | |
| REQUESTED_MODELS = already_submitted_models(EVAL_REQUESTS_PATH) | |
| if model_name is None or model_name == "": | |
| return styled_error("Please enter a model name") | |
| if model_link is None or model_link == "": | |
| return styled_error("Please provide a link to your model") | |
| current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | |
| training_dataset_type = Training_Dataset.from_str(training_dataset) | |
| if training_dataset_type.name != Training_Dataset.Other.name: | |
| training_dataset = training_dataset_type.name | |
| training_dataset = "_".join(training_dataset.split()) | |
| model_name = "_".join(model_name.split()) | |
| testing_type = testing_type.lower() | |
| print("Adding new eval") | |
| eval_entry = { | |
| "model_name": model_name, | |
| "model_link": model_link, | |
| "model_backbone": model_backbone, | |
| "precision": precision, | |
| "model_parameters": model_parameters, | |
| "paper_name": paper_name, | |
| "paper_link": paper_link, | |
| "status": "PENDING", | |
| "submitted_time": current_time, | |
| "training_dataset": training_dataset, | |
| "testing_type": testing_type, | |
| "claimed_cmap": cmap_value, | |
| "claimed_auroc": auroc_value, | |
| "claimed_t1acc": t1acc_value | |
| } | |
| if f"{model_name}_{training_dataset}_{testing_type}_{precision}" in REQUESTED_MODELS: | |
| return styled_warning("This model has been already submitted.") | |
| print("Creating eval file") | |
| try: | |
| OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model_name}" | |
| os.makedirs(OUT_DIR, exist_ok=True) | |
| out_path = f"{OUT_DIR}/{model_name}_eval_request_{precision}_{training_dataset}_{testing_type}.json" | |
| with open(out_path, "w") as f: | |
| f.write(json.dumps(eval_entry)) | |
| except: | |
| return styled_error("There was an error while creating your request. Make sure there are no \"/\" in your model name.") | |
| print("Uploading eval file") | |
| try: | |
| API.upload_file( | |
| path_or_fileobj=out_path, | |
| path_in_repo=out_path.split("eval-queue/")[1], | |
| repo_id=QUEUE_REPO, | |
| repo_type="dataset", | |
| commit_message=f"Add {model_name}_{training_dataset}_{testing_type} to eval queue", | |
| ) | |
| except: | |
| return styled_error("There was an error while uploading your request.") | |
| # Remove the local file | |
| os.remove(out_path) | |
| return styled_message( | |
| "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list." | |
| ) | |