leaderboard

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+.env
+__pycache__/
+requested_models/
+repos/
+TODO

README.md

@@ -0,0 +1,12 @@
+---
+title: Open Object Detection Leaderboard
+emoji: 🏆
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.38.0
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,142 @@
+import gradio as gr
+import pandas as pd
+import json
+from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS
+from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
+from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
+from datetime import datetime, timezone
+
+column_names = {"AP-IoU=0.50:0.95-area=all-maxDets=100": "AP",
+                 "AP-IoU=0.50-area=all-maxDets=100": "AP@.50",
+                 "AP-IoU=0.75-area=all-maxDets=100": "AP@.75",
+                 "AP-IoU=0.50:0.95-area=small-maxDets=100" : "AP-S",
+                 "AP-IoU=0.50:0.95-area=medium-maxDets=100": "AP-M",
+                 "AP-IoU=0.50:0.95-area=large-maxDets=100": "AP-L",
+                 "AR-IoU=0.50:0.95-area=all-maxDets=1": "AR1",
+                 "AR-IoU=0.50:0.95-area=all-maxDets=10": "AR10",
+                 "AR-IoU=0.50:0.95-area=all-maxDets=100": "AR100",
+                 "AR-IoU=0.50:0.95-area=small-maxDets=100": "AR-S",
+                 "AR-IoU=0.50:0.95-area=medium-maxDets=100": "AR-M",
+                 "AR-IoU=0.50:0.95-area=large-maxDets=100": "AR-L"}
+
+eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub()
+
+if not csv_results.exists():
+    raise Exception(f"CSV file {csv_results} does not exist locally")
+    
+# Get csv with data and parse columns
+original_df = pd.read_csv(csv_results)
+
+# Formats the columns
+def formatter(x):
+    x = "{:.2%}".format(x)
+    while len(x) < 6:
+        x = f"0{x}"
+    return x
+
+for col in original_df.columns:
+    if col == "model":
+        original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x)))
+    else:
+        original_df[col] = original_df[col].apply(formatter) # For % values
+        # original_df[col] = original_df[col].multiply(100).round(2) 
+ 
+original_df.rename(columns=column_names, inplace=True)
+
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
+
+
+def request_model(model_text, chbcoco2017):
+    
+    # Determine the selected checkboxes
+    dataset_selection = []
+    if chbcoco2017:
+        dataset_selection.append("COCO validation 2017 dataset")
+
+    if len(dataset_selection) == 0:
+        return styled_error("You need to select at least one dataset")
+        
+    base_model_on_hub, error_msg = is_model_on_hub(model_text)
+
+    if not base_model_on_hub:
+        return styled_error(f"Base model '{model_text}' {error_msg}")
+    
+    # Construct the output dictionary
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    required_datasets = ', '.join(dataset_selection)
+    eval_entry = {
+        "date": current_time,
+        "model": model_text,
+        "datasets_selected": required_datasets
+    }
+    
+    # Prepare file path 
+    DIR_OUTPUT_REQUESTS.mkdir(parents=True, exist_ok=True)
+    
+    fn_datasets = '@ '.join(dataset_selection)
+    filename = model_text.replace("/","@") + "@@" + fn_datasets 
+    if filename in requested_models:
+        return styled_error(f"A request for this model '{model_text}' and dataset(s) was already made.")
+    try:
+        filename_ext = filename + ".txt"
+        out_filepath = DIR_OUTPUT_REQUESTS / filename_ext
+
+        # Write the results to a text file
+        with open(out_filepath, "w") as f:
+            f.write(json.dumps(eval_entry))
+            
+        upload_file(filename, out_filepath)
+        
+        # Include file in the list of uploaded files
+        requested_models.append(filename)
+        
+        # Remove the local file
+        out_filepath.unlink()
+
+        return styled_message("🤗 Your request has been submitted and will be evaluated soon!</p>")
+    except Exception as e:
+        return styled_error(f"Error submitting request!")
+
+with gr.Blocks() as demo:
+    gr.HTML(BANNER, elem_id="banner")
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 COCO val 2017", elem_id="od-benchmark-tab-table", id=0):
+            leaderboard_table = gr.components.Dataframe(
+                value=original_df,
+                datatype=TYPES,
+                max_rows=None,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+                )
+
+        with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=1):
+            gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table", id=2):
+            with gr.Column():
+                gr.Markdown("# ✉️✨ Request results for a new model here!", elem_classes="markdown-text")
+            with gr.Column():
+                gr.Markdown("Select a dataset:", elem_classes="markdown-text")
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name (user_name/model_name)")
+                    chb_coco2017 = gr.Checkbox(label="COCO validation 2017 dataset", visible=False, value=True, interactive=False)
+                with gr.Column():
+                    mdw_submission_result = gr.Markdown()
+                    btn_submitt = gr.Button(value="🚀 Request")
+                    btn_submitt.click(request_model, 
+                                      [model_name_textbox, chb_coco2017], 
+                                      mdw_submission_result)
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            gr.Textbox(
+                value=CITATION_TEXT, lines=7,
+                label="Copy the BibTeX snippet to cite this source",
+                elem_id="citation-button",
+            ).style(show_copy_button=True)
+
+demo.launch(debug=True)

constants.py

@@ -0,0 +1,89 @@
+
+from pathlib import Path
+
+# Directory where request by models are stored
+DIR_OUTPUT_REQUESTS = Path("requested_models")
+EVAL_REQUESTS_PATH = Path("eval_requests")
+
+##########################
+# Text definitions       #
+##########################
+
+banner_url = "https://huggingface.co/spaces/rafaelpadilla/object_detection_leaderboard/resolve/main/assets/leaderboard_small.png"
+BANNER = f'<div style="display: flex; justify-content: space-around;"><img src="{banner_url}" alt="Banner" style="width: 40vw; min-width: 300px; max-width: 600px;"> </div>'
+
+TITLE = "<html> <head> <style> h1 {text-align: center;} </style> </head> <body> <h1> 🤗 Open Object Detection Leaderboard </b> </body> </html>"
+
+INTRODUCTION_TEXT = "📐 The 🤗 Open Object Detection Leaderboard aims to track, rank and evaluate vision models \
+    available in the hub designed to detect objects in images. \
+    Anyone from the community can request a model to be evaluated and added to the leaderboard. \
+    \nCheck the 📈 Metrics tab to understand how the models are evaluated. \
+    \nIf you want results for a model that is not listed here, you can  ✉️✨ request results for it."
+
+CITATION_TEXT = '''@misc{open-od-leaderboard,
+  author = {Rafael Padilla, Amy Roberts and the Hugging Face Team},
+  title = {Open Object Detection Leaderboard},
+  year = {2023},
+  publisher = {Hugging Face},
+  howpublished = "\\url{https://huggingface.co/spaces/rafaelpadilla/object_detection_leaderboard}"
+}
+'''
+
+METRICS_TAB_TEXT = '''
+🎯 Average Precision and Average Recall  are popular metrics for evaluating the accuracy of object detectors by estimating the Precision-Recall relationship.
+
+Here you will find details about the object detection metrics reported in our leaderboard.
+
+# Metrics
+
+There are plenty of variations of these metrics, depending on the IoU threshold, the area of the object, and the number of detections per image. The most popular ones are:
+
+## Average Precision (AP) 
+- **AP**: AP at IoU=.50:.05:.95
+- **AP@.50 (APIoU=.50)**: AP at IoU=.50 (similar to mAP PASCAL VOC metric)
+- **AP@.75 (APIoU=.75)**: AP at IoU=.75 (strict metric)
+
+## Average Precision Across Scales 
+- **AP-S (APsmall)**: AP for small objects: area < 322.
+- **AP-M (APmedium)**: AP for medium objects: 322 < area < 962.
+- **AP-L (APlarge)**: AP for large objects: area > 962.
+
+## Average Recall (AR) 
+- **AR1 (ARmax=1)**: AR given 1 detection per image.
+- **AR10 (ARmax=10)**: AR given 10 detections per image.
+- **AR100 (ARmax=100)**: AR given 100 detections per image.
+
+## Average Recall Across Scales 
+- **AR-S (ARsmall)**: AR for small objects: area < 322.
+- **AR-M (ARmedium)**: AR for medium objects: 322 < area < 962.
+- **AR-L (ARlarge)**: AR for large objects: area > 962.
+
+## How to reproduce our results
+
+To compute these metrics, various tools employ different methods. For this leaderboard's evaluation, we utilize the COCO evaluation approach, which can be found in the [COCO evaluation toolkit](https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py).
+
+The 🤗 `Evaluate` metric used to measure the results is also accessible in the hub: [detection_metrics](https://huggingface.co/spaces/rafaelpadilla/detection_metrics)
+
+It is essential to note that slight differences in results may arise between the results presented here and those from other sources. These differences can be attributed to numerical approximations, variations in batch sizes, and other hyperparameters. To ensure a consistent evaluation, we recommend using a batch size of 1 and setting the confidence threshold to 0 when evaluating your model.
+
+## Benchmark datasets
+
+We understand that the object detection task can be quite diverse and the requirements can vary greatly across different domains such as autonomous driving, medical imaging, aerial imaging, etc. 
+
+With this in mind, we are interested in knowing if there are specific domains or applications that you believe should be addressed by our benchmarks. Please, [join our discussion](https://huggingface.co/spaces/rafaelpadilla/object_detection_leaderboard/discussions/1) and give your suggestion. 
+
+### COCO dataset
+
+The Microsoft Common Objects in Context (COCO) dataset is a highly regarded benchmark for object detection models due to its comprehensive set of 80 object categories, extensive volume of images with complex scenes, and high-quality, manually annotated labels. 
+
+Moreover, its versatility in supporting multiple computer vision tasks, along with a standardized format and an active community, makes it a robust, challenging, and easily comparable benchmark. 
+
+The benchmarking COCO validation 2017 dataset is available in the 🤗 hub: [coco2017](https://huggingface.co/datasets/rafaelpadilla/coco2017) 
+
+## 📚 Useful Readings 
+
+For further insight into the subject, you may find the following readings helpful:
+
+- [A Survey on Performance Metrics for Object-Detection Algorithms](https://www.researchgate.net/profile/Rafael-Padilla/publication/343194514_A_Survey_on_Performance_Metrics_for_Object-Detection_Algorithms/links/5f1b5a5e45851515ef478268/A-Survey-on-Performance-Metrics-for-Object-Detection-Algorithms.pdf), R Padilla, SL Netto, EAB Da Silva - *IWSSIP, 2020* 
+- [A Comparative Analysis of Object Detection Metrics with a Companion Open-Source Toolkit](https://www.mdpi.com/2079-9292/10/3/279/pdf), R Padilla, WL Passos, TLB Dias, SL Netto, EAB Da Silva - *Journal Electronics, 2021*
+'''

init.py

@@ -0,0 +1,93 @@
+import os 
+from constants import EVAL_REQUESTS_PATH
+from pathlib import Path
+from huggingface_hub import HfApi, Repository
+
+TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
+QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
+QUEUE_PATH = os.environ.get("QUEUE_PATH", None)
+
+hf_api = HfApi(
+    endpoint="https://huggingface.co", 
+    token=TOKEN_HUB, 
+)
+
+def load_all_info_from_dataset_hub():
+    eval_queue_repo = None
+    results_csv_path = None
+    requested_models = None
+
+    passed = True
+    if TOKEN_HUB is None:
+        passed = False
+    else:
+        print("Pulling evaluation requests and results.")
+
+        eval_queue_repo = Repository(
+            local_dir=QUEUE_PATH,
+            clone_from=QUEUE_REPO,
+            use_auth_token=TOKEN_HUB,
+            repo_type="dataset",
+        )
+        eval_queue_repo.git_pull()
+        
+        # Local directory where dataset repo is cloned + folder with eval requests
+        directory = QUEUE_PATH / EVAL_REQUESTS_PATH
+        requested_models = get_all_requested_models(directory)
+        requested_models = [p.stem for p in requested_models]
+        # Local directory where dataset repo is cloned
+        csv_results = get_csv_with_results(QUEUE_PATH)
+        if csv_results is None:
+            passed = False
+    if not passed:
+        print("No HuggingFace token provided. Skipping evaluation requests and results.")
+
+    return eval_queue_repo, requested_models, csv_results
+
+
+def upload_file(requested_model_name, path_or_fileobj):
+    dest_repo_file = Path(EVAL_REQUESTS_PATH) / path_or_fileobj.name
+    dest_repo_file = str(dest_repo_file)
+    hf_api.upload_file(
+            path_or_fileobj=path_or_fileobj,
+            path_in_repo=str(dest_repo_file),
+            repo_id=QUEUE_REPO,
+            token=TOKEN_HUB,
+            repo_type="dataset",
+            commit_message=f"Add {requested_model_name} to eval queue")
+
+def get_all_requested_models(directory):
+    directory = Path(directory)
+    all_requested_models = list(directory.glob("*.txt"))
+    return all_requested_models
+
+def get_csv_with_results(directory):
+    directory = Path(directory)
+    all_csv_files = list(directory.glob("*.csv"))
+    latest = [f for f in all_csv_files if f.stem.endswith("latest")]
+    if len(latest) != 1:
+        return None
+    return latest[0]
+
+
+
+def is_model_on_hub(model_name, revision="main") -> bool:
+    try:
+        model_name = model_name.replace(" ","")
+        author = model_name.split("/")[0]
+        model_id = model_name.split("/")[1]
+        if len(author) == 0 or len(model_id) == 0:
+            return False, "is not a valid model name. Please use the format `author/model_name`."
+    except Exception as e:
+        return False, "is not a valid model name. Please use the format `author/model_name`."
+
+    try:
+        models = list(hf_api.list_models(author=author, search=model_id))
+        matched = [model_name for m in models if m.modelId == model_name]
+        if len(matched) != 1:
+            return False, "was not found on the hub!"
+        else:
+            return True, None
+    except Exception as e:
+        print(f"Could not get the model from the hub.: {e}")
+        return False, "was not found on hub!"

requirements.txt

@@ -0,0 +1,61 @@
+aiohttp==3.8.4
+aiosignal==1.3.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2023.7.22
+charset-normalizer==3.2.0
+cmake==3.26.4
+contourpy==1.1.0
+Cython==3.0.0
+datasets==2.13.1
+dill==0.3.6
+filelock==3.12.2
+fonttools==4.40.0
+frozenlist==1.4.0
+fsspec==2023.6.0
+huggingface-hub==0.16.4
+idna==3.4
+Jinja2==3.1.2
+kiwisolver==1.4.4
+lit==16.0.6
+MarkupSafe==2.1.3
+matplotlib==3.7.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+networkx==3.1
+numpy==1.25.2
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+packaging==23.1
+pandas==2.0.3
+Pillow==10.0.0
+pyarrow==12.0.1
+python-dateutil==2.8.2
+pytz==2023.3
+PyYAML==6.0.1
+regex==2023.6.3
+requests==2.31.0
+responses==0.18.0
+safetensors==0.3.1
+six==1.16.0
+sympy==1.12
+tokenizers==0.13.3
+torch==2.0.1
+torchvision==0.15.2
+tqdm==4.65.0
+triton==2.0.0
+typing_extensions==4.7.1
+tzdata==2023.3
+urllib3==2.0.4
+xxhash==3.2.0
+yarl==1.9.2

utils_display.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+
+# These classes are for user facing column names, to avoid having to change them 
+# all around the code when a modif is needed 
+@dataclass
+class ColumnContent:
+    name: str
+    type: str 
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+@dataclass(frozen=True)
+class AutoEvalColumn: # Auto evals column
+    model = ColumnContent("Model", "markdown")
+    ap = ColumnContent("AP", "str")
+    ap50 = ColumnContent("AP@.50", "number")
+    ap75 = ColumnContent("AP@.75 ⬆️", "number")
+    ap_small = ColumnContent("APₛ", "number")
+    ap_medium = ColumnContent("APₘ", "number")
+    ap_large = ColumnContent("APₗ", "number")
+    ar_1 = ColumnContent("AR₁", "number")
+    ar_10 = ColumnContent("AR₁₀", "number")
+    ar_100 = ColumnContent("AR₁₀₀", "number")
+    ar_small = ColumnContent("ARₛ", "number")
+    ar_medium = ColumnContent("ARₘ", "number")
+    ar_large = ColumnContent("ARₗ", "number")
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"