benchmark_catalog.yaml

# OTX Benchmark Dataset Catalog
#
# Each entry declares a dataset used for benchmarking.  The runner will
# execute the preparation script to produce the dataset in the data folder.
#
# Datasets are shared across tasks — each dataset is declared once and
# referenced by name from the benchmark manifest.
#
# All datasets are stored in the experimental Datumaro format.
#
# Fields:
#   name        – unique identifier (referenced from benchmark_manifest.yaml)
#   script      – path to preparation script (relative to library/ root).
#                 Mutually exclusive with `local_path` — exactly one must be set.
#   local_path  – path to an already-prepared dataset directory, used as-is
#                 with no script execution. Supports ${VAR}/~ expansion so the
#                 same entry resolves differently per machine/CI runner (e.g.
#                 "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/my_dataset"). Use this
#                 for datasets that were prepared out-of-band (manually, or on
#                 a shared/network mount) and should never be re-downloaded.
#   raw_dir     – optional, only valid together with `script`. Path (also
#                 supports ${VAR}/~ expansion) to a pre-fetched raw archive or
#                 directory, forwarded to the script as `--raw-dir` so it can
#                 skip its own network download step (e.g. for credentialed
#                 sources like Kaggle) while still running the same
#                 transform/export logic. If the path doesn't exist, the
#                 script falls back to its normal download behavior.
#   size_tier   – tiny | small | medium | large
#                 rough dataset size.
#   data_group  – weekly | extended | all (default: all, may be omitted)
#                 `weekly` datasets are only included in weekly-benchmark runs;
#                 `extended` datasets are only included in extended/full runs;
#                 `all` (the default) is included in both.
#   url         – upstream source the preparation script downloads from
#
# See src/getitune/benchmark/README.md ("Datasets requiring credentials or
# manual placement") for a walkthrough of `local_path` / `raw_dir` usage.

version: 1

datasets:
  - name: wgisd
    script: "scripts/benchmark_datasets/prepare_wgisd.py"
    size_tier: small
    data_group: weekly
    url: "https://github.com/thsant/wgisd"
    description: "Wine Grape Instance Segmentation Dataset — 137 images of grape clusters with COCO-style polygon annotations (5 varieties)."
    compatible_tasks:
      - instance_segmentation
      - detection # polygons → bounding boxes
      - rotated_detection # polygons → oriented bounding boxes
      - semantic_segmentation # polygons → pixel masks
      - classification/multi_label_cls # crop-level or image-level class from category

  - name: flowers102
    script: "scripts/benchmark_datasets/prepare_flowers102.py"
    size_tier: large
    data_group: extended
    url: "https://www.robots.ox.ac.uk/~vgg/data/flowers/102/"
    description: "Oxford Flowers 102 — 8,189 images across 102 flower categories with official train/val/test splits."
    compatible_tasks:
      - classification/multi_class_cls

  - name: paddy_disease
    script: "scripts/benchmark_datasets/prepare_paddy_disease.py"
    size_tier: small
    data_group: weekly
    url: "https://huggingface.co/datasets/anthony2261/paddy-disease-classification"
    description: "Paddy Disease Classification — public HF mirror of a rice disease dataset with image-level class labels."
    compatible_tasks:
      - classification/multi_class_cls

  - name: chest_ct
    script: "scripts/benchmark_datasets/prepare_chest_ct.py"
    size_tier: large
    data_group: extended
    url: "https://github.com/shreyabandyopadhyay/CTScanImageClassification/blob/main/DS3.zip"
    description: "Chest CT Scan Images — public 3-class (abdomen/chest/head) body-part CT classification dataset."
    compatible_tasks:
      - classification/multi_class_cls

  - name: aircraft
    script: "scripts/benchmark_datasets/prepare_aircraft.py"
    raw_dir: "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/aircraft_raw"
    size_tier: large
    data_group: weekly
    url: "https://www.kaggle.com/datasets/seryouxblaster764/fgvc-aircraft"
    description: >-
      FGVC-Aircraft — 10,200 aircraft images with official train/val/test splits and
      hierarchical labels (variant / family / manufacturer). This benchmark uses the
      variant level and requires a Kaggle account for automatic download, or a
      pre-extracted raw directory at $GETITUNE_BENCHMARK_EXTERNAL_DATA/aircraft_raw.
    compatible_tasks:
      - classification/multi_class_cls

  - name: food41
    script: "scripts/benchmark_datasets/prepare_food41.py"
    raw_dir: "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/food41_raw"
    size_tier: large
    data_group: extended
    url: "https://www.kaggle.com/datasets/kmader/food41"
    description: >-
      Food-101 — 101 food categories with official train/test splits, sourced from
      a Kaggle mirror. This benchmark uses a stratified validation split from the
      training set and supports Kaggle auto-download or a pre-extracted raw
      directory at $GETITUNE_BENCHMARK_EXTERNAL_DATA/food41_raw.
    compatible_tasks:
      - classification/multi_class_cls

  - name: oxford_pets_hlabel
    script: "scripts/benchmark_datasets/prepare_oxford_pets_hlabel.py"
    size_tier: large
    data_group: extended
    url: "https://www.robots.ox.ac.uk/~vgg/data/pets/"
    description: "Oxford-IIIT Pet with hierarchical labels — 7,349 images with natural 2-level hierarchy: species (cat/dog) → 37 breeds."
    compatible_tasks:
      - classification/h_label_cls

  - name: oxford_pets
    script: "scripts/benchmark_datasets/prepare_oxford_pets.py"
    size_tier: large
    data_group: extended
    url: "https://www.robots.ox.ac.uk/~vgg/data/pets/"
    description: "Oxford-IIIT Pet — 7,349 images of 37 pet breeds with pixel-level trimap segmentation masks."
    compatible_tasks:
      - semantic_segmentation
      - instance_segmentation # trimap foreground → single instance polygon
      - detection # foreground mask → bounding box

  - name: bccd
    script: "scripts/benchmark_datasets/prepare_bccd.py"
    size_tier: small
    data_group: weekly
    url: "https://github.com/Shenggan/BCCD_Dataset"
    description: "Blood Cell Count and Detection — 364 microscopy blood-smear images with Pascal VOC bounding-box annotations for 3 cell types (RBC, WBC, Platelets)."
    compatible_tasks:
      - detection

  - name: aid_multilabel
    script: "scripts/benchmark_datasets/prepare_aid_multilabel.py"
    size_tier: medium
    data_group: weekly
    url: "https://huggingface.co/datasets/jonathan-roberts1/AID_MultiLabel"
    description: "AID Multi-Label — 3,000 overhead aerial-scene images (600×600) with multi-label annotations across 17 co-occurring land-cover classes."
    compatible_tasks:
      - classification/multi_label_cls

  - name: axial_mri
    script: "scripts/benchmark_datasets/prepare_axial_mri.py"
    size_tier: small
    data_group: extended
    url: "https://huggingface.co/datasets/Francesco/axial-mri"
    description: "Axial MRI (Roboflow-100) — 371 brain axial-MRI slices with tumour bounding-box annotations (negative/positive). Tiny, privacy-safe medical detection benchmark."
    compatible_tasks:
      - detection

  - name: cable_damage
    script: "scripts/benchmark_datasets/prepare_cable_damage.py"
    size_tier: medium
    data_group: extended
    url: "https://huggingface.co/datasets/Francesco/cable-damage"
    description: "Cable Damage (Roboflow-100) — 1,318 close-up cable images with damage bounding-box annotations (break/thunderbolt). Industrial damage-inspection detection benchmark."
    compatible_tasks:
      - detection

  - name: vehicles
    script: "scripts/benchmark_datasets/prepare_vehicles.py"
    size_tier: large
    data_group: extended
    url: "https://huggingface.co/datasets/Francesco/vehicles-q0x2v"
    description: "Vehicles (Roboflow-100) — 4,058 road-scene images with bounding-box annotations across 12 vehicle types (cars, buses, trucks). Large multi-class on-road detection benchmark."
    compatible_tasks:
      - detection

  - name: pothole
    script: "scripts/benchmark_datasets/prepare_pothole.py"
    size_tier: tiny
    data_group: weekly
    url: "https://huggingface.co/datasets/keremberke/pothole-segmentation"
    description: "Pothole Segmentation — 90 road-surface images with polygon annotations for potholes. Tiny real-world segmentation benchmark."
    compatible_tasks:
      - instance_segmentation
      - semantic_segmentation
      - detection # polygons -> bounding boxes

  - name: pcb_defect
    script: "scripts/benchmark_datasets/prepare_pcb_defect.py"
    size_tier: small
    data_group: weekly
    url: "https://huggingface.co/datasets/keremberke/pcb-defect-segmentation"
    description: "PCB Defect Segmentation — 189 close-up PCB images with polygon annotations across 4 defect classes. Small industrial segmentation benchmark."
    compatible_tasks:
      - instance_segmentation
      - semantic_segmentation
      - detection # polygons -> bounding boxes

  - name: satellite_building
    script: "scripts/benchmark_datasets/prepare_satellite_building.py"
    size_tier: large
    data_group: extended
    url: "https://huggingface.co/datasets/keremberke/satellite-building-segmentation"
    description: "Satellite Building Segmentation — 9,665 aerial images with polygon annotations for building footprints. Medium-scale segmentation benchmark."
    compatible_tasks:
      - instance_segmentation
      - semantic_segmentation
      - detection # polygons -> bounding boxes

  - name: corrosion
    script: "scripts/benchmark_datasets/prepare_corrosion.py"
    size_tier: tiny
    data_group: weekly
    url: "https://huggingface.co/datasets/rkumari/corrosion_segment"
    description: "Corrosion segmentation — 44 images with per-pixel severity masks across 4 classes. Tiny semantic-segmentation benchmark."
    compatible_tasks:
      - semantic_segmentation

  - name: crack_semantic
    script: "scripts/benchmark_datasets/prepare_crack_semantic.py"
    size_tier: large
    data_group: weekly
    url: "https://huggingface.co/datasets/varcoder/crack-segmentation-dataset"
    description: "Crack segmentation — 11,298 images with binary crack masks. Large semantic-segmentation benchmark."
    compatible_tasks:
      - semantic_segmentation

  - name: brain_tumor
    script: "scripts/benchmark_datasets/prepare_brain_tumor.py"
    raw_dir: "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/brain_tumor_raw"
    size_tier: large
    data_group: extended
    url: "https://www.kaggle.com/datasets/pkdarabi/medical-image-dataset-brain-tumor-segmentation"
    description: >-
      Brain Tumor instance segmentation — ~1,744 MRI images with polygon annotations
      (3 tumor-likelihood classes) from a Kaggle-gated Roboflow YOLO export. Requires a
      Kaggle account: either configure Kaggle API credentials (see the preparation
      script's docstring) for automatic download, or place a pre-fetched copy at
      $GETITUNE_BENCHMARK_EXTERNAL_DATA/brain_tumor_raw (raw_dir is forwarded to
      the script and used instead of downloading when present).
    compatible_tasks:
      - instance_segmentation
      - detection # polygons → bounding boxes
      - semantic_segmentation # polygons → pixel masks

  - name: cardd
    script: "scripts/benchmark_datasets/prepare_cardd.py"
    raw_dir: "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/cardd_instseg_raw"
    size_tier: large
    data_group: extended
    url: "https://www.kaggle.com/datasets/issamjebnouni/cardd"
    description: >-
      CarDD instance segmentation — 4,000 images with COCO polygon annotations across 6
      damage classes (dent, scratch, crack, glass shatter, lamp broken, tire flat), from
      a Kaggle-gated COCO-format mirror. Requires a Kaggle account: either configure
      Kaggle API credentials (see the preparation script's docstring) for automatic
      download, or place a pre-fetched copy at $GETITUNE_BENCHMARK_EXTERNAL_DATA/cardd_instseg_raw
      (raw_dir is forwarded to the script and used instead of downloading when present).
    compatible_tasks:
      - instance_segmentation
      - detection # polygons → bounding boxes
      - semantic_segmentation # polygons → pixel masks