MedMNIST_2D_3D.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f1eae39e-4fb0-41e0-83cb-862ac208f657",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BMEhw2.ipynb  experiments  MedMNIST2D  ondemand  Untitled.ipynb\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "773c3cfa-071f-4a43-9234-b19f1628180c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the device you want to use\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b8a6ddb5-2c18-457c-b14f-6fa26d3fa941",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c827f740-5168-44e8-986d-f33fca2d26fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting git+https://github.com/M3DV/ACSConv.git\n",
      "  Cloning https://github.com/M3DV/ACSConv.git to /tmp/pip-req-build-kmv24qus\n",
      "  Running command git clone --filter=blob:none --quiet https://github.com/M3DV/ACSConv.git /tmp/pip-req-build-kmv24qus\n",
      "  Resolved https://github.com/M3DV/ACSConv.git to commit 95dc860a77e309f010a3d8be1f675e77c7dfeda4\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: fire in ./.local/lib/python3.9/site-packages (from ACSConv==0.1.1) (0.5.0)\n",
      "Requirement already satisfied: numpy in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from ACSConv==0.1.1) (1.22.4)\n",
      "Requirement already satisfied: matplotlib in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from ACSConv==0.1.1) (3.5.2)\n",
      "Requirement already satisfied: pandas in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from ACSConv==0.1.1) (1.4.2)\n",
      "Requirement already satisfied: tqdm in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from ACSConv==0.1.1) (4.63.0)\n",
      "Requirement already satisfied: scikit-image in ./.local/lib/python3.9/site-packages (from ACSConv==0.1.1) (0.21.0)\n",
      "Requirement already satisfied: scikit-learn in ./.local/lib/python3.9/site-packages (from ACSConv==0.1.1) (1.3.0)\n",
      "Requirement already satisfied: scipy in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from ACSConv==0.1.1) (1.8.1)\n",
      "Requirement already satisfied: tensorboardx in ./.local/lib/python3.9/site-packages (from ACSConv==0.1.1) (2.6.2.2)\n",
      "Requirement already satisfied: torch in ./.local/lib/python3.9/site-packages (from ACSConv==0.1.1) (2.0.1)\n",
      "Requirement already satisfied: torchvision in ./.local/lib/python3.9/site-packages (from ACSConv==0.1.1) (0.15.2)\n",
      "Requirement already satisfied: six in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from fire->ACSConv==0.1.1) (1.16.0)\n",
      "Requirement already satisfied: termcolor in ./.local/lib/python3.9/site-packages (from fire->ACSConv==0.1.1) (2.3.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from matplotlib->ACSConv==0.1.1) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from matplotlib->ACSConv==0.1.1) (4.33.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from matplotlib->ACSConv==0.1.1) (1.4.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from matplotlib->ACSConv==0.1.1) (21.3)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from matplotlib->ACSConv==0.1.1) (9.1.1)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from matplotlib->ACSConv==0.1.1) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from matplotlib->ACSConv==0.1.1) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from pandas->ACSConv==0.1.1) (2022.1)\n",
      "Requirement already satisfied: networkx>=2.8 in ./.local/lib/python3.9/site-packages (from scikit-image->ACSConv==0.1.1) (3.1)\n",
      "Requirement already satisfied: imageio>=2.27 in ./.local/lib/python3.9/site-packages (from scikit-image->ACSConv==0.1.1) (2.31.2)\n",
      "Requirement already satisfied: tifffile>=2022.8.12 in ./.local/lib/python3.9/site-packages (from scikit-image->ACSConv==0.1.1) (2023.8.25)\n",
      "Requirement already satisfied: PyWavelets>=1.1.1 in ./.local/lib/python3.9/site-packages (from scikit-image->ACSConv==0.1.1) (1.4.1)\n",
      "Requirement already satisfied: lazy_loader>=0.2 in ./.local/lib/python3.9/site-packages (from scikit-image->ACSConv==0.1.1) (0.3)\n",
      "Requirement already satisfied: joblib>=1.1.1 in ./.local/lib/python3.9/site-packages (from scikit-learn->ACSConv==0.1.1) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in ./.local/lib/python3.9/site-packages (from scikit-learn->ACSConv==0.1.1) (3.2.0)\n",
      "Requirement already satisfied: protobuf>=3.20 in ./.local/lib/python3.9/site-packages (from tensorboardx->ACSConv==0.1.1) (4.24.2)\n",
      "Requirement already satisfied: filelock in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (3.12.3)\n",
      "Requirement already satisfied: typing-extensions in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (4.7.1)\n",
      "Requirement already satisfied: sympy in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (1.12)\n",
      "Requirement already satisfied: jinja2 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (11.7.101)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (8.5.0.96)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (11.10.3.66)\n",
      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (10.9.0.58)\n",
      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (10.2.10.91)\n",
      "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (11.4.0.1)\n",
      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (11.7.4.91)\n",
      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (2.14.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (11.7.91)\n",
      "Requirement already satisfied: triton==2.0.0 in ./.local/lib/python3.9/site-packages (from torch->ACSConv==0.1.1) (2.0.0)\n",
      "Requirement already satisfied: setuptools in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch->ACSConv==0.1.1) (61.2.0)\n",
      "Requirement already satisfied: wheel in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch->ACSConv==0.1.1) (0.37.1)\n",
      "Requirement already satisfied: cmake in ./.local/lib/python3.9/site-packages (from triton==2.0.0->torch->ACSConv==0.1.1) (3.27.2)\n",
      "Requirement already satisfied: lit in ./.local/lib/python3.9/site-packages (from triton==2.0.0->torch->ACSConv==0.1.1) (16.0.6)\n",
      "Requirement already satisfied: requests in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from torchvision->ACSConv==0.1.1) (2.27.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from jinja2->torch->ACSConv==0.1.1) (2.1.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from requests->torchvision->ACSConv==0.1.1) (1.26.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from requests->torchvision->ACSConv==0.1.1) (2022.5.18.1)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from requests->torchvision->ACSConv==0.1.1) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from requests->torchvision->ACSConv==0.1.1) (3.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in ./.local/lib/python3.9/site-packages (from sympy->torch->ACSConv==0.1.1) (1.3.0)\n"
     ]
    }
   ],
   "source": [
    "# Install relevant dependencies\n",
    "!pip install git+https://github.com/M3DV/ACSConv.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c3d45af5-07b8-4009-9645-62dc7eb0720f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: medmnist in ./.local/lib/python3.9/site-packages (2.2.3)\n",
      "Requirement already satisfied: numpy in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from medmnist) (1.22.4)\n",
      "Requirement already satisfied: pandas in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from medmnist) (1.4.2)\n",
      "Requirement already satisfied: scikit-learn in ./.local/lib/python3.9/site-packages (from medmnist) (1.3.0)\n",
      "Requirement already satisfied: scikit-image in ./.local/lib/python3.9/site-packages (from medmnist) (0.21.0)\n",
      "Requirement already satisfied: tqdm in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from medmnist) (4.63.0)\n",
      "Requirement already satisfied: Pillow in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from medmnist) (9.1.1)\n",
      "Requirement already satisfied: fire in ./.local/lib/python3.9/site-packages (from medmnist) (0.5.0)\n",
      "Requirement already satisfied: torch in ./.local/lib/python3.9/site-packages (from medmnist) (2.0.1)\n",
      "Requirement already satisfied: torchvision in ./.local/lib/python3.9/site-packages (from medmnist) (0.15.2)\n",
      "Requirement already satisfied: six in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from fire->medmnist) (1.16.0)\n",
      "Requirement already satisfied: termcolor in ./.local/lib/python3.9/site-packages (from fire->medmnist) (2.3.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from pandas->medmnist) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from pandas->medmnist) (2022.1)\n",
      "Requirement already satisfied: scipy>=1.8 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from scikit-image->medmnist) (1.8.1)\n",
      "Requirement already satisfied: networkx>=2.8 in ./.local/lib/python3.9/site-packages (from scikit-image->medmnist) (3.1)\n",
      "Requirement already satisfied: imageio>=2.27 in ./.local/lib/python3.9/site-packages (from scikit-image->medmnist) (2.31.2)\n",
      "Requirement already satisfied: tifffile>=2022.8.12 in ./.local/lib/python3.9/site-packages (from scikit-image->medmnist) (2023.8.25)\n",
      "Requirement already satisfied: PyWavelets>=1.1.1 in ./.local/lib/python3.9/site-packages (from scikit-image->medmnist) (1.4.1)\n",
      "Requirement already satisfied: packaging>=21 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from scikit-image->medmnist) (21.3)\n",
      "Requirement already satisfied: lazy_loader>=0.2 in ./.local/lib/python3.9/site-packages (from scikit-image->medmnist) (0.3)\n",
      "Requirement already satisfied: joblib>=1.1.1 in ./.local/lib/python3.9/site-packages (from scikit-learn->medmnist) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in ./.local/lib/python3.9/site-packages (from scikit-learn->medmnist) (3.2.0)\n",
      "Requirement already satisfied: filelock in ./.local/lib/python3.9/site-packages (from torch->medmnist) (3.12.3)\n",
      "Requirement already satisfied: typing-extensions in ./.local/lib/python3.9/site-packages (from torch->medmnist) (4.7.1)\n",
      "Requirement already satisfied: sympy in ./.local/lib/python3.9/site-packages (from torch->medmnist) (1.12)\n",
      "Requirement already satisfied: jinja2 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from torch->medmnist) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (11.7.101)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (8.5.0.96)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (11.10.3.66)\n",
      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (10.9.0.58)\n",
      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (10.2.10.91)\n",
      "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (11.4.0.1)\n",
      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (11.7.4.91)\n",
      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (2.14.3)\n",
      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (11.7.91)\n",
      "Requirement already satisfied: triton==2.0.0 in ./.local/lib/python3.9/site-packages (from torch->medmnist) (2.0.0)\n",
      "Requirement already satisfied: setuptools in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch->medmnist) (61.2.0)\n",
      "Requirement already satisfied: wheel in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch->medmnist) (0.37.1)\n",
      "Requirement already satisfied: cmake in ./.local/lib/python3.9/site-packages (from triton==2.0.0->torch->medmnist) (3.27.2)\n",
      "Requirement already satisfied: lit in ./.local/lib/python3.9/site-packages (from triton==2.0.0->torch->medmnist) (16.0.6)\n",
      "Requirement already satisfied: requests in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from torchvision->medmnist) (2.27.1)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from packaging>=21->scikit-image->medmnist) (3.0.9)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from jinja2->torch->medmnist) (2.1.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from requests->torchvision->medmnist) (1.26.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from requests->torchvision->medmnist) (2022.5.18.1)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from requests->torchvision->medmnist) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from requests->torchvision->medmnist) (3.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in ./.local/lib/python3.9/site-packages (from sympy->torch->medmnist) (1.3.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install medmnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2791ff01-c32c-4cc3-9008-4b361395c60e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: tensorboardX in ./.local/lib/python3.9/site-packages (2.6.2.2)\n",
      "Requirement already satisfied: numpy in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from tensorboardX) (1.22.4)\n",
      "Requirement already satisfied: packaging in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from tensorboardX) (21.3)\n",
      "Requirement already satisfied: protobuf>=3.20 in ./.local/lib/python3.9/site-packages (from tensorboardX) (4.24.2)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /packages/apps/jupyter/2022-05-26/lib/python3.9/site-packages (from packaging->tensorboardX) (3.0.9)\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorboardX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5f1cd2bf-550d-4c03-a81d-1865e482706c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'experiments' already exists and is not an empty directory.\n"
     ]
    }
   ],
   "source": [
    "# Clone the repo to fetch the files\n",
    "!git clone https://github.com/MedMNIST/experiments.git"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fdccb19-9f80-4546-b0a1-38be3b7e644d",
   "metadata": {},
   "source": [
    "#### PART B : For all 2D datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c1692021-f7a5-4cc0-a501-1a144572fb98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a list of dataset flags\n",
    "dataset_flags = [\"pathmnist\", \"octmnist\", \"pneumoniamnist\",\n",
    "                 \"dermamnist\", \"retinamnist\", \"breastmnist\",\n",
    "                 \"bloodmnist\", \"tissuemnist\", \"organamnist\",\n",
    "                 \"organcmnist\", \"organsmnist\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "74ef8483-6060-4f56-86d2-4db203a154b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.984834899409294\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:50<01:41, 50.73s/it]cur_best_auc: 0.9858806012916779\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [01:40<00:50, 50.38s/it]cur_best_auc: 0.9975459930352589\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:31<00:00, 50.40s/it]\n",
      "pathmnist\n",
      "train  auc: 0.99769  acc: 0.93347\n",
      "val  auc: 0.99755  acc: 0.93133\n",
      "test  auc: 0.97182  acc: 0.77577\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9750747155189747\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:50<01:41, 50.66s/it]cur_best_auc: 0.9845362461106085\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:31<00:00, 50.49s/it]\n",
      "pathmnist\n",
      "train  auc: 0.98527  acc: 0.79042\n",
      "val  auc: 0.98454  acc: 0.78159\n",
      "test  auc: 0.91783  acc: 0.58245\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9794059379373314\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:50<01:41, 50.81s/it]cur_best_auc: 0.9945446194074719\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [01:40<00:50, 50.40s/it]cur_best_auc: 0.9976193094648335\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:31<00:00, 50.40s/it]\n",
      "pathmnist\n",
      "train  auc: 0.99768  acc: 0.94400\n",
      "val  auc: 0.99762  acc: 0.94392\n",
      "test  auc: 0.97461  acc: 0.88942\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9604737873292812\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:50<01:40, 50.35s/it]cur_best_auc: 0.9804929478612568\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [01:40<00:49, 49.96s/it]cur_best_auc: 0.9927638509354941\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:29<00:00, 49.92s/it]\n",
      "pathmnist\n",
      "train  auc: 0.99316  acc: 0.81464\n",
      "val  auc: 0.99276  acc: 0.81317\n",
      "test  auc: 0.98675  acc: 0.82117\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pathmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9754826563491719\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:50<01:41, 50.75s/it]cur_best_auc: 0.9893866094393707\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [01:40<00:50, 50.38s/it]cur_best_auc: 0.997295615081584\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:30<00:00, 50.32s/it]\n",
      "pathmnist\n",
      "train  auc: 0.99750  acc: 0.93412\n",
      "val  auc: 0.99730  acc: 0.93383\n",
      "test  auc: 0.97588  acc: 0.83524\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9532400693069044\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:51<01:42, 51.30s/it]cur_best_auc: 0.9554397798509795\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [01:41<00:50, 50.92s/it]cur_best_auc: 0.9630518184732145\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:32<00:00, 50.87s/it]\n",
      "octmnist\n",
      "train  auc: 0.96581  acc: 0.89291\n",
      "val  auc: 0.96305  acc: 0.88802\n",
      "test  auc: 0.95210  acc: 0.75400\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.946596461186108\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:51<01:43, 51.58s/it]cur_best_auc: 0.9621529576055518\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:33<00:00, 51.15s/it]\n",
      "octmnist\n",
      "train  auc: 0.96545  acc: 0.89987\n",
      "val  auc: 0.96215  acc: 0.89356\n",
      "test  auc: 0.94888  acc: 0.73700\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9513880231860019\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:51<01:43, 51.54s/it]cur_best_auc: 0.9602355716753702\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [01:42<00:51, 51.14s/it]cur_best_auc: 0.9678991476519462\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:33<00:00, 51.13s/it]\n",
      "octmnist\n",
      "train  auc: 0.97281  acc: 0.91252\n",
      "val  auc: 0.96790  acc: 0.90583\n",
      "test  auc: 0.95120  acc: 0.72400\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9566986532894914\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:51<01:43, 51.73s/it]cur_best_auc: 0.9642765447738652\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [01:42<00:51, 51.33s/it]cur_best_auc: 0.9655892762746111\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:33<00:00, 51.28s/it]\n",
      "octmnist\n",
      "train  auc: 0.97221  acc: 0.91291\n",
      "val  auc: 0.96559  acc: 0.90574\n",
      "test  auc: 0.96592  acc: 0.76800\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/octmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.953766959962865\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:51<01:43, 51.53s/it]cur_best_auc: 0.9649232608147921\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [02:33<00:00, 51.21s/it]\n",
      "octmnist\n",
      "train  auc: 0.96692  acc: 0.90003\n",
      "val  auc: 0.96492  acc: 0.89503\n",
      "test  auc: 0.96389  acc: 0.78900\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.920003808435685\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:03<00:06,  3.28s/it]cur_best_auc: 0.9950680757878702\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:08<00:00,  2.83s/it]\n",
      "pneumoniamnist\n",
      "train  auc: 0.99596  acc: 0.97260\n",
      "val  auc: 0.99507  acc: 0.96756\n",
      "test  auc: 0.96609  acc: 0.86218\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.961658573740836\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:03<00:06,  3.27s/it]cur_best_auc: 0.9841378653717985\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:05<00:02,  2.87s/it]cur_best_auc: 0.9941350090450347\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:08<00:00,  2.82s/it]\n",
      "pneumoniamnist\n",
      "train  auc: 0.99788  acc: 0.97855\n",
      "val  auc: 0.99414  acc: 0.97137\n",
      "test  auc: 0.96337  acc: 0.83333\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6457393125773588\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:03<00:06,  3.29s/it]cur_best_auc: 0.9871274873845568\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:05<00:02,  2.89s/it]cur_best_auc: 0.9920213272398363\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:08<00:00,  2.84s/it]\n",
      "pneumoniamnist\n",
      "train  auc: 0.99730  acc: 0.96856\n",
      "val  auc: 0.99202  acc: 0.95038\n",
      "test  auc: 0.96682  acc: 0.90545\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9403027706369609\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:03<00:06,  3.31s/it]cur_best_auc: 0.9946301056840903\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:08<00:00,  2.84s/it]\n",
      "pneumoniamnist\n",
      "train  auc: 0.99744  acc: 0.97621\n",
      "val  auc: 0.99463  acc: 0.95611\n",
      "test  auc: 0.94854  acc: 0.81891\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/pneumoniamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.600866419118347\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:03<00:06,  3.28s/it]cur_best_auc: 0.9935066171570026\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:08<00:00,  2.83s/it]\n",
      "pneumoniamnist\n",
      "train  auc: 0.99691  acc: 0.96134\n",
      "val  auc: 0.99351  acc: 0.95420\n",
      "test  auc: 0.94654  acc: 0.78686\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.7921115219100621\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:04<00:09,  5.00s/it]cur_best_auc: 0.8719814511201316\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:09<00:04,  4.60s/it]cur_best_auc: 0.8803924341206529\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:13<00:00,  4.55s/it]\n",
      "dermamnist\n",
      "train  auc: 0.86163  acc: 0.67846\n",
      "val  auc: 0.88039  acc: 0.67498\n",
      "test  auc: 0.84884  acc: 0.67581\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.840440602140235\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:04<00:09,  4.99s/it]cur_best_auc: 0.8827556474148824\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:13<00:00,  4.54s/it]\n",
      "dermamnist\n",
      "train  auc: 0.87071  acc: 0.68874\n",
      "val  auc: 0.88276  acc: 0.70588\n",
      "test  auc: 0.85987  acc: 0.67980\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8237507401479779\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:04<00:09,  4.98s/it]cur_best_auc: 0.8767406633959502\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:09<00:04,  4.59s/it]cur_best_auc: 0.885120265372106\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:13<00:00,  4.54s/it]\n",
      "dermamnist\n",
      "train  auc: 0.87902  acc: 0.71086\n",
      "val  auc: 0.88512  acc: 0.71685\n",
      "test  auc: 0.86937  acc: 0.70673\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8053040593003399\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:04<00:09,  4.99s/it]cur_best_auc: 0.8650177605534267\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:09<00:04,  4.59s/it]cur_best_auc: 0.8737317257800031\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:13<00:00,  4.54s/it]\n",
      "dermamnist\n",
      "train  auc: 0.85695  acc: 0.68631\n",
      "val  auc: 0.87373  acc: 0.69890\n",
      "test  auc: 0.85715  acc: 0.66883\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/dermamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8370313696098008\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:05<00:10,  5.01s/it]cur_best_auc: 0.8820679544137758\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:13<00:00,  4.55s/it]\n",
      "dermamnist\n",
      "train  auc: 0.86704  acc: 0.70287\n",
      "val  auc: 0.88207  acc: 0.71186\n",
      "test  auc: 0.85910  acc: 0.69875\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.2876243584000258\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:01<00:02,  1.40s/it]cur_best_auc: 0.6433930415997541\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:02<00:00,  1.07it/s]\n",
      "retinamnist\n",
      "train  auc: 0.65290  acc: 0.18704\n",
      "val  auc: 0.64339  acc: 0.18333\n",
      "test  auc: 0.64153  acc: 0.18500\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6445852084830728\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:01<00:02,  1.41s/it]cur_best_auc: 0.6535426167526854\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:02<00:01,  1.01s/it]cur_best_auc: 0.7869021726839425\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:02<00:00,  1.05it/s]\n",
      "retinamnist\n",
      "train  auc: 0.73165  acc: 0.48704\n",
      "val  auc: 0.78690  acc: 0.55833\n",
      "test  auc: 0.70662  acc: 0.49750\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6006806699261695\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:01<00:02,  1.40s/it]cur_best_auc: 0.6902675398054499\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:02<00:01,  1.00s/it]cur_best_auc: 0.7100117229833478\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:02<00:00,  1.05it/s]\n",
      "retinamnist\n",
      "train  auc: 0.68551  acc: 0.44722\n",
      "val  auc: 0.71001  acc: 0.45833\n",
      "test  auc: 0.69808  acc: 0.45750\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.4747227474336551\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:01<00:02,  1.40s/it]cur_best_auc: 0.6826237715133978\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:02<00:01,  1.00s/it]cur_best_auc: 0.7372759106080234\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:02<00:00,  1.05it/s]\n",
      "retinamnist\n",
      "train  auc: 0.69672  acc: 0.33519\n",
      "val  auc: 0.73728  acc: 0.40833\n",
      "test  auc: 0.71237  acc: 0.44000\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/retinamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.7368263440037659\n",
      "cur_best_epoch 0\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:02<00:00,  1.05it/s]\n",
      "retinamnist\n",
      "train  auc: 0.68505  acc: 0.19074\n",
      "val  auc: 0.73683  acc: 0.23333\n",
      "test  auc: 0.67395  acc: 0.23000\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.4954051796157059\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:01<00:02,  1.05s/it]cur_best_auc: 0.5413533834586466\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:01<00:00,  1.69it/s]\n",
      "breastmnist\n",
      "train  auc: 0.60641  acc: 0.73077\n",
      "val  auc: 0.54135  acc: 0.73077\n",
      "test  auc: 0.57707  acc: 0.73077\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5137844611528822\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:01<00:02,  1.04s/it]cur_best_auc: 0.5522138680033416\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:01<00:00,  1.55it/s]cur_best_auc: 0.6131996658312449\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:01<00:00,  1.68it/s]\n",
      "breastmnist\n",
      "train  auc: 0.65502  acc: 0.73077\n",
      "val  auc: 0.61320  acc: 0.73077\n",
      "test  auc: 0.63304  acc: 0.73077\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.609857978279031\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:01<00:02,  1.05s/it]cur_best_auc: 0.6324143692564745\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:01<00:00,  1.54it/s]cur_best_auc: 0.6917293233082706\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:01<00:00,  1.68it/s]\n",
      "breastmnist\n",
      "train  auc: 0.70796  acc: 0.65568\n",
      "val  auc: 0.69173  acc: 0.67949\n",
      "test  auc: 0.69089  acc: 0.69231\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6482873851294904\n",
      "cur_best_epoch 0\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:01<00:00,  1.72it/s]\n",
      "breastmnist\n",
      "train  auc: 0.63543  acc: 0.73077\n",
      "val  auc: 0.64829  acc: 0.73077\n",
      "test  auc: 0.62114  acc: 0.73077\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/breastmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5522138680033417\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:01<00:02,  1.04s/it]cur_best_auc: 0.5806182121971595\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:01<00:00,  1.55it/s]cur_best_auc: 0.5839598997493735\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:01<00:00,  1.69it/s]\n",
      "breastmnist\n",
      "train  auc: 0.64438  acc: 0.73077\n",
      "val  auc: 0.58396  acc: 0.73077\n",
      "test  auc: 0.61884  acc: 0.73077\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8516145656225299\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:08<00:16,  8.03s/it]cur_best_auc: 0.9363678582703782\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:15<00:07,  7.63s/it]cur_best_auc: 0.9825097625047504\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:22<00:00,  7.58s/it]\n",
      "bloodmnist\n",
      "train  auc: 0.98375  acc: 0.81060\n",
      "val  auc: 0.98251  acc: 0.80724\n",
      "test  auc: 0.98062  acc: 0.80327\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9662480334218055\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:07<00:15,  8.00s/it]cur_best_auc: 0.9751260945441618\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:15<00:07,  7.63s/it]cur_best_auc: 0.978449905937129\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:22<00:00,  7.58s/it]\n",
      "bloodmnist\n",
      "train  auc: 0.97922  acc: 0.74964\n",
      "val  auc: 0.97845  acc: 0.72897\n",
      "test  auc: 0.97380  acc: 0.73107\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9612102355093619\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:08<00:16,  8.05s/it]cur_best_auc: 0.9814832314864831\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:22<00:00,  7.59s/it]\n",
      "bloodmnist\n",
      "train  auc: 0.98291  acc: 0.82365\n",
      "val  auc: 0.98148  acc: 0.81776\n",
      "test  auc: 0.97927  acc: 0.81263\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9856047545283751\n",
      "cur_best_epoch 0\n",
      " 67%|██████████████████████████████               | 2/3 [00:15<00:07,  7.62s/it]cur_best_auc: 0.9881348694450212\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:22<00:00,  7.57s/it]\n",
      "bloodmnist\n",
      "train  auc: 0.99034  acc: 0.88218\n",
      "val  auc: 0.98813  acc: 0.87091\n",
      "test  auc: 0.98806  acc: 0.86671\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/bloodmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9784325809501677\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:08<00:16,  8.00s/it]cur_best_auc: 0.9805119934615951\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:15<00:07,  7.62s/it]cur_best_auc: 0.9869478966247373\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:22<00:00,  7.56s/it]\n",
      "bloodmnist\n",
      "train  auc: 0.98877  acc: 0.82248\n",
      "val  auc: 0.98695  acc: 0.82593\n",
      "test  auc: 0.98510  acc: 0.80970\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8546568056266057\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [01:37<03:15, 97.87s/it]cur_best_auc: 0.907034597800071\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [03:15<01:37, 97.52s/it]cur_best_auc: 0.9071573206664372\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [04:52<00:00, 97.38s/it]\n",
      "tissuemnist\n",
      "train  auc: 0.91602  acc: 0.64435\n",
      "val  auc: 0.90716  acc: 0.63240\n",
      "test  auc: 0.90971  acc: 0.63280\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8768353826435702\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [01:37<03:15, 97.96s/it]cur_best_auc: 0.9020568487699291\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [04:52<00:00, 97.45s/it]\n",
      "tissuemnist\n",
      "train  auc: 0.90709  acc: 0.61834\n",
      "val  auc: 0.90206  acc: 0.60948\n",
      "test  auc: 0.90253  acc: 0.60876\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8901979003252277\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [01:37<03:15, 97.67s/it]cur_best_auc: 0.8950117536941615\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [03:14<01:37, 97.33s/it]cur_best_auc: 0.8993024209736145\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [04:51<00:00, 97.27s/it]\n",
      "tissuemnist\n",
      "train  auc: 0.90900  acc: 0.62424\n",
      "val  auc: 0.89930  acc: 0.61552\n",
      "test  auc: 0.90320  acc: 0.61343\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8756151621681485\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [01:37<03:15, 97.93s/it]cur_best_auc: 0.9038888648993705\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [03:15<01:37, 97.54s/it]cur_best_auc: 0.9070715269834054\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [04:52<00:00, 97.42s/it]\n",
      "tissuemnist\n",
      "train  auc: 0.91505  acc: 0.64596\n",
      "val  auc: 0.90707  acc: 0.63304\n",
      "test  auc: 0.91031  acc: 0.63439\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/tissuemnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8767652037768436\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [01:37<03:15, 97.57s/it]cur_best_auc: 0.9017678572052388\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [04:52<00:00, 97.47s/it]\n",
      "tissuemnist\n",
      "train  auc: 0.90726  acc: 0.62349\n",
      "val  auc: 0.90177  acc: 0.61844\n",
      "test  auc: 0.90241  acc: 0.61423\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9967885438541789\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:22<00:45, 22.95s/it]cur_best_auc: 0.9995695377274187\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [01:07<00:00, 22.51s/it]\n",
      "organamnist\n",
      "train  auc: 0.99957  acc: 0.97539\n",
      "val  auc: 0.99957  acc: 0.95979\n",
      "test  auc: 0.99448  acc: 0.90955\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9933081977841481\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:23<00:46, 23.13s/it]cur_best_auc: 0.9990748986762917\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:45<00:22, 22.74s/it]cur_best_auc: 0.9994515083656975\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [01:08<00:00, 22.68s/it]\n",
      "organamnist\n",
      "train  auc: 0.99982  acc: 0.97880\n",
      "val  auc: 0.99945  acc: 0.95347\n",
      "test  auc: 0.99516  acc: 0.90123\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9046556910768502\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:23<00:46, 23.10s/it]cur_best_auc: 0.9987763978081037\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:45<00:22, 22.70s/it]cur_best_auc: 0.9992058895031636\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [01:07<00:00, 22.66s/it]\n",
      "organamnist\n",
      "train  auc: 0.99982  acc: 0.98285\n",
      "val  auc: 0.99921  acc: 0.96010\n",
      "test  auc: 0.99463  acc: 0.89931\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9990837492049252\n",
      "cur_best_epoch 0\n",
      "100%|█████████████████████████████████████████████| 3/3 [01:07<00:00, 22.65s/it]\n",
      "organamnist\n",
      "train  auc: 0.99606  acc: 0.93563\n",
      "val  auc: 0.99908  acc: 0.95317\n",
      "test  auc: 0.98822  acc: 0.85910\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organamnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9976026151979455\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:23<00:46, 23.12s/it]cur_best_auc: 0.9994140406830812\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [01:08<00:00, 22.73s/it]\n",
      "organamnist\n",
      "train  auc: 0.99958  acc: 0.97128\n",
      "val  auc: 0.99941  acc: 0.94854\n",
      "test  auc: 0.99317  acc: 0.89453\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9920965662672597\n",
      "cur_best_epoch 0\n",
      " 67%|██████████████████████████████               | 2/3 [00:18<00:09,  9.03s/it]cur_best_auc: 0.9991161736658366\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:26<00:00,  8.98s/it]\n",
      "organcmnist\n",
      "train  auc: 0.99866  acc: 0.95223\n",
      "val  auc: 0.99912  acc: 0.96237\n",
      "test  auc: 0.98672  acc: 0.86817\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9278414601244709\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:09<00:18,  9.46s/it]cur_best_auc: 0.9984907416684657\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:18<00:09,  9.06s/it]cur_best_auc: 0.9986693065543044\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:27<00:00,  9.01s/it]\n",
      "organcmnist\n",
      "train  auc: 0.99734  acc: 0.90485\n",
      "val  auc: 0.99867  acc: 0.92391\n",
      "test  auc: 0.98552  acc: 0.81894\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9946887446959933\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:09<00:18,  9.39s/it]cur_best_auc: 0.9970694892873041\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:26<00:00,  8.94s/it]\n",
      "organcmnist\n",
      "train  auc: 0.99321  acc: 0.88385\n",
      "val  auc: 0.99707  acc: 0.90803\n",
      "test  auc: 0.97790  acc: 0.81434\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9977862192121821\n",
      "cur_best_epoch 0\n",
      " 67%|██████████████████████████████               | 2/3 [00:18<00:09,  9.04s/it]cur_best_auc: 0.9988703781172041\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:26<00:00,  8.99s/it]\n",
      "organcmnist\n",
      "train  auc: 0.99863  acc: 0.94623\n",
      "val  auc: 0.99887  acc: 0.96405\n",
      "test  auc: 0.98490  acc: 0.86478\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organcmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9976053991923404\n",
      "cur_best_epoch 0\n",
      " 67%|██████████████████████████████               | 2/3 [00:18<00:09,  9.06s/it]cur_best_auc: 0.9992551814169682\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:27<00:00,  9.01s/it]\n",
      "organcmnist\n",
      "train  auc: 0.99816  acc: 0.94538\n",
      "val  auc: 0.99926  acc: 0.95945\n",
      "test  auc: 0.98647  acc: 0.85873\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9790075625357288\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:10<00:20, 10.00s/it]cur_best_auc: 0.9901199208957255\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:19<00:09,  9.61s/it]cur_best_auc: 0.9909371107477245\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:28<00:00,  9.56s/it]\n",
      "organsmnist\n",
      "train  auc: 0.99102  acc: 0.86664\n",
      "val  auc: 0.99094  acc: 0.86134\n",
      "test  auc: 0.97273  acc: 0.74731\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9810508176683791\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:10<00:20, 10.02s/it]cur_best_auc: 0.9892470757636034\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:19<00:09,  9.63s/it]cur_best_auc: 0.9893657385818226\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:28<00:00,  9.58s/it]\n",
      "organsmnist\n",
      "train  auc: 0.99113  acc: 0.86112\n",
      "val  auc: 0.98937  acc: 0.84339\n",
      "test  auc: 0.96288  acc: 0.72919\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9799949555162478\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:10<00:20, 10.05s/it]cur_best_auc: 0.9868104005598234\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:19<00:09,  9.65s/it]cur_best_auc: 0.9885548143416181\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:28<00:00,  9.60s/it]\n",
      "organsmnist\n",
      "train  auc: 0.99076  acc: 0.86026\n",
      "val  auc: 0.98855  acc: 0.82504\n",
      "test  auc: 0.96866  acc: 0.72941\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9706108008498675\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:10<00:20, 10.07s/it]cur_best_auc: 0.989828499200994\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:28<00:00,  9.61s/it]\n",
      "organsmnist\n",
      "train  auc: 0.98740  acc: 0.83551\n",
      "val  auc: 0.98983  acc: 0.82871\n",
      "test  auc: 0.96638  acc: 0.73610\n",
      "\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/organsmnist.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.9776162714758015\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:10<00:20, 10.04s/it]cur_best_auc: 0.9920072669027235\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:28<00:00,  9.59s/it]\n",
      "organsmnist\n",
      "train  auc: 0.98607  acc: 0.80854\n",
      "val  auc: 0.99201  acc: 0.84584\n",
      "test  auc: 0.96436  acc: 0.70699\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Directory containing the log files\n",
    "base_directory = 'experiments/MedMNIST2D'\n",
    "\n",
    "# Loop through each dataset flag\n",
    "for dataset_flag in dataset_flags:\n",
    "    # Loop through each run for each dataset\n",
    "    for run in range(5):\n",
    "        output_folder = f\"./MedMNIST2D/{dataset_flag}/run_{run}\"\n",
    "        command = f\"python experiments/MedMNIST2D/train_and_eval_pytorch.py --data_flag {dataset_flag} --num_epochs 3 --download --model_flag resnet50 --output_root {output_folder}\"\n",
    "        !{command}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "462cee2e-fbe6-4591-8549-ae4f16e20131",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BMEhw2.ipynb  MedMNIST2D       MedMNIST3D  Untitled.ipynb\n",
      "experiments   MedMNIST2D_data  ondemand\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "27110e9e-e6ad-4cef-a64c-5c7781e057c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Dataset    Set  Mean AUC  Mean Accuracy   STD AUC  STD Accuracy\n",
      "0       bloodmnist   test  0.981370       0.804676  0.005499      0.048365\n",
      "1       bloodmnist  train  0.984998       0.817710  0.004530      0.047166\n",
      "2       bloodmnist    val  0.983504       0.810162  0.003997      0.051472\n",
      "3      breastmnist   test  0.628196       0.723078  0.040943      0.017200\n",
      "4      breastmnist  train  0.649840       0.715752  0.037177      0.033581\n",
      "5      breastmnist    val  0.615706       0.720514  0.057845      0.022933\n",
      "6       dermamnist   test  0.858866       0.685984  0.007329      0.016042\n",
      "7       dermamnist  train  0.867070       0.693448  0.008486      0.013131\n",
      "8       dermamnist    val  0.880814       0.701694  0.004309      0.016371\n",
      "9         octmnist   test  0.956398       0.754400  0.007887      0.025540\n",
      "10        octmnist  train  0.968640       0.903648  0.003580      0.008763\n",
      "11        octmnist    val  0.964722       0.897636  0.002253      0.007885\n",
      "12     organamnist   test  0.993132       0.892744  0.002842      0.019575\n",
      "13     organamnist  train  0.998970       0.968790  0.001631      0.019021\n",
      "14     organamnist    val  0.999344       0.955014  0.000196      0.004909\n",
      "15     organcmnist   test  0.984302       0.844992  0.003653      0.026152\n",
      "16     organcmnist  train  0.997200       0.926508  0.002293      0.030395\n",
      "17     organcmnist    val  0.998598       0.943562  0.000884      0.025859\n",
      "18     organsmnist   test  0.967002       0.729800  0.003870      0.014723\n",
      "19     organsmnist  train  0.989276       0.846414  0.002371      0.024343\n",
      "20     organsmnist    val  0.990140       0.840864  0.001356      0.014565\n",
      "21       pathmnist   test  0.965378       0.780810  0.027178      0.118068\n",
      "22       pathmnist  train  0.994260       0.883330  0.005385      0.074373\n",
      "23       pathmnist    val  0.993954       0.880768  0.005648      0.077081\n",
      "24  pneumoniamnist   test  0.958272       0.841346  0.009906      0.044944\n",
      "25  pneumoniamnist  train  0.997098       0.971452  0.000725      0.006800\n",
      "26  pneumoniamnist    val  0.993874       0.959924  0.001188      0.009051\n",
      "27     retinamnist   test  0.686510       0.362000  0.029103      0.143455\n",
      "28     retinamnist  train  0.690366       0.329446  0.028286      0.139874\n",
      "29     retinamnist    val  0.722882       0.368330  0.052404      0.156724\n",
      "30     tissuemnist   test  0.905632       0.620722  0.004013      0.011949\n",
      "31     tissuemnist  train  0.910884       0.631276  0.004325      0.012884\n",
      "32     tissuemnist    val  0.903472       0.621776  0.003494      0.010502\n"
     ]
    }
   ],
   "source": [
    "# Directory containing the log files\n",
    "base_directory = 'MedMNIST2D'\n",
    "\n",
    "# Initialize an empty list to store log file paths\n",
    "log_file_paths = []\n",
    "\n",
    "# Recursively search for log files\n",
    "for root, _, files in os.walk(base_directory):\n",
    "    for file in files:\n",
    "        if file.endswith('_log.txt'):\n",
    "            log_file_paths.append(os.path.join(root, file))\n",
    "\n",
    "# Initialize an empty list to store extracted data\n",
    "all_data = []\n",
    "\n",
    "# Loop through each log file path\n",
    "for log_file_path in log_file_paths:\n",
    "    # Extract the dataset name and run information from the path\n",
    "    parts = log_file_path.split(os.path.sep)\n",
    "    dataset_name = parts[-3]\n",
    "    run_info = parts[-4]\n",
    "\n",
    "    # Check if the log file exists\n",
    "    if os.path.exists(log_file_path):\n",
    "        # Read the contents of the log file\n",
    "        with open(log_file_path, 'r') as file:\n",
    "            lines = file.readlines()\n",
    "\n",
    "        # Initialize dictionaries to store metrics for each dataset\n",
    "        dataset_metrics = {'Run': [], 'Set': [], 'AUC': [], 'Accuracy': [],  'Dataset': []}\n",
    "\n",
    "        # Process each line (starting from the second line) and extract the relevant information\n",
    "        for line in lines[1:]:\n",
    "            parts = line.split()\n",
    "            if len(parts) == 5 and parts[0] in ['train', 'val', 'test']:\n",
    "                dataset_metrics['Run'].append(run_info)\n",
    "                dataset_metrics['Set'].append(parts[0])\n",
    "                dataset_metrics['AUC'].append(float(parts[2]))\n",
    "                dataset_metrics['Accuracy'].append(float(parts[4]))\n",
    "                dataset_metrics['Dataset'].append(dataset_name)\n",
    "\n",
    "        # Append the extracted data for this log file to the list\n",
    "        all_data.append(pd.DataFrame(dataset_metrics))\n",
    "    else:\n",
    "        print(f\"Log file not found: {log_file_path}\")\n",
    "\n",
    "# Concatenate all dataframes into a single dataframe if there is data\n",
    "if all_data:\n",
    "    final_df_2d = pd.concat(all_data, ignore_index=True)\n",
    "    final_df_2d = final_df_2d.sort_values(by=['Run'])\n",
    "\n",
    "    # Calculate mean and standard deviation for 'train', 'val', and 'test' sets across all runs\n",
    "    grouped = final_df_2d.groupby(['Dataset', 'Set'])  # Group by both Dataset and Set\n",
    "    result_dfs = []\n",
    "    for (dataset, name), group in grouped:\n",
    "        set_mean = group[['AUC', 'Accuracy']].mean()\n",
    "        set_std = group[['AUC', 'Accuracy']].std()\n",
    "\n",
    "        result_df = pd.DataFrame({\n",
    "            'Dataset': [dataset],\n",
    "            'Set': [name],\n",
    "            'Mean AUC': [set_mean['AUC']],\n",
    "            'Mean Accuracy': [set_mean['Accuracy']],\n",
    "            'STD AUC': [set_std['AUC']],\n",
    "            'STD Accuracy': [set_std['Accuracy']]\n",
    "        })\n",
    "        result_dfs.append(result_df)\n",
    "\n",
    "    result_df = pd.concat(result_dfs, ignore_index=True)\n",
    "    print(result_df)\n",
    "else:\n",
    "    print(\"No log files were processed.\")\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d2382683-148a-4652-9018-3800f4e24d9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_flags = ['nodulemnist3d','adrenalmnist3d','fracturemnist3d','vesselmnist3d','synapsemnist3d']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f6ca84a6-8992-41e6-afe7-249e528b2238",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Downloading https://zenodo.org/record/6496656/files/nodulemnist3d.npz?download=1 to /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "100%|███████████████████████████| 29299364/29299364 [00:47<00:00, 611743.13it/s]\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6504065040650407\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:15<00:31, 15.93s/it]cur_best_auc: 0.8828881145954317\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:46<00:00, 15.34s/it]\n",
      "nodulemnist3d\n",
      "train  auc: 0.83790  acc: 0.82211\n",
      "val  auc: 0.88289  acc: 0.83636\n",
      "test  auc: 0.80227  acc: 0.84194\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5718157181571816\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:15<00:31, 15.72s/it]cur_best_auc: 0.6205962059620597\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:30<00:15, 15.33s/it]cur_best_auc: 0.7899728997289972\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:45<00:00, 15.28s/it]\n",
      "nodulemnist3d\n",
      "train  auc: 0.78501  acc: 0.81606\n",
      "val  auc: 0.78997  acc: 0.82424\n",
      "test  auc: 0.77973  acc: 0.84516\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6368563685636855\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:15<00:31, 15.72s/it]cur_best_auc: 0.7861014324428958\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:30<00:15, 15.33s/it]cur_best_auc: 0.8221060782036391\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:45<00:00, 15.28s/it]\n",
      "nodulemnist3d\n",
      "train  auc: 0.81624  acc: 0.82729\n",
      "val  auc: 0.82211  acc: 0.86667\n",
      "test  auc: 0.76715  acc: 0.80645\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.285230352303523\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:15<00:31, 15.73s/it]cur_best_auc: 0.7446767324816105\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:30<00:15, 15.34s/it]cur_best_auc: 0.7868757259001161\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:45<00:00, 15.28s/it]\n",
      "nodulemnist3d\n",
      "train  auc: 0.79654  acc: 0.81347\n",
      "val  auc: 0.78688  acc: 0.83030\n",
      "test  auc: 0.75337  acc: 0.82581\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/nodulemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.66221447928765\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:15<00:31, 15.72s/it]cur_best_auc: 0.7963608207510646\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:30<00:15, 15.34s/it]cur_best_auc: 0.827138985675571\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:45<00:00, 15.29s/it]\n",
      "nodulemnist3d\n",
      "train  auc: 0.83796  acc: 0.81952\n",
      "val  auc: 0.82714  acc: 0.82424\n",
      "test  auc: 0.84076  acc: 0.84194\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Downloading https://zenodo.org/record/6496656/files/adrenalmnist3d.npz?download=1 to /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "100%|███████████████████████████████| 276833/276833 [00:00<00:00, 435244.66it/s]\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.17165071770334928\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:16<00:32, 16.02s/it]cur_best_auc: 0.7888755980861244\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:31<00:15, 15.58s/it]cur_best_auc: 0.8283492822966507\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:46<00:00, 15.52s/it]\n",
      "adrenalmnist3d\n",
      "train  auc: 0.83073  acc: 0.73990\n",
      "val  auc: 0.82835  acc: 0.74490\n",
      "test  auc: 0.87159  acc: 0.75168\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.8295454545454545\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:15<00:31, 15.93s/it]cur_best_auc: 0.8313397129186603\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:31<00:15, 15.53s/it]cur_best_auc: 0.8325358851674641\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:46<00:00, 15.47s/it]\n",
      "adrenalmnist3d\n",
      "train  auc: 0.86617  acc: 0.81229\n",
      "val  auc: 0.83254  acc: 0.79592\n",
      "test  auc: 0.86438  acc: 0.78188\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5992822966507177\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:15<00:31, 15.94s/it]cur_best_auc: 0.8367224880382774\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:46<00:00, 15.48s/it]\n",
      "adrenalmnist3d\n",
      "train  auc: 0.81974  acc: 0.82492\n",
      "val  auc: 0.83672  acc: 0.82653\n",
      "test  auc: 0.81324  acc: 0.78188\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.827153110047847\n",
      "cur_best_epoch 0\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:46<00:00, 15.47s/it]\n",
      "adrenalmnist3d\n",
      "train  auc: 0.80919  acc: 0.79461\n",
      "val  auc: 0.82715  acc: 0.78571\n",
      "test  auc: 0.85368  acc: 0.77517\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/adrenalmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6572966507177034\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:15<00:31, 15.94s/it]cur_best_auc: 0.8325358851674641\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:31<00:15, 15.53s/it]cur_best_auc: 0.8528708133971291\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:46<00:00, 15.48s/it]\n",
      "adrenalmnist3d\n",
      "train  auc: 0.85885  acc: 0.85859\n",
      "val  auc: 0.85287  acc: 0.84694\n",
      "test  auc: 0.87653  acc: 0.82550\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Downloading https://zenodo.org/record/6496656/files/fracturemnist3d.npz?download=1 to /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "100%|█████████████████████████████| 3278419/3278419 [00:08<00:00, 397508.27it/s]\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6343424534947926\n",
      "cur_best_epoch 0\n",
      " 67%|██████████████████████████████               | 2/3 [00:27<00:13, 13.46s/it]cur_best_auc: 0.6598016877904923\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:40<00:00, 13.42s/it]\n",
      "fracturemnist3d\n",
      "train  auc: 0.70944  acc: 0.48588\n",
      "val  auc: 0.65980  acc: 0.49515\n",
      "test  auc: 0.66990  acc: 0.52500\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6707743357683382\n",
      "cur_best_epoch 0\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:40<00:00, 13.48s/it]\n",
      "fracturemnist3d\n",
      "train  auc: 0.65845  acc: 0.45472\n",
      "val  auc: 0.67077  acc: 0.40777\n",
      "test  auc: 0.65537  acc: 0.37500\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6473016084571461\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:13<00:27, 13.88s/it]cur_best_auc: 0.6802624452864358\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:40<00:00, 13.42s/it]\n",
      "fracturemnist3d\n",
      "train  auc: 0.70948  acc: 0.52288\n",
      "val  auc: 0.68026  acc: 0.47573\n",
      "test  auc: 0.70887  acc: 0.48750\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5845640579746937\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:13<00:27, 13.87s/it]cur_best_auc: 0.6092684090385011\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:27<00:13, 13.47s/it]cur_best_auc: 0.6308896547201226\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:40<00:00, 13.42s/it]\n",
      "fracturemnist3d\n",
      "train  auc: 0.70911  acc: 0.43135\n",
      "val  auc: 0.63089  acc: 0.38835\n",
      "test  auc: 0.69340  acc: 0.49167\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/fracturemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.4056804791310989\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:13<00:27, 13.87s/it]cur_best_auc: 0.6698156716149519\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:27<00:13, 13.48s/it]cur_best_auc: 0.677618899530135\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:40<00:00, 13.42s/it]\n",
      "fracturemnist3d\n",
      "train  auc: 0.67513  acc: 0.49270\n",
      "val  auc: 0.67762  acc: 0.52427\n",
      "test  auc: 0.67261  acc: 0.48333\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Downloading https://zenodo.org/record/6496656/files/vesselmnist3d.npz?download=1 to /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "100%|███████████████████████████████| 398373/398373 [00:00<00:00, 423121.28it/s]\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5136363636363637\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:17<00:35, 17.99s/it]cur_best_auc: 0.5967914438502674\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:35<00:17, 17.64s/it]cur_best_auc: 0.7005347593582888\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:52<00:00, 17.59s/it]\n",
      "vesselmnist3d\n",
      "train  auc: 0.74529  acc: 0.86217\n",
      "val  auc: 0.70053  acc: 0.80729\n",
      "test  auc: 0.69534  acc: 0.85602\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.770053475935829\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:18<00:36, 18.22s/it]cur_best_auc: 0.79572192513369\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:35<00:17, 17.81s/it]cur_best_auc: 0.8553475935828877\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:53<00:00, 17.75s/it]\n",
      "vesselmnist3d\n",
      "train  auc: 0.87242  acc: 0.88764\n",
      "val  auc: 0.85535  acc: 0.88542\n",
      "test  auc: 0.84901  acc: 0.88743\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.7286096256684493\n",
      "cur_best_epoch 0\n",
      " 67%|██████████████████████████████               | 2/3 [00:35<00:17, 17.60s/it]cur_best_auc: 0.8021390374331551\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:52<00:00, 17.56s/it]\n",
      "vesselmnist3d\n",
      "train  auc: 0.78254  acc: 0.76479\n",
      "val  auc: 0.80214  acc: 0.73958\n",
      "test  auc: 0.75537  acc: 0.73822\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.7764705882352942\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:17<00:35, 17.98s/it]cur_best_auc: 0.7975935828877005\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:35<00:17, 17.63s/it]cur_best_auc: 0.8403743315508022\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:52<00:00, 17.59s/it]\n",
      "vesselmnist3d\n",
      "train  auc: 0.85348  acc: 0.35655\n",
      "val  auc: 0.84037  acc: 0.29167\n",
      "test  auc: 0.79667  acc: 0.31414\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/vesselmnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.4700534759358289\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:17<00:35, 17.97s/it]cur_best_auc: 0.6836898395721924\n",
      "cur_best_epoch 1\n",
      " 67%|██████████████████████████████               | 2/3 [00:35<00:17, 17.63s/it]cur_best_auc: 0.7775401069518718\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:52<00:00, 17.58s/it]\n",
      "vesselmnist3d\n",
      "train  auc: 0.85568  acc: 0.60000\n",
      "val  auc: 0.77754  acc: 0.54688\n",
      "test  auc: 0.81533  acc: 0.60209\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Downloading https://zenodo.org/record/6496656/files/synapsemnist3d.npz?download=1 to /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "100%|██████████████████████████| 38034583/38034583 [00:32<00:00, 1170136.73it/s]\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6291989664082687\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:16<00:33, 16.66s/it]cur_best_auc: 0.6740956072351423\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:48<00:00, 16.24s/it]\n",
      "synapsemnist3d\n",
      "train  auc: 0.93684  acc: 0.79756\n",
      "val  auc: 0.67410  acc: 0.74011\n",
      "test  auc: 0.73201  acc: 0.76136\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.6684431524547804\n",
      "cur_best_epoch 0\n",
      " 67%|██████████████████████████████               | 2/3 [00:32<00:16, 16.29s/it]cur_best_auc: 0.6807170542635658\n",
      "cur_best_epoch 2\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:48<00:00, 16.24s/it]\n",
      "synapsemnist3d\n",
      "train  auc: 0.84043  acc: 0.73984\n",
      "val  auc: 0.68072  acc: 0.73446\n",
      "test  auc: 0.74442  acc: 0.72727\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5494186046511628\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:16<00:33, 16.83s/it]cur_best_auc: 0.7811692506459947\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:48<00:00, 16.33s/it]\n",
      "synapsemnist3d\n",
      "train  auc: 0.91502  acc: 0.28049\n",
      "val  auc: 0.78117  acc: 0.27119\n",
      "test  auc: 0.68454  acc: 0.26989\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5855135658914729\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:16<00:33, 16.65s/it]cur_best_auc: 0.6976744186046512\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:48<00:00, 16.23s/it]\n",
      "synapsemnist3d\n",
      "train  auc: 0.97992  acc: 0.85041\n",
      "val  auc: 0.69767  acc: 0.59887\n",
      "test  auc: 0.70084  acc: 0.61648\n",
      "\n",
      "\n",
      "The ``converters`` are currently experimental. It may not support operations including (but not limited to) Functions in ``torch.nn.functional`` that involved data dimension\n",
      "==> Preparing data...\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "Using downloaded and verified file: /home/atricham/.medmnist/synapsemnist3d.npz\n",
      "==> Building and training model...\n",
      "  0%|                                                     | 0/3 [00:00<?, ?it/s]cur_best_auc: 0.5809108527131782\n",
      "cur_best_epoch 0\n",
      " 33%|███████████████                              | 1/3 [00:16<00:33, 16.66s/it]cur_best_auc: 0.7151162790697675\n",
      "cur_best_epoch 1\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:48<00:00, 16.24s/it]\n",
      "synapsemnist3d\n",
      "train  auc: 0.91479  acc: 0.86179\n",
      "val  auc: 0.71512  acc: 0.68927\n",
      "test  auc: 0.75224  acc: 0.74432\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Directory containing the log files\n",
    "base_directory = 'experiments/MedMNIST3D'\n",
    "\n",
    "# Loop through each dataset flag\n",
    "for dataset_flag in data_flags:\n",
    "    # Loop through each run for each dataset\n",
    "    for run in range(5):\n",
    "        output_folder = f\"./MedMNIST3D/{dataset_flag}/run_{run}\"\n",
    "        command = f\"python experiments/MedMNIST3D/train_and_eval_pytorch.py --data_flag {dataset_flag} --num_epochs 3 --download --model_flag resnet50 --output_root {output_folder}\"\n",
    "        !{command}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "56d6b508-5f8d-4818-a890-e01c28f34d98",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            Dataset    Set  Mean AUC  Mean Accuracy   STD AUC  STD Accuracy\n",
      "0    adrenalmnist3d   test  0.855884       0.783222  0.025340      0.026696\n",
      "1    adrenalmnist3d  train  0.836936       0.806062  0.024693      0.043767\n",
      "2    adrenalmnist3d    val  0.835526       0.800000  0.010402      0.039256\n",
      "3   fracturemnist3d   test  0.680030       0.472500  0.021073      0.056948\n",
      "4   fracturemnist3d  train  0.692322       0.477506  0.024042      0.035388\n",
      "5   fracturemnist3d    val  0.663868       0.458254  0.020060      0.058009\n",
      "6     nodulemnist3d   test  0.788656       0.832260  0.034232      0.016291\n",
      "7     nodulemnist3d  train  0.814730       0.819690  0.023942      0.005374\n",
      "8     nodulemnist3d    val  0.821798       0.836362  0.038708      0.017672\n",
      "9    synapsemnist3d   test  0.722810       0.623864  0.029004      0.205827\n",
      "10   synapsemnist3d  train  0.917400       0.706018  0.050556      0.242751\n",
      "11   synapsemnist3d    val  0.709756       0.606780  0.042980      0.195938\n",
      "12    vesselmnist3d   test  0.782344       0.679580  0.059235      0.233089\n",
      "13    vesselmnist3d  train  0.821882       0.694230  0.055020      0.219942\n",
      "14    vesselmnist3d    val  0.795186       0.654168  0.061198      0.238293\n"
     ]
    }
   ],
   "source": [
    "# Directory containing the log files\n",
    "base_directory_3d = 'MedMNIST3D'\n",
    "\n",
    "# Initialize an empty list to store log file paths\n",
    "log_file_paths_3d = []\n",
    "\n",
    "# Recursively search for log files\n",
    "for root, _, files in os.walk(base_directory_3d):\n",
    "    for file in files:\n",
    "        if file.endswith('_log.txt'):\n",
    "            log_file_paths_3d.append(os.path.join(root, file))\n",
    "\n",
    "# Initialize an empty list to store extracted data\n",
    "all_data = []\n",
    "\n",
    "# Loop through each log file path\n",
    "for log_file_path in log_file_paths_3d:\n",
    "    # Extract the dataset name and run information from the path\n",
    "    parts = log_file_path.split(os.path.sep)\n",
    "    dataset_name = parts[-3]\n",
    "    run_info = parts[-4]\n",
    "\n",
    "    # Check if the log file exists\n",
    "    if os.path.exists(log_file_path):\n",
    "        # Read the contents of the log file\n",
    "        with open(log_file_path, 'r') as file:\n",
    "            lines = file.readlines()\n",
    "\n",
    "        # Initialize dictionaries to store metrics for each dataset\n",
    "        dataset_metrics = {'Run': [], 'Set': [], 'AUC': [], 'Accuracy': [],  'Dataset': []}\n",
    "\n",
    "        # Process each line (starting from the second line) and extract the relevant information\n",
    "        for line in lines[1:]:\n",
    "            parts = line.split()\n",
    "            if len(parts) == 5 and parts[0] in ['train', 'val', 'test']:\n",
    "                dataset_metrics['Run'].append(run_info)\n",
    "                dataset_metrics['Set'].append(parts[0])\n",
    "                dataset_metrics['AUC'].append(float(parts[2]))\n",
    "                dataset_metrics['Accuracy'].append(float(parts[4]))\n",
    "                dataset_metrics['Dataset'].append(dataset_name)\n",
    "\n",
    "        # Append the extracted data for this log file to the list\n",
    "        all_data.append(pd.DataFrame(dataset_metrics))\n",
    "    else:\n",
    "        print(f\"Log file not found: {log_file_path}\")\n",
    "\n",
    "# Concatenate all dataframes into a single dataframe if there is data\n",
    "if all_data:\n",
    "    final_df_3d = pd.concat(all_data, ignore_index=True)\n",
    "    final_df_3d = final_df_3d.sort_values(by=['Run'])\n",
    "\n",
    "    # Calculate mean and standard deviation for 'train', 'val', and 'test' sets across all runs\n",
    "    grouped = final_df_3d.groupby(['Dataset', 'Set'])  # Group by both Dataset and Set\n",
    "    results_dfs = []\n",
    "    for (dataset, name), group in grouped:\n",
    "        set_mean = group[['AUC', 'Accuracy']].mean()\n",
    "        set_std = group[['AUC', 'Accuracy']].std()\n",
    "\n",
    "        results_df = pd.DataFrame({\n",
    "            'Dataset': [dataset],\n",
    "            'Set': [name],\n",
    "            'Mean AUC': [set_mean['AUC']],\n",
    "            'Mean Accuracy': [set_mean['Accuracy']],\n",
    "            'STD AUC': [set_std['AUC']],\n",
    "            'STD Accuracy': [set_std['Accuracy']]\n",
    "        })\n",
    "        results_dfs.append(results_df)\n",
    "\n",
    "    results_df = pd.concat(results_dfs, ignore_index=True)\n",
    "    print(results_df)\n",
    "else:\n",
    "    print(\"No log files were processed.\")\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "02e263b3-160e-436d-b5c3-62c1483048b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/atricham/MedMNIST2D_partb.zip.zip'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import shutil\n",
    "\n",
    "folder_to_zip = 'MedMNIST2D'\n",
    "output_zip_file = 'MedMNIST2D_partb.zip'\n",
    "\n",
    "shutil.make_archive(output_zip_file, 'zip', folder_to_zip)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7debf17f-7d9d-4900-b868-ff7f2a9ec589",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/atricham/MedMNIST3D_partb.zip.zip'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import shutil\n",
    "\n",
    "folder_to_zip = 'MedMNIST3D'\n",
    "output_zip_file = 'MedMNIST3D_partb.zip'\n",
    "\n",
    "shutil.make_archive(output_zip_file, 'zip', folder_to_zip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "430a92bb-601f-4293-a321-1c6432b75dba",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}