visualization/main_figures/4a_prospective_auc_open.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from pandas.io.json._normalize import nested_to_record\n",
    "import json, wandb\n",
    "from omegaconf import OmegaConf\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gather data df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pull_from_wandb = False\n",
    "\n",
    "if pull_from_wandb:\n",
    "    username = \"lavender\"\n",
    "    project_name = \"eval_threshold\"\n",
    "    api = wandb.Api()\n",
    "\n",
    "    def artifact_to_df(artifact):\n",
    "        local_path = artifact.file()\n",
    "        f = open(local_path)\n",
    "        data = json.load(f)\n",
    "        df = pd.DataFrame(data=data[\"data\"], columns=data[\"columns\"])\n",
    "        return df\n",
    "\n",
    "    def combine(\n",
    "        trials=[\"temporal\", \"prospective\", \"random\"],\n",
    "        metrics=[\"fpr\", \"tpr\"],\n",
    "        artifact_names=[\n",
    "            \"new_temporal_auc_curve\",\n",
    "            \"prospective_auc\",\n",
    "            \"new_random_test_auc_curve\",\n",
    "        ],\n",
    "    ):\n",
    "        dfs = []\n",
    "        subs = []\n",
    "        for artifact_name in artifact_names:\n",
    "            artifact = api.artifact(f\"{username}/{project_name}/{artifact_name}:v0\")\n",
    "            df = artifact_to_df(artifact)\n",
    "            dfs.append(df)\n",
    "        for idx, artifact_df in enumerate(dfs):\n",
    "            print(artifact_df)\n",
    "            df_sub = artifact_df[metrics]\n",
    "            df_sub[\"trial\"] = trials[idx]\n",
    "            print(df_sub)\n",
    "            subs.append(df_sub)\n",
    "        combined = pd.concat(subs, ignore_index=True)\n",
    "        return combined\n",
    "\n",
    "    combined = combine()\n",
    "    df_plot = combined[combined.trial.isin([\"temporal\", \"random\"])]\n",
    "    df_plot[\"trial\"] = df_plot[\"trial\"].replace(\n",
    "        {\"random\": \"random-split\", \"temporal\": \"temporal-split\"}\n",
    "    )\n",
    "    df_plot[\"trial\"] = pd.Categorical(\n",
    "        df_plot[\"trial\"], categories=[\"random-split\", \"temporal-split\"]\n",
    "    )\n",
    "    df_plot = df_plot.sort_values(by=\"trial\")\n",
    "    df_plot.to_csv(\"raw_data/4a_auc.csv\")\n",
    "\n",
    "\n",
    "else:\n",
    "    df_plot = pd.read_csv(\"raw_data/4a_auc.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'axes.labelsize': 7, 'xtick.labelsize': 7, 'ytick.labelsize': 7, 'legend.fontsize': 7, 'font.size': 7, 'font.family': 'sans-serif', 'font.sans-serif': 'Arial, Helvetica', 'mathtext.fontset': 'dejavusans', 'axes.linewidth': 0.5, 'grid.linewidth': 0.5, 'lines.linewidth': 1.0, 'lines.markersize': 3, 'pdf.fonttype': 42, 'ps.fonttype': 42, 'svg.fonttype': 'none'}\n"
     ]
    }
   ],
   "source": [
    "plt_conf = OmegaConf.load(\"nature.yaml\")\n",
    "slack = 5\n",
    "single_col_width = 89 - slack\n",
    "double_col_width = 183 - slack\n",
    "height = 89 - slack\n",
    "\n",
    "\n",
    "def mm_to_inch(mm):\n",
    "    return mm / 25.4\n",
    "\n",
    "\n",
    "print(plt_conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5v/561pm5gx1h9fgr_77mp6091m0000gn/T/ipykernel_18853/1272353011.py:5: FutureWarning: \n",
      "\n",
      "The `ci` parameter is deprecated. Use `errorbar=('ci', 100)` for the same effect.\n",
      "\n",
      "  sns.lineplot(x='fpr', y='tpr', data=df_plot, hue='trial', palette=['purple'], ci=100)\n",
      "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAC0CAYAAAB2QIL2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoOElEQVR4nO3deVyU5f7/8RfLMICAuOFGIiLiFi4cBEEFFBUpLUhFU4+SZudr5+eSx9I4R9NfKmknreybqSWSmmYe1447bmjibimigqCQqIPsqwxzff/wOMlBcUYZBvB6Ph48krm57/s9xHzmuu+5789lIoQQSJIkPcLU2AEkSap5ZGGQJKkCWRgkSapAFgZJkiqQhUGSpApkYZAkqQJZGCRJqkAWBkmSKjA3doDn1alTJ1xcXIwdQ5JqjaSkJC5dulTpz9T6wuDi4sL27duNHUOSao0hQ4Y89WfkoYQkSRVUa2HYunUr4eHhFR6PiIjAy8uLPn36cO3ateqMJEnSY1RbYZgxYwYzZ87kv+/ZOnPmDOfPnycuLo7Fixfz/vvvV1ckSZKeoNrOMfTo0YPg4GDWrFlT7vFjx44xYMAAALy8vLhw4cITt1FSUkJJSUm5xzQaTdWHlSQjU5eoUZeoKSstIzc1F6ERFN4tpEBVgLpITVZyFsVZxWjK/vj7V9oq6T6hO006Nnnu/VdbYRg2bBiHDh2q8Hhubi4tW7bUfl/ZXeALFy5k7ty55R5r165dlWWUpKqkvq8mLzWPe9fuIYQgPz2fAlUBxdnF3M+7T0luCffz7z/4KrjP/dz7FKgKKMktQV2krnTbJqYmWNpbYqp4ZNBvAi4DXGpXYXgSOzs78vLytN+bmj756GbWrFm899575R578803DZZNkh5VnFNMdnI2ub/nUnC3gKKMIgoyCsi7lUf+7XxKsksoyf3jq7Sw9LHbMVOaYWFjgUU9CxT1FFjYPPivvbM9LTxbYNXQCquGViisFZgpzbBpZoPSVokQgvqt6mNmYYbdS3aYmZsZ7LkavTD07NmTefPmMXnyZE6cOEHHjh2f+LNKpRKlUlnuscoKiST9N41Gg7pQTXFuMUWZReSn55P3ex63L9x+8E6e/+Cduyi7iJLsEopzilEXq1EXqykrKauwPXMrc6wbWVOvaT2sGlph72yP0k6Jsr4S64bWWDWxokGbBlg3ssaygSX1HOqhsFJgYmJihGevO6MVhunTpzNu3Dg8PT1xd3fH29sbgNWrVxsrklQHaDQa0s+kk3IwhZybORTeK6Q4s5gCVQHZydmU5JYgNBUPVxX1FFjaW2JRzwILWwssG1hSv1V9LOtborBWYG5pTj2Hetg52mHnaIdNMxvqNa2HhY1FjX+RPwuT2t7abciQIfICpxdUYWYhqcdSuXn0Jncv3SXv9zzybuVRqCoEwLKBJZb2llg1sMK6sTX1W9fHppmN9sVvYWuBXQs7rBpZ0dS9aZ18gT+OLq8Zox9KSJIuCjMLubTxEpmJmWQmZpKdks29hHuU3S/D1NyUxu0bY/eSHc27N6dV71a4Brti09TG2LFrLVkYpBop63oWSfuSSDuRhuqiiju/3aGspAwzpRkNXRpi29IW10GuuAx0oYVHC5R2yqdvVNKZLAyS0RXnFnMz9iZpx9O4dfoW6WfTtYcD1g7WNHZrzJ/+50+4j3anWddmmJrJE86GJguDVO3UJWqu77/O9f3XubbzGlnXsxAaganClAZtGuAa7Epzj+a09m9N05ebGjvuC0mvwpCbm0tcXByZmZk4ODjg7e2NlZWVobJJdYjQCC5vvsyZVWf4Pe53SnJKMFOa0bx7c14e/TKOXo606t0Ki3oWxo4qoWNhUKlUzJo1i8uXL+Pm5kbz5s3Jzs5m9uzZdOrUiblz59K0qazsUnnqYjXXdl3jyvYrJO1JIj89H/vW9nR4owMdQjrgMsAFMwvDXaQjPTudCsOCBQuYNWvWYxuiXL16lYULF7J06dKqzibVQnd+vcPZb8+SdiKNjMsZ3M+7j9JOiXM/ZzqP6EyHNzrIcwS1gE6FYcmSJQCsWLGCiRMnllvWrl07WRQk0s+lc3jeYa7tvIapwpRmXZvx8psv0/719rQJbIOpuSwGtYle5xh++OEHJkyYIC9DlrTS4tI4Ov8o1/59DaWdkh6Te+A32w/L+pbGjiY9B70KQ0ZGBi+99BKurq6YmJhgYmJCTEyMobJJNVjm9Ux2T95N4u5ELO0t6fleT/r8ow9KW3k9QV2gV2HYuXMnACYmJpXeHi3VXVlJWeybuY+r26+isFbQa2Yven3YCwtr+WlCXaJXYSgsLOT9998nMTGRTp06sXjxYkPlkmqY30/+zpEFR0j8ORGFtYKub3Wlzz/6YNfCztjRJAPQqzCMGzeORYsW4e3tzfHjxwkPD39s8xWp7kg9nsrRBUe59vM1LO0t8XzXE58ZPti1lAWhLtOrMNja2uLn5wdAQEAA8+fPN0goyfiEEGyfsJ3zq89j1dCK3hG98fmbD5b28qTii0CvwtC0aVOmTJmCn58fZ86cASA6OhqAP//5z1WfTqp2GrWGk1+d5PTXp7l35R4e73gw4J8D5BWJLxi9CsPD/oq//fYbFhYW9O7dm+Tk5BfmPva6LvV4KtvGb+Newj1aerVk6I9D6TSsk7FjSUagV2GwsrIq19593rx5zJ49+6nraTQaJkyYwJUrV7C1tSU6OhoHBwft8smTJ3Py5EnMzc1ZtWoV7du31yeW9JyKsovYPWU3F3+4SP2X6jNi+wjcBrsZO5ZkRDoVhjVr1hAVFcWFCxfYvXs38ODFnpubq1Nh2LJlC1ZWVhw7doyNGzeycOFC7dWUFy5c4Ndff+XEiRPs37+fuXPn8sMPPzzHU5L0Eb85nt1TdlOYUUjX8K70X9RfXpwk6VYYRo4ciZ+fH5988gkzZ84EHjRhffRdvzKPzh0RFBREZGSkdlmLFi2wtLSktLSUvLw8FArFE7cj55WoOpmJmcT8PYb4n+Jx6OxA2JYwWnq2fPqK0gtBp8IQGRnJtGnT+Prrryssy8nJYenSpcyZM+eJ6+fm5mJn9+DjLVtb23Lt4hUKBSUlJbi5uZGTk1NpLzo5r0TV+HXdr/z8l5/BBDwmehD0eRBmCnmXo/QHnQrD0KFDGT16NAqFAnd3d1q0aEF2djZxcXGUlJSwYMGCStd/dO6IvLw86tevr10WHR1NmzZtOHDgAHfu3KFfv36cPXsWS8uKw1k5r8TzUV1WsePtHaT9kkbLHi0Zvnk4ti1sjR1LqoF0KgwdO3Zk27ZtXL16lcOHD6NSqWjSpAnz58/X6R27Z8+e7N27lyFDhrBr1y58fHy0y+zt7bG1tcXU1JQGDRqgVqtRqx8/C4+cV+LZlKnLODDzAKe/Po2FrQX+H/njO8vXoBOWSLWb3h9XPsvQPTQ0lF27duHr64tCoWDDhg3aeSXefPNNjhw5gq+vL2q1mn/84x/Y2MjuvlWlQFXA2oFruX3+Np2Gd2Lg0oHYNpOjBKlycl6JOuzetXv8FPYT967e47Wo1+g0VF6TIMl5JV5YxTnFnFx2kmOfHMPMwozQtaG0f11eGyLpTq/CUFJSwpdffklaWhoDBgzA3d0dR0dHQ2WTnkHKkRT+NfJf5N3KwyXIhde+fU2eYJT0pteZu7feegtzc3NOnTqFlZUVY8eONVQuSU9CCE4sPcH6QetRWCv4c8yfGb1rtCwK0jPRqzCkp6czdepUlEolAQEB3L9/31C5JD0UZxezpu8a9kzbw0u+LzEhbgLOAc7GjiXVYnodSgghSExMBOD27duVXqUoVY9T/3uKw3MPU5JXQvD/BvOnv/xJ3tQmPTe9CsPnn3/OmDFjSEhI4I033mDZsmWGyiU9RfaNbLa9tY2UmBRa9W5F0NIgmndvbuxYUh2hV2G4ffs2v/zyi/b7HTt20K1btyoPJVXu+v7r/PjGj5iYmRD4SSA9/9ZTXuglVSmdCsPu3buJi4tjzZo1jBs3Dnhw89KPP/7I4MGDDZlPeoQQggMRBzj5+UkatGnAyB0jsW9tb+xYUh2k8yXRd+7cQalU0rp1a4QQmJqasn79ekPnk/5DXaJm/avrSd6fTPuQ9rzy9SvYNJVXiEqGodP4s1WrVowdO5YLFy7g6uqKs7MzrVq1IiEhwdD5JP741OHGoRv0/7Q/Yf8Kk0VBMii9zjGEhIRgbm7OzZs3EULQuHFjRowYYahsEg9OMq4btI6cGzmEfB9C5xGdjR1JegHodcaqoKCAbdu20b17d+Li4igqKjJULokHt0lH9Ymi4G4BYVvDZFGQqo1eIwZTU1PUajUFBQUolUqKi4sNleuFd/qb0+z/YP+DqxgP/JlmXZoZO5L0AtGrMEycOJFPP/2Ufv360bZtW/lRpYHsmLiDsyvP4uTnREh0CPVb1X/6SpJUhfQqDI+eTwgLC6vQf1F6fvve38fZlWf50//8ieCvguVVjJJR6HSO4dy5cwwdOpSJEydy7949ADZu3Ej37t0NGu5Fc3D2QY4vPo77GHdZFCSj0mnEMGHCBD755BNSU1OZOXMmBQUF3L17l4MHD+q0k6fNK7Fq1SpWrVpFaWkp77zzDhMnTny2Z1NLaco0bBmzhYs/XKTzyM68vuZ1WRQko9JpxGBjY0NgYCDh4eHs3r2bTp06sX//ftq2bavTTh6dVyI8PJyFCxdql127do2oqCiOHDnCsWPHSE9Pf7ZnUkuVqcvYGLqRixsu4vO+DyFrQ2RRkIxOpxGDmdkfTUMdHR2JiIjQayeVzSsRExND9+7dGTlyJPfu3Su37L/VtXklSotK+WHwD6QcTGHApwPo+V5PY0eSJEDHwlBcXExqaioajQYTExNSU1N52CqyVatWT12/snklVCoVsbGxHD16lLt37xIcHEx8fPxj3zXr2rwS/xr1L24cucEry1/B420PY8eRJC2dCoNSqWTs2LEIIVAqldqZrU1MTIiJiXnq+pXNK9GoUSP8/PyoV68ezs7O2NnZoVKpHjvLVV2aV+Jo5FEStiTQd0FfWRSkGkenwqDrScYnqWxeCR8fH7755htKS0vJyckhKyuLRo0aPXY7dWVeiYfXKbi95kbvWb2NHUeSKqiWLtGVzSvRpUsXRo0aRc+ePRFCsGTJknLnNOqaIx8f4ezKs3hO8iToiyBjx5Gkx5LzSlSjX5b8wt7pe+k4tCNDNw6Vnz5IRqHLa0bvcXh6ejqxsbGoVKpnDvYiOrvqLPv+to92g9sRujZUFgWpRtPrUCIqKoqVK1eSn5/P6NGjycjI4JNPPjFUtjqjOLuYAx8ewNHHkeGbh8s5I6UaT68Rw4oVKzh8+DANGzZkxowZHDhwwFC56oyy+2WsC15HSW4Jr3z1iiwKUq2gV2EwMTHB3NxcOwy2srIySKi6oqy0jE3DN5F2Io3BKwfT1L2psSNJkk70OpQYPnw4AQEBJCcnExISwmuvvWaoXHXCT2E/cXXnVQLmBdBlTBdjx5EknelVGN59910CAwOJj4/Hzc0Nd3d3Q+Wq1UqLStkxcQcJWxLo848+9Pl7H2NHkiS96HUo0aVLF7799ls6duwoi8IT3M+/z5q+a7i04RLe07zx+8jP2JEkSW96FYYLFy4QEBDA3Llz8ff3Z8WKFYbKVSvlpOYQ5R/FnfN3CF0XysDPBtbKKzMlSa+/WnNzc4KDgxkzZgx2dnZ8//33hspV65TdL2PtwLVkJWURsjaETsM7GTuSJD0zvc8xHD16lODgYBYvXoybm5uhctU6u6ft5t6Ve7z585u0DdKtT4Uk1VR6FYagoCC+/PJLOTz+L/H/iufM8jN0GdtFFgWpTtDpFf7wVufPPvuMwMBA+vbtS0BAAH379jVouNog9ZdUdozfQbNuzXjl61eMHUeSqoROI4YPPvgAeHBJ9KMKCwurPFBt8vvJ39k0bBOW9paEbQnDXFktN6tKksHpNGJ4OC3dyJEjMTU1xcTEBCEE4eHhhs5XY2UlZ7H+1fWYmpsyYvsI6r8k536Q6g6d3uKOHDnCF198weXLl7Xdm0xNTQkMDDRouJqqtKiUH179AVEmGLNvDI1cH99YRpJqK50KQ0hICCEhIezdu1fb1FUfT2sfDw96P3br1o3Y2Fhat26t9z6q067Ju8i4ksGoXaNkUZDqJJ0OJR72VXznnXdo06YNbdq0wdnZmTZt2ui0k8rax8ODwvHXv/4Va2trPeNXv6T9SZxffZ5u47vh0t/F2HEkySB0GjGsX78egOTk5GfaSWXt4wHmzp3LqFGjuHPnTqXbMXb7+MTdiWwZs4VGro0Y9PmgatuvJFU3vS5IiImJ4d///jfbt2+nbdu2REdH67ReZe3j9+zZQ1FREUOGDHnqdhYuXEj9+vXLfV27dk2fp/DMVAkqNr+5GcsGlgzbPAxzS/kJhFR36VUYPvzwQzp37sxXX33FqVOn+Pbbb3Var7L28WvWrOH48eP4+/tz/vx5RowYQU5OzmO3M2vWLHJycsp9ubq66vMUnommTMPWP2/F1NyUsTFjcehYsbW9JNUler3tWVpaYmdnh5WVFQ0aNKCsrEyn9SprH//wMAXA39+fqKiocoXjUcZqH39oziFunbrFGz+8gZ2jncH3J0nGpterysnJiT59+hAeHs78+fPp2LGjTuuFhoZSWFiIr68vy5cvJyIigunTp/Pbb789U+jqlJWSRdzncbi95kbnEZ2NHUeSqoXe7ePz8/OxsbHh9u3bNGvWzFC5dGbo9vGbhm8icXci7ya8i10LOVqQar8qbx9/4cIF+vbti7OzM4MGDeL06dPPFbCmu3PxDpf/dZkuY7vIoiC9UPQqDFOmTGHNmjUkJyezbt06Jk+ebKhcNcLe6XuxamhFwNwAY0eRpGqlV2EQQtChQwcAOnbsiEKhMEiomuDc6nNc33sdr8leWDWU3bClF4ten0rY2Njw7bff0rt3b2JjY7G3tzdQLOPKu53Hvr/t4yXfl+g1q5ex40hStdNrxLBu3TouX77MtGnTiI+PZ/Xq1YbKZVS7J++mtKiUId8NwdRMNqWRXjw6jRgKCwtZu3Yttra2LF68uE7Puxjz9xjiN8XjO9OXxu0aGzuOJBmFToVhzJgxuLq6kp2dzcWLF5k/f76hcxlFVkoWJ5acoMMbHei3oJ+x40iS0ehUGDIzM4mMjEQIQf/+/Q2dyWgOf3QYIQRBXwTV6VGRJD2NTgfQDy87fti5qS7KSc3h0o+X6Dyis7xmQXrh6TRiKCoqIjU1FY1Go/33wwLRqlUrgwasLnun78XExISAefKaBUnSqTAolUrGjh2LEAKlUqlt72ZiYkJMTIxBA1aHpL1JxP8UT8/3esqbpCQJHQvDwYMHDZ3DqA7PO0wDlwb0nS/b4UsS6HkdQ11059c7pB5Ppdu4brL9uyT9xwtfGA7//8NY2FrQ4//1MHYUSaox9C4M6enpxMbGolKpDJGnWqX+ksrlny7TfUJ3lHbKp68gSS8IvcbOUVFRrFy5kvz8fEaPHk1GRgaffPKJobIZXNzncVg2sKTvx/LcgiQ9Sq8Rw4oVKzh8+DANGzZkxowZHDhwQKf1NBoNb731Fr6+vgQFBXH37l3tMiEEkyZNolevXnh6ehq06cqjykrLSNqbhOsrriis6u5dopL0LPQqDCYmJpibm2uvCrSy0u125Mrmldi1axf5+fnExsayZ88e7QS6hnbq61MUZxXTbVy3atmfJNUmeh1KDB8+nICAAJKTkwkJCeG1117Tab3K5pUICAjA19cXeDB6MDMze+J2qmpeiTJ1GXFL4nDs6YhzP2e915ekuk6vwjBlyhQCAwOJj4/Hzc0Nd3d3ndarbF4JKysrrKysKCgoYPjw4URERDxxOwsXLmTu3LnlHmvXrp0+TwGAI/OOkJ2SzSvfyGnrJelx9DqUmDdvHps3b+by5cts3bqVefPm6bReZfNKANy5c4fAwEDCwsK0V1U+TlXMK6EuUXPqq1O4vupK2wFt9VpXkl4Ueo0YnJycgAdD/rNnz1JYWKjTepXNK5GXl0f//v1ZsGABr776aqXbqYp5JS6uv0hRZhE9p/fUaz1JepHoVRjGjh2r/fe4ceN0nvk6NDSUXbt24evri0KhYMOGDUyfPp1x48axY8cObt26xaeffsqnn34KwIEDByo91/A8zq0+R8O2DXH2l+cWJOlJ9CoMR44c0f77999/5/bt2zqtZ2ZmxnfffVfusX/+858AvPzyy3z44Yf6xHhmFzdc5ObRmwxcMrBa9ifpr6ysjNLSUmPHqBMUCsUzv8HqVRge7fFoaWlZ63o+nv76NA3bNqTHZHn5c02Un59PWlpane35Ud1MTExwdHTExsZG73X1KgwWFhZ88803eu+kJrh37R43j92kz9/7VMt8l5J+ysrKSEtLw9ramiZNmsgOWs9JCIFKpSItLQ1XV1e9Rw56FQaVSsXly5e1c0vUJie/OImpuSk9/ipHCzVRaWkpQgiaNGmi84VzUuWaNGlCSkoKpaWlhi0MSUlJBAcHa1u8mZiYcP36db12aAxCCK7suIJzX2esG1sbO45UCTlSqDrP87vUaUw9e/Zs4MHclcnJyVy/fl3739rgxuEb5NzIofNIOVu1JOlCp8IQGxtr6BwG9dv637CwsaDTsE7GjiJJT3Xjxg127doFwNSpU3X+9K8q6XQokZ6eTnR09GOXVXalYk2RFpdGS++WmFvKDk1SzXfw4EFSUlIYNGgQS5cuNUoGnV4pxcXFpKSkVPgYqTYcD2o0GrISs3AOkBc01SalhaVkJGRU6TYbt2+MwvrJt9hHRUWxY8cOsrOzuXfvHkuXLiU8PJy2bdvi4+PDkCFDmDx5Mubm5tjb27N69WrKysoICwujrKwMU1NToqOjSUpK0s7Dkp6eTkREBGFhYcTExDB79mxMTU15+eWXWbZsGYWFhYwbN460tDQ0Gg1RUVFERkZSXFyMt7c3kZGRrFy5ksGDB3PhwgWUSiXTpk2jd+/euLi4MGXKFACaN2/Od999V2UnbnUqDK1bt9aeZ6htcpJzKC0sxaGzg7GjSHrISMhghceKKt3mxDMTad69eaU/U1payv79+7l+/TqhoaGkpqZy7tw57O3t6dGjB9HR0bRv355vvvmG+fPn07dvX1566SVWrFjByZMnycrKAuDu3bvExcWRn5+Ph4cHoaGhTJo0idjYWBo3bsyUKVP46aefuHHjBp06dWLTpk2cOXOGU6dOMXPmTFJSUrR3IisUCl555RV+/vlnhgwZwv79+1m0aBG9e/cmOjqadu3asWTJEpYtW8aMGTOq5HelU2Fwc3Orkp0ZQ96dBzdv2TTX/yIPyXgat2/MxDMTq3ybT9OnTx9MTExwcXEhOzubFi1aaGd1z8jIoH379tqf27p1K4sWLSIhIYFXX32VevXqsWDBAgB8fHxQKBQ0aNAABwcHbt26RXp6OkOHDgWgoKCAli1bcv36dUJCQgDw8PDAw8ODqKioCrnGjx/P3//+dywsLBgwYAAKhYKEhAQmTnzwOyopKcHT0/N5f0VaOhWG5cuXV9kOq1thxoMbveTHlLWLwlrx1Hd3Qzhz5gwAiYmJODg4lOtt2qhRI65evUq7du04fPgwLi4uHD58GCcnJ/bt28e2bdtYtGgR48aN49y5c2g0GnJycsjMzMTR0REnJyd27tyJjY0NmzZtomXLllhYWHDq1CkGDhxIXFwcGzdupGvXrhX6jHTs2JHMzEy++eYbbT+TDh06sH79elq0aEFMTIzONzXqos6fjSvOLAbAupEsDNLT3bx5k379+pGfn89XX33F8OHDtcuWL1/O+PHjEUJgZ2dHVFQUpqamhIWF8dVXX6HRaPj0008pKCigsLCQgQMHkpWVxZIlSzAzM2Px4sUMGjQItVpNkyZNiI6OpmvXroSHh+Pv748QgpUrV1JQUMCCBQvw8PAol23UqFGsWrWKTp0efLq2bNkyRo0aRWlpKQqFgjVr1lTdL0LUcoMHD650+fHPjouP+EgUZhVWUyLpWRQVFYn4+HhRVFRktAyrV68Wc+bMee7tHDx4UIwdO/a5t/O8nvQ7fdprRggh6vxNAyU5JZiYmsj28JKkh7p/KJFTjKKeQt44JT3VuHHjqmQ7/v7++Pv7V8m2jKXOF4bm3ZvXiustpAeEvOW6yjzP77JaCoNGo2HChAlcuXIFW1tboqOjcXD447qCiIgI9u/fj1Kp5Ntvv9W7j2NlOg3rhOugqtueZBgKhQITExNUKpW87boKiP/cdm1iYoJCof+8KdVSGB6dV2Ljxo0sXLiQJUuWAA8+Hjp//jxxcXHExcXx/vvvs2XLlirbt7mlubwUuhYwMzPD0dGRtLQ0UlJSjB2nTnjYqOVZujhVyyumsnklHl3m5eXFhQsXnridqppXQqqZbGxscHV1la3dqki1tXZ7VpXNK5Gbm0vLli2131d2XFRV80pINZeZmZnBGgFLuquWU/WVzSvx6DKovB18VcwrIUnS01VLYXg4rwRQYV6Jnj17sm/fPgBOnDhBx44dn7gdpVKJnZ1duS/5MaQkVb1qOZSobF4JT09P3N3d8fb2BtC783RSUhJDhgx54nKNRsO1a9dwdXWtUUWkJuaSmXRTEzOB7rmSkpKeui0TUcc/OM7NzaV+/frk5ORoz3PUBDUxl8ykm5qYCao2V80pd5Ik1RiyMEiSVIEsDJIkVVDnC4NSqWTOnDkVZsk2tpqYS2bSTU3MBFWbq86ffJQkSX91fsQgSZL+ZGGQJKkCWRgkSaqgThUGjUbDW2+9ha+vL0FBQdy9e7fc8oiICLy8vOjTpw/Xrl0zeiYhBJMmTaJXr154enqyfft2o2d6SKVS4ejoWK23QD8t16pVq/D29sbDw4MVK6p2zolnzTR58mS8vb3p1asXCQkJ1ZLpoa1btxIeHl7h8Sr5O6/i/pNG9dNPP4lJkyYJIYTYsGGDmDp1qnbZ6dOnRXBwsBBCiBMnTojXX3/d6Jl+/vlnMWbMGCGEEPfu3RMuLi5GzySEEGVlZWL48OHC1dVVJCcnV0ump+W6evWq8PX1FSUlJaKoqEh89NFHRs90/vx54efnJ4QQYt++fWLEiBHVkkkIIf72t78JNze3Ck1nq+rvvE6NGP6778OhQ4ceu+xpfR+qK1NAQABffvkl8GD0UF23G1eWCWDu3LmMGjWKFi1aVEseXXLFxMTQvXt3Ro4cSVBQEAMHDjR6phYtWmBpaUlpaSl5eXnP1CnpWfXo0YOvv/66wuNV9Xdep1obVVXfh+rKZGVlhZWVFQUFBQwfPpyIiAijZ9qzZw9FRUUMGTKEzz77rFry6JJLpVIRGxvL0aNHuXv3LsHBwcTHxxu8BVxlmRQKBSUlJbi5uZGTk1Nth4IAw4YNq1DQoer+zuvUiKGq+j5UVyaAO3fuEBgYSFhYWLXNHF5ZpjVr1nD8+HH8/f05f/48I0aMICcnx+i5GjVqhJ+fH/Xq1cPZ2Rk7O7tys0QZI1N0dDRt2rQhMTGRixcv8vbbb1NcXGzwTJWpqr/zOlUYqqrvQ3VlysvLo3///kRERGjnIDR2pvXr1xMbG8uhQ4fo2rUrGzZsqFDMjJHLx8eHgwcPUlpaSkZGBllZWTRq1Miomezt7bG1tcXU1JQGDRqgVqtRq9UGz1SZKvs7f64zIDWMWq0W4eHhwsfHR/j5+Yn09HTx3nvviV9//VUIIcSHH34ovLy8hJeXl4iPjzd6pvnz54tGjRoJPz8/7ZdarTZqpkf5+flV68nHp+VatGiR8PDwEN27dxc7d+40eqbS0lIxfvx44ePjI3r06CGio6OrJdNDj854VdV/5/KSaEmSKqhThxKSJFUNWRgkSapAFgZJkiqQhUGSpApkYZAkqQJZGCRJqkAWBkmSKpCFoYY4dOgQzZo1w9/fX/v1pFm/U1JS8Pf3f+Z9ffTRR7Rv3x5/f3/69u1L7969uXTpkl7bmDp1Krdv3+bXX3/l+PHjAIwYMaJKMvn5+dG5c2eioqKe+PPFxcWVLv9vRUVFjBs3DoAvv/ySjh07smPHjqeu991332mvJHyhVM01WNLzevQqtqdJTk7W3u77LObMmSNWr16t/f7f//63CAsLq5JtVVWmzMxM0apVqyf+vL6/g48//ljs2bNHCCFEQECAuHr1qk7rqdVqERgYKMrKynTeV10gRww13M2bNwkODqZ///54eHgQGxtbbvmHH36Ir68vnp6erFu3Dnhwi3KvXr3o06cP77777lPvsMvKysLW1hZ4MBLw9vbGy8tLu71ly5bRs2dPvLy8WLx4MQD+/v6cOHGCqKgoIiMjOX/+PK1bt0alUtG1a1fttkNCQjh79qzeme7evYuVlRUAFy5coH///vTr1w8vLy8SEhK0+4yMjCQrK4vQ0FACAgLo168fiYmJ5bYlhODHH3+kX79+REVFcfbsWcaMGcOGDRu0t3C7u7uzceNGANzd3Xn99deZOHEiZmZmdOnShT179lSat84xcmGS/uPgwYOiadOm2nsmQkJChBBC7NmzRxw7dkwIIcTmzZvFhAkTyr1btmvXTqSlpYns7Gzx/fffC41GI9zc3IRKpRJCCDF58mTx448/ltvXnDlzhJubm/Dz8xN9+/YVo0ePFunp6WL79u1i1KhRQgghCgoKRIcOHYRKpRKenp7i0qVLorS0VCxfvlwI8cd9FI++0zs5OQkhhAgNDRVnzpwRGRkZwtPTU69MvXv3Fk5OTmLAgAHizJkzQggh1q9fLxISEoQQQvzzn/8UH3/8cbnfwYwZM8SqVauEEEKcPXtW26jkoStXrpQbXTzMfvDgQdGtWzdx//59kZmZKZydncX9+/eFk5OT+O2337Q/HxUVJWbOnKnj/8m6oU71Y6jtgoKCKhw3t2jRgvnz57N8+XKKioqoV69eueUrV65k2rRpqFQq3nzzTVQqFenp6QwdOhSAgoKCcvfnPzRz5kztMfdDly9fplevXgBYW1vz8ssvk5SUxKpVq4iMjOTGjRvaJiCVGT9+PGvXrqVNmzaMGTNG70wHDhxg6tSptGnTBgBHR0dmz56NpaUld+7c0U6A/NDFixc5fPgw33//PfDgfMKjMjIynth0xsfHB4VCQYMGDXBwcCAjIwOg3F2JzZs3155HeVHIwlDDzZ49mylTpuDn58fChQvL9RUsKSlh27ZtbNy4EbVajZOTE2PHjsXJyYmdO3diY2PDpk2bHvsifJz27duzadMm/vKXv1BYWMi5c+do3bo1kZGRrFy5EgsLCzw8PBg1apR2HVNTUzQaTbntDBw4kLlz53Lu3Dk2b96Mvb29Xpn69evHsGHDmDRpEuvXr2fKlCls2bIFJycn3nnnHYQQ5fbbvn17evXqRWhoKGlpadpDgoeaNGlCdnb2Y/d17tw5NBoNOTk5ZGZm4uDgoH1eD2VnZ2sff1HIwlDDvfHGG7z99ts4ODjg6OhIZmamdplSqcTW1hZvb28sLS159913sbCwYPHixQwaNAi1Wk2TJk2Ijo7WaV+DBw8mJiYGX19fiouL+eCDD2jatKn2hWdra4uPjw9OTk7adbp3784HH3xQ7h3WzMyMwMBAEhMTadiwIYDemWbOnEm3bt3Yv38/I0eOZMCAATRu3JimTZsihMDBwYHc3Fw++ugjIiIiGD9+PF988QX5+flERkaW21bbtm25desWQogKHZ8KCwsZOHAgWVlZLFmy5LHt9U6ePMmgQYN0+h3WFfK2a+mFMG/ePLy8vMr1ijx06BBRUVGVfuxZVlZGUFAQe/bsqbauXzXBi/NMpRfa9OnTWbt2rd7rrV69munTp79QRQHkiEGSpMd4scqgJEk6kYVBkqQKZGGQJKkCWRgkSapAFgZJkiqQhUGSpApkYZAkqQJZGCRJquD/AImWCYNJR7FSAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 264.567x165.354 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams.update(plt_conf)\n",
    "plt.rcParams[\"figure.figsize\"] = (\n",
    "    mm_to_inch(single_col_width * 4 / 5),\n",
    "    mm_to_inch(height / 2),\n",
    ")\n",
    "sns.lineplot(x=\"fpr\", y=\"tpr\", data=df_plot, hue=\"trial\", palette=[\"purple\"], ci=100)\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.xlabel(\"False Positive Rate (fpr)\")\n",
    "plt.ylabel(\"True Positive Rate (tpr)\")\n",
    "plt.savefig(\n",
    "    \"AIP_plots/4a_auc.pdf\", bbox_inches=\"tight\", dpi=300, transparent=True, format=\"pdf\"\n",
    ")\n",
    "plt.savefig(\"AIP_plots/4a_auc.svg\", bbox_inches=\"tight\", format=\"svg\")\n",
    "plt.savefig(\n",
    "    \"AIP_plots/4a_auc.eps\", bbox_inches=\"tight\", dpi=300, transparent=True, format=\"eps\"\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ds",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  },
  "vscode": {
   "interpreter": {
    "hash": "f2d120223a6515dc0c7c16162fc5a2fa4110b35f32e696807c5ea82b3cb4ddfe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}