visualization/main_figures/3a_auc_open.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from pandas.io.json._normalize import nested_to_record\n",
    "import json\n",
    "import wandb\n",
    "from omegaconf import OmegaConf\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pull_from_wandb = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pull_from_wandb:\n",
    "    doctor_path = \"/gpfs/data/oermannlab/users/lavender/NYUTron_hf/src/finetune_models/readmission/compare_with_doctors/inference/result/metric_comparison_new.csv\"\n",
    "    keep = (\n",
    "        [f\"doctor_{n+1}\" for n in range(6)]\n",
    "        + [\"nyutron_max_f1_label\"]\n",
    "        + [\"median_doctor\"]\n",
    "    )\n",
    "    doctors = pd.read_csv(doctor_path, index_col=0)\n",
    "    doctors = doctors[doctors.competitor.isin(keep)]\n",
    "    doctors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pull_from_wandb:\n",
    "    small_case_auc = pd.read_csv(\n",
    "        \"/gpfs/data/oermannlab/users/lavender/NYUTron_hf/src/finetune_models/readmission/compare_with_doctors/inference/result/20_cases_auc_new.csv\",\n",
    "        index_col=0,\n",
    "    )\n",
    "    small_case_auc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gather data df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pull_from_wandb:\n",
    "    username = \"lavender\"\n",
    "    project_name = \"eval_threshold\"\n",
    "    artifact_names = [\"temporal_auc\", \"prognostic_auc\"]\n",
    "    api = wandb.Api()\n",
    "\n",
    "    def artifact_to_df(artifact):\n",
    "        local_path = artifact.file()\n",
    "        f = open(local_path)\n",
    "        data = json.load(f)\n",
    "        df = pd.DataFrame(data=data[\"data\"], columns=data[\"columns\"])\n",
    "        return df\n",
    "\n",
    "    def combine_df(artifact_dfs, small_df, trials=[\"semi-prospective\", \"prospective\"]):\n",
    "        print(\"combining...\")\n",
    "        artifact_subs = []\n",
    "        for idx, artifact_df in enumerate(artifact_dfs):\n",
    "            artifact_sub = artifact_df[[\"fpr\", \"tpr\"]]\n",
    "            artifact_sub[\"trial\"] = trials[idx]\n",
    "            print(artifact_sub)\n",
    "            artifact_subs.append(artifact_sub)\n",
    "        small_sub = small_df[[\"fpr\", \"tpr\"]]\n",
    "        small_sub[\"trial\"] = \"retrospective-sample\"\n",
    "        print(small_sub)\n",
    "        combined = pd.concat(artifact_subs + [small_sub], ignore_index=True)\n",
    "        return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pull_from_wandb:\n",
    "    dfs = []\n",
    "    for artifact_name in artifact_names:\n",
    "        artifact = api.artifact(f\"{username}/{project_name}/{artifact_name}:v0\")\n",
    "        df = artifact_to_df(artifact)\n",
    "        dfs.append(df)\n",
    "    print(\"*****df*****\")\n",
    "    print(dfs)\n",
    "    print(\"****small auc*****\")\n",
    "    print(small_case_auc)\n",
    "    combined = combine_df(dfs, small_case_auc)\n",
    "    combined_sub = combined[combined.trial == \"retrospective-sample\"]\n",
    "    combined_sub[\"trial\"] = combined_sub[\"trial\"].replace(\n",
    "        {\"retrospective-sample\": \"nyutron\"}\n",
    "    )\n",
    "    print(combined)\n",
    "    print(\"************\")\n",
    "    legend = [\"physician\"] * 6 + [\"nyutron\", \"median physician\"]\n",
    "    doctors[\"predictor\"] = legend\n",
    "    print(doctors)\n",
    "    print(\"************\")\n",
    "    combined_sub[\"trial\"] = \"nyutron-auc\"\n",
    "    print(combined_sub)\n",
    "    doctors.to_csv(\"raw_data/3a_scatter.csv\")\n",
    "    combined_sub.to_csv(\"raw_data/3a_auc.csv\")\n",
    "else:\n",
    "    doctors = pd.read_csv(\"raw_data/3a_scatter.csv\", index_col=0)\n",
    "    combined_sub = pd.read_csv(\"raw_data/3a_auc.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.4, 0.7607843137254902, 0.6470588235294118),\n",
       " (0.9882352941176471, 0.5529411764705883, 0.3843137254901961),\n",
       " (0.5529411764705883, 0.6274509803921569, 0.796078431372549),\n",
       " (0.8980392156862745, 0.7686274509803922, 0.5803921568627451)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "palette = sns.color_palette(\"Set2\", 9)\n",
    "palette = [palette[0], palette[1], palette[2], palette[6]]\n",
    "palette"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'axes.labelsize': 7, 'xtick.labelsize': 7, 'ytick.labelsize': 7, 'legend.fontsize': 7, 'font.size': 7, 'font.family': 'sans-serif', 'font.sans-serif': 'Arial, Helvetica', 'mathtext.fontset': 'dejavusans', 'axes.linewidth': 0.5, 'grid.linewidth': 0.5, 'lines.linewidth': 1.0, 'lines.markersize': 3, 'pdf.fonttype': 42, 'ps.fonttype': 42, 'svg.fonttype': 'none'}\n"
     ]
    }
   ],
   "source": [
    "plt_conf = OmegaConf.load(\"nature.yaml\")\n",
    "slack = 5\n",
    "single_col_width = 89 - slack\n",
    "double_col_width = 183 - slack\n",
    "height = 89 - slack\n",
    "\n",
    "\n",
    "def mm_to_inch(mm):\n",
    "    return mm / 25.4\n",
    "\n",
    "\n",
    "print(plt_conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUkAAAFJCAYAAAAWit+oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+x0lEQVR4nO3dfXhU9Z3//+c5c3+TSSA3BCGI4U5EuSvoKmJb3VJt69ZAVVzXtm69FLu1a7XeoBertlpAFLutWrHqsloX1Ktqf23dr1K1hdUKKt7LjdwESLjJhNxM5sxM5pw55/fHJAMhyeRukkky78d1ealzZuZ8JmFefM55n/N5K5ZlWQghhOiQmu0BCCHEYCYhKYQQaUhICiFEGhKSQgiRhoSkEEKkISEphBBpSEgKIUQaEpJCCJHGsArJu+66K9tDEEIMM8MqJI8cOZLtIQghhplhFZJCCJFpEpJCCJGGhKQQQqQhISmEEGlISAohRBoSkkIIkYaE5DBgWSaJuNbp9kRzOIP7stK+Xyb31dX7DfS+Mrk+ddp9xbWM76uz90voUSwzkbl9xTUsy+xwm2nEMY14xvZl6jEs0+hwm2UmSOjRjOxHQnIYsBIGjXs2YcQa220LV39ArL4ygzszCVW+jR4OttsUqdlBpGZb5nZlWTRVvUdzw4F222L1+wlXb81omGiHPyMa/KLd4/GmI4T2/R3I3L5idXsJH/yo3eNGtIHGPZs6/fL3RrzpME37N7f7WSXiGo27/4qZyFxwGdEGGndvbBeUphGnYdcbmEZmggsgEQ/TsOvNdj8ry0zQsOuNjP0lOqAh+Ze//IWlS5e2e/yhhx7i0ksv5corr6SysnIghzQsqHYnBRO/Sqjy722CMlz9AYpqx1d6esb2pag2CiadT7j6gzZBGanZgRFrwD92Tub2pSjkl59HtHZ3m6CM1e8nVldJ4JT5KIqSsf3ljTsTPXK0TVDGm46gHfqYgonnoyiZ+7r4Rp8B0CYojWgDoX3vUDDxfFSbI2P78hRNxOEtpGn/O6mgTMQ1GvdsJL/8PGwOT8b25cofg6doQpugbA3IwMn/gN2dn7F9OXxF+E+a2SYoWwPSN3o6Tn9xRvYzYCG5cuVKHnjggXZ/m3366ads27aNF154gVtvvZVVq1YN1JCGFdXuahOUqYBs+TJmkqLa2wRla0DmlZ2Z0dACUBSV/AnHgrI1IPPLMxuQyX0p5I07KxWUbQJStWV0XwD+k2YAyaBsE5B2Z8b35SmehMNbRNP+d9oGpNOX8X25CspSQWkasWMB6SnI+L4c/uJUULaGsW/0dJx5ozK2D3vG3qkL06dP58tf/jIvv/xym8e3bt3KvHnzAJgxYwbbt28fqCENO61BeXjzE3hLT6dgwpf7bV+mEcfmKSD40fOpx8IH3uu3/QFEjnx27L8Pf9Kv+zpeNLiz3/cR2vt/ABw+urvf9xWu2gpA7Oieft9X6+8sNgCfKxrcyYhTv5HRgIQBDMmLLrqIzZs3t3s8HA4zatSxDyXNG/tGO/QJnlHTMCJ1GLHGjB7eWJZFvOkwkUOfEG35Q2/3FqKoNlwFZdhceRnb14n0SB2GFgQLHP4S7N4R/bavRCxEc2MVis2F3ZWHI6+k3/Zl6lGiR/dgc/lQFDuugrH9t69EnGhwJ3Z3AVYijruwvN/2ZZkJIjXbcXhGkGhuwlMyGcjszP+4nSX35SsiWvsF7hFlKGrmom3AQrIzfr8fTTtWmVVVqSX1Vmj/FiyjGXf+STByPKHKvxMYf3afgzL55foC7dAnGJGj2Nz5uAsnYHP5CJx8DlgJGr54A0/hBBwZOg90vFj9fnTtKIWnVwAWjbs3YncHcBWUZXxf8aYjxEMHKTpjISgqTfs3o9qceIonZXxfrYfYRadfgmp3Ej74EZZppA7DM6n1ELvwtG9hc/qIBr9Aj9SSN+4fMn7aovWwd+SUBdg9BTQ3HCBau5v8Cedl9LwuHDsHmV9+Hs68UejhIA273qRg4lczFpRZT6SZM2fy9ttvA/Dhhx8yceLELI9o6LEsi9C+dzD1KM6CMhS7G9XWcTGnJ/RIPY17NnHk3f+mcfffsLvzGHnaxfjHzsHuySdw8jkoitLuHGUmnXgO8sRzlJl04jnIE89RZlJH5yCPP0eZSR2dgzz+HGUmj97aFGlazkEef46ys8uDeqNNkablEPv4c5SZukIgazPJFStWUFFRwfTp05kyZQqXXXYZAMuXL8/WkIYkyzKJh4OYLYdPqt2FaTQDx85Rxo7u6fZs0rJMYnV70Q59SryxCtXhwTf6dLyjpmF3B5KHUUc+b1ekaQ3KyJHtGZtNWpaFEW1oV6RpDUrt0Kc488dmbCakh4PtijStQRk5/AmWZWZsJhQPHeywSOM/aQba4c8xjXjGCjjxxoMdFmk8xZOgVsGMa9hc/szsK3SwwyJN66zfiNTj8BVmZl/hmg6LNK1BGW86git/TJ/3o1jD6CTgkiVLeOyxx7I9jAFjmQn0SB2JWBOKw5W6bMQ0mlEUG678k7r9Xom4RuTI52iHP8OMazjySvGVno6naGK/VHaFGCqyfk5S9I5pxNEjRzHjEVSnt1dBZlkW8dAhtMOfEDu6B0VR8RRPxld6er+cWxRiKJKQHIISehQjcjR5SOby9/hw00zEidbsRDv8aUshpoDA+HPwlpyKanf106iFGJokJIcQy7Iw42H0SB2WZaE6fT0KSD1SR+Twp0RqtmMlDNwjx5N/yryMntcTYriRkBwiLMvEiDZixOpRVAc2R/dmfJaZIFZXiXb4E+KN1S2FmOl4S6dh78frGoUYLiQkh4BjBZoQisPdrft6TT1G0/530Y4kCzHOvNEUTP4ansIJUogRogckJAe5nhZoLMtCO/gRzfX7UVRbshAz+gwcvqIBGrEQw4uE5CDWmwJN5NAnNNfvw1t6OoGT/0EKMUL0kYTkIGRZFol4GEOrw6L7BZrmhgNEgzvwjDoNT9EkCUghMkBCcpBpV6DpZtAZ0QaaDryLq2Ac7sIJ/TxKIXKHhOQgYpkGeqS+RwUaSJ63DFW+jc2Zh7/sSxldjl+IXCchOUj09g4ay7Jo2vcOVkInf8KXUVS7hKQQGSQhOQj05Q6ayKFP0MNHCPTTKtNC5DoJySzqbYGmVXP9fqLBHfhOmpHx1ZiFEEkSkv3EiIWIhw7hLJrInlAtu0JBQnqMgMPNJLeXUqsZd15pjws0qfePNtB04D1cBeNwjhhPtHYXnqL2a3Em9Cixo3vwlU7L1EcTIqdISPYTmyuPePRztn2xifdMGyoKbpudumgNocPV1BZO5AzVjt3p7XFnPNNoJlT5FjZXslCTXIFZIVKzA2/JlNTzEnqUxt1/Jf+UczP86YTIHVlfmXy4UhSFmsB4QuEgUxIapd4AJTaV2c1HOOo9iSNNtRzW9R4HpGWZLYUag8Ap56SWqPcUTUBR7URqdgDJIG0NyP7sPSPEcCch2Y92NdWyy1OK39QpDe2nrGEXQW8pLptKs+riQKTnbRWShZogeSef3X6l6dagPPI52sEPJSCFyAAJyX4U0mO47Q6O+kYxJrSPuM0FChg2By6bDc2I9+j9koWanfhOmo6zkw5+roKxxGp3oTo8EpBCZICEZD8KONwo8QhjG/aws/A0bKaBNx4GRaHZNPH1oIfJ8YUad1HHnftMo5nwgffJL/8ydm8h4eoPMvVRhMhZEpL9aJLby9RINfs8paiWSY3vJJyJZryRIBYmZb7u9Y5OFWrcLYWaDi4TMo1mwlXv4ymZgt1XSN7Y2SiqQ4JSiD6SkOwniWaNkUe3Exl1BlFdI6jrNBlxttsCEI9wqs1ijK+gy/dpU6gZf06HvYTNhE646n28paej2t3Y3fkoqg3f6NOTQZnhFqVC5BK5BKifqA43BRPPZ060gYOY7Neb0Yw4+S4vxUVljHZ7sald/x3VWqgJTOj8jhpFteMf+yUsI47N5Ud1eFLbfKNPJ6FHM/a5hMg1EpL9RFFtYMRAjzB2xGjKerEaeCxVqJmB099xoQaSlxtZgGJzYPfktzsctx0XmkKInpHD7X5imQmMaCOotl61SzCiDYQPvIdrROeFmtS+LCs5i3Tno9oy09BeCJEkIdlPjOYmTD2Kanf3+LWm0Uxob0uhZuycLu/nNvUoqtOL3eXv7XCFEJ2QkOwHptFMItaIYnf1uFVrqlBjGgTGz+u6p41pgGWlijVCiMySkMwwy7IwYiEsM4Hag+sgW2mtd9SMPxub09vl8814FJs7r02xRgiRORKSGWbqERLNTb0KrVj9fmLdKNSk9mU0J4s17kCPZ6xCiO6RkMygvhRrjEg94QPv4hpxMu4Oljxrty/LlGKNEANAQjKDelusSd5R8zZ2dz7+sR3fUdPuNXpMijVCDAAJyQzpbbEmWaj5O5aVIG/8Od2agUqxRoiBIyGZAX0p1mgHP0YP1xI4uXuFGpBijRADSUIyA3pbrInV7SNW+wW+k2bg8Bd3b19SrBFiQElI9lGyWNPQ42KNEaknXPVetws1IMUaIbJBQrKPjFhTsojSg2JNbwo1IMUaIbJBQrIPTKOZRHPHxRrLsrDMRLvXWJZJU+XfsczuF2pAijVCZIusAtRLxxdrbK725yJNPYp28CO8Y7/EwWiYw3UHyTu0G2e+j0C8Hnfx5G4XaqClWOPJl2KNEANMQrKXuirW2Jxe3CVTqd69iQ8cBYypPcSIWD12T5yg3YfmKGCWaXZrTUnTaEaxO6VYI0QWyOF2L3S3WHM4YbLN5ueMaA1joyFsBV5M3SToKmV/uI5qraHrfbUUa+xSrBEiKyQke6G7xZoDWj1Rm4uEDo4CB5aloBzVKA4fRUHlgFbf5b5aizWdrUouhOhfEpI9lCrWOLq+s0Yz4uQZOoFYQ/K1JsSdbvLrasgz4l22lJVijRDZJyHZA23urOnGoa/P7qQsVI1NsbAsaG5WUfOcOIxm8utrumwpK3fWCJF9A1K4MU2TO++8k7179+Lz+bj//vspLCxMbb/33nv5+OOPsdls3HvvvUyYMGEghtVjZrxnd9aUR49iRMOYTjtWQsGyVOJxC6XAS2FjEKdteuf7kmKNEIPCgMwkN2zYgNvtZv369SxatIg1a9aktm3fvp0dO3bw/PPPc8MNN/DII48MxJB6zDITGLGGbt9Zk4hrBJpj+MLNKA6VRHMcok0ktCZi4WbynSqjmxo63pcUa4QYNAYkJLdu3cq8efMAmD9/Plu2bEltKykpwel0ous6mqZhtw/Oq5J6emeNzenDixv/yFGoqoJiqqDacDlcjHQFKHAXocY7bvXa2rNGijVCZN+AJFI4HMbvT95K5/P50DTt2ADsduLxOBdeeCHhcJhHH310IIbUIz0p1hxPOfVMErW7ofoDRp/9HRRb1z9uK2GAhRRrhBgkBmQm6ff7U8GoaRp5eXmpbS+//DJlZWVs2LCBP/7xjyxbtozm5uaBGFa3WJaFEW3sdrHmRIYWxO4d0b2AtCxMPYrNHZBijRCDxICE5MyZM3nrrbcA2LhxI7NmzUptCwQC+Hw+VFUlPz+fRCKBYRgDMaxuMeMREvFwr0LLsix0rRaHr6h7z0/EW4o1eVKsEWKQGJDD7QULFrBx40YWL16Mw+Fg9erVrFixgoqKCr71rW/x7rvvsnjxYhKJBNdffz0+3+A4F9fTYs2JzHgEU49i93W9VmRrscbhL5ZijRCDiGJZlpXtQWTKkiVLeOyxxzL2fnqkASNyFNXl79XMLlZXSfjAu4yc9u0uVyxPxDVUuxtn3igURS5fFWKwkG9jJ3pbrDmertUmzy92EZCpYo2nQAJSiEFGvpEdaC3W0MtiTSs9HMTRxaG2ZVmYRqwlTHvWZVEI0f8kJDvQWqxR+lBhNvUYZjzcZdHGSsSlZ40Qg5iE5An6WqxppWtBAOxpGny1vbPG0et9CSH6j4TkCXrTs6YjulaL6vRhSzMbTd1ZIz1rhBi0JCSPYxrNJGJ9K9a06up8pBRrhBga5NvZIlWssfpWrAEwE3ESscZOz0dKsUaIoUNCskUmijWtDO0oAI5OzkdKsUaIoSOnQlLXjhIP17R73DITNIcOokfrM7KohB4OothdGM3h9vuyTBLNGolYSIo1QgwBORWSdk8+2sGP2gVlc+gQ2sGPcPhKMrKf5P3axeihg8SbDrfZZkQbidZsw1UwNiP7EkL0r5wKSUW1UzDxq22CUtdqadq/Gd/Y2dgcrj7vwzINjGgdDn8xvjGziDceC8pEs0bk0CcETjkXuzu/z/sSQvS/wbnCbT9qDcqGXW+SGFmOdvgTfKXTsGfoMhw9UgeWhcNXhKIo+MbMQqv+ANOI01xXiX/cmd1eFUgIkX05NZNspah28srmUr/j/+EMnIQtg7M6I1yLYnOk3lNRFLyl0wjv34wzUIorMFqKNUIMITkZkkYsRGjfOxSesYh4QxWJWChj761rQewts0hIXnsZrnqfwISvkGjWiIcOZWxfQoj+l3MhacRChCrfpmDCV7A5vXhLTyca3IkRqe/ze1uWiR45mrqIvDUgfSfNxObwkHfyWcTq9hKr39/nfQkhBkZOhaTR3JQKSNWRvIhbUW34y+ZkJCiNSD2YCRy+orYB6fSCBarNQX75/GRQNhzIxEcSQvSznApJRbG1CcjU4y1BqfTxThtdqwXVht0zAkWxpQLSskxQFBTVhqKo5JfPx+7K6/oNhRBZl1PVbZvT2+k2RbVhc/WtbYShBXF4C1FUFVCxtTb/Mk0UVUVRkv+vKCp2T0Gf9iWEGBg5NZPsT+mafllWInknj7SIFWLIkZDMkEQshJXQO2z6ZVkmiuqQS3+EGIIkJDMkuciugsM3sv1GM9Hn851CiOyQkMwQXavF7h2BonZ8mlex5dTpXyGGDQnJDLAsC6OTRXZbO/YqipyPFGIokpDMADOuYRoxHP4O7sm2LFDUTmeYQojBTUIyA3StFgC7t5PKtqJmZJ1KIcTAk5DMAD0cxObOR7V3UJwxE8lZpPSxEWJIkm9uBrQustsRyzJRbHL5jxBDlYRkH5l6FDMe7vh8JIBl9rmxmBAieyQk+6j1fGS6hXTlfKQQQ5eEZB/p4SCq04/aQZfF1st/5HZEIYaunApJy7KOBVcn23tK12rTHmorik1mkkIMYTkVkrpWS6jy7Q7DMBr8gnjoYI/ezzTiJGKNnRdtzOTCFnKNpBBDV06FpNNfjDMwul1QRoNfAODKH9Oj9zMiXZyPtExoWUNSCDE05dy311NY3iYoY/WVyceLJ/X4vfRwLarDg+rseB1Ky0pIZVuIIS7nQhKOBWXwg/8Bq3cBCe2bfrVjWSg2Rx9GKoTItpwMSYBEcxMOXzGmEetVwcZKGBiR+k7PRyafJJf/CDHU5WRIhg9+BMCIKQuwe0YQOfRxj4NSj9QBVqfnI4/vayOEGLpyLiRbA9J/0gwAnHmjsPuKehyUuhZEsTmxuQMdP+GEvjZCiKEpp0IyHq4BjgVkK1f+GOy+IuKN1d1+L6Oln01n5yOlr40Qw0NOTXOc/hKc/pIOt/Xk8h/LNNG1o3hLp3X+HMtEtbllYQshhricmklmihGtByuBw5+maCN9bYQYFgZkJmmaJnfeeSd79+7F5/Nx//33U1hYmNr+wgsv8MILL2AYBpdffjmXX375QAyr13QtCKqty97Z0tdGiKFvQGaSGzZswO12s379ehYtWsSaNWtS2yorK3nxxRf53e9+x7p16wgGgwMxpD7RtVoc3sJO76SRvjZCDB8DEpJbt25l3rx5AMyfP58tW7aktr3zzjtMmzaNm2++mWuuuYZzzz13IIbUa5ZlJYs26Q61LbOlZYPMJIUY6gbkWxwOh/H7/QD4fD40TUttq6ur4/333+fZZ5/l6NGjXHvttbzyyiuDtuCRiDViJXTsadaPTF4jKX1thBgOBmQm6ff7U8GoaRp5eXmpbQUFBcydOxev10tZWRl+v5+6urqBGFav6FotKAoOb2HnT5K+NkIMGwPyLZ45cyZvvfUWABs3bmTWrFmpbbNnz2bz5s3ouk5dXR2hUIiCgoKBGFav6OEgds/ItLNE6WsjxPAxIIfbCxYsYOPGjSxevBiHw8Hq1atZsWIFFRUVnHrqqVx88cWpivbSpUux2QbnYaplWehaLe6R47t4ovS1EWK4UKzerO4wSC1ZsoTHHnus2883YqGWdrB5XT+Z5KIY9dv/H4FTzsUZGJ3meWGc/hJsLn+3xyKEGJzkpFkP6OHkIrvpizYtl/9IZVuIYUFCsgd0LYjNXYCabo3Ilr42qPKjFWI4kG9yD6Rt+tVC+toIMbxISHZTQo9ixrX0i+wCWAlQ7dLXRohhIqemO0a0ARQFuzu/3TbTiJOIh3F4R3b82nDydslOm361sCwTW5ZaNhhmgj2hWnaFgoT0GAGHm4mBYsoDRdjlwnYheiWnpjuqw0Oo8u8YscY2j5tGnHDVe6h2V6ev1bVabK48VIc7/U6y1NfGMBNsOryLNw/tpFprQE8YVGsNvHloJ5sO78IwEwM+JiGGg9wKSbuLgolfbROUZiIZkL6TZmDrpOshHGv61aUs9bXZE6plZ2MNI11eSr0BClr+PdLpZWdjDXtCtQM+JiGGg5wKSWgblLp2lMjhz7oMSNNoJhELdXk+Mpt9bXaFgqgouE+YxbrtDlRUdoUG/+pKQgxGOReSkAzKwPizqd/+Cq4R49IGJICuHQXosrKdzb42IT2Gu5P1K902GyE9NsAjEmJ4yMmQNPUYocq/M2LqN2mu30+iOZz2+YYWRHV4UB3etM9L9rWxZ6WvTcDhJpYwOtwWSyQIdHUuVQjRoZwLSVOP0bD7rwTGn4PDOxJv6elohz5OG5S6VovDV9zlghWWZaKo9qwsbDExUIyJRczQ2zweM3RMTCYGurh0SQjRoZwKyeMD0t7SCla1OfCXzek0KK2EgRGpx97VoTZkta9NeaCIyfkl1MUjHI400dCc/HddPMLk/BLKA90YvxCinZy6TtJMNLcJyFaqzYm/bA5ms9buNXrkKGB1fRE5gAVqlvra2FUb80snMsZbkLpOstDtl+skheijnArJji4ib6XanKje9rNAXQui2JzYXOlXCrIsCxQgi31t7KqNyQWjmFwwKmtjEGK4yanD7d4wwrU4fEVdn2eUvjZCDEsSkmlYZgI9cjR906/W51omqDbpayPEMNOjac+bb77JV7/61f4ay6BjROvBMrt3p01LX5vjQ3Lr50eoD3V+feKIgJvZp8mhsRCDWY9mkk888UR/jWNQ0sO1KKodu6egy+e29rU5nsdtp7GpmYRpYppW6p+EadLY1IzHLYfmQgx2PfqW6rrOd77zHSZOnJg6R7d8+fJ+GdhgoGtB7N7C7i17Zpmo9raFn3GjAxw43ISigM9zLEC1qI7f52Tc6MCJ7yKEGGR6FJI//elPAVAUhWHUGqdDlmVhaLV4Sk7t9muUEyrbPo+DstI8dlbW4XXbUz+3SFRn8viRbYJTCDE49ehw+5RTTuHVV19lzZo1vPnmm0yePLm/xpV1iVgjlml0uX4kpO9rM250AI/bQSSWvGUwEjPwuB0yixRiiOhRSP7kJz/hjDPO4I477mDSpEnccsst/TWurNPDQVBU7J0swttGqq9N+8p262wyEtVTs8iy0jyZRQoxRPQoJG02G5dccgnl5eUsXLiQeDzeX+PKOl2rxe4d2a1Leo71ten4ua2zydqGqMwihRhiehSSJ510EmvXruWzzz7j+eefx+128+677/Luu+/21/iywrIsdC3YrUPt5AvS97VpnU1aJjKLFGKI6VHhZvPmzQDs2LEDgM8//5wXX3wRgLlz52Z4aNljxsNYRnP37tcGLNPE5kwffONGB4jGdJlFCjHEdCsk//d//5eNGze2Obw2TZNAIDAsLwHSW5p+2X2F3XxF131tfB4Hs08r7ePIhBADrVshec4551BYWEgwGKSiogIAVVW56aab+nVw2aJrtdg9I1C729ArS31thBD9r1shmZ+fz5lnnsmZZ57Z3+PpV/FwDfHGatyjp7MnVMv+ugOY0TocrjzGW3FGuv14RpSha0GcgTHdes9jfW06/lFKm1chhracui/O6S8hFjrEpzteZzMefAmdQiuBLXSQI4kYhwonMSMWxoxHurWoBXBcX5v2gdfa5nVnY01Lky471fEGDmj1VEcamF86UYJSiEEup0IS4LC3lMNH9nC6rZmwbxRF4RAOxaTKP4aQVs/oRAQXdLuyna6vzfFtXo/vYhgzdHY21jDGWyBrPwoxyOXcUmm7QkEOuIux2RxMOPo5biNG0Dcal92Bgkq06Qg2Vx6q3dWt90vX10bavAox9OVcSLa2XtVVBy4jSkKxQUvAuVQVV7yp25f+AGn72kibVyGGvpwLyYDDTUG0Fp8e5vOS2ZiKwshIDQAJQ8eTaO5e069WafraSJtXIYa+nAvJKVYzASPCbu8YUBRC7pFYqATChwkkko3Aun0ReRd9baTNqxBDX04VbuLhGgrNGPtLp1MXCtKc0Cm0mjmkeik16jnF0lEdXmxOb/fesIu+NuWBIqojDcnqdlzFbbMRSyQwMaXNqxBDRE6FpMNXTL6vmPmWyRjfiNR1kiUuP2OKysg79D52b/eDq6u+NtLmVYihL6dCsrUCbVeSrVfL3R70cBCbOw8zoVMXbcRTOKH7b9hBX5sTSZtXIYa2nDsn2RkjchSwutf0q0VHfW2EEMOLhGQLPVyLYndhc+V1/0Ud9LURQgwvEpItWteP7Oii8HQ6uh1RCDF8SEiSXFnciNT16CLydH1thBDDh4QkYETqwTK7vxI5gJXotK+NEGL4GJCQNE2TpUuXsnjxYn7wgx9w9OjRds+pq6vjvPPOo6qqaiCG1IauBVFUOzZPQbdfY5lm2r42QojhYUBCcsOGDbjdbtavX8+iRYtYs2ZNm+2mafKzn/0Mj8czEMNpR9dqsff0fGQXfW2EEMPDgHzDt27dyrx58wCYP38+W7ZsabP94Ycf5uKLL6akpGQghtOGZVkYWm3PDrVJziRVu1z+I8RwNyAhGQ6H8fv9APh8PjRNS23btGkTsViMCy64YCCG0k6iOYRlGj1b+QcAC0WVkBRiuBuQkPT7/alg1DSNvLxj1yK+/PLLfPDBB1x11VVs27aNm266iaampoEYFtBStFFU7N4RPXuh9LURIicMyPUrM2fO5K233uKCCy5g48aNzJo1K7XtwQcfTP33VVddxfLly9uEaH8zovXYvYU9Cryu+toIIYaPAZlJLliwgGg0yuLFi1m/fj1LlixhxYoVqf7d2WJZFka0vsfnI9P1tRFCDC+K1XpV9DCwZMkSHnvssU63J+IaKCo2R7KKHms4QN1n/x+B8vNw+Iow9Sg2l7/L/ZhGM4qi4gyc1OM7dIQQQ0tuXb9iWTTu/isJPQqA3nQEULC7CwgfeA/L7HgV8XZvYyZQbA4JSCFyQE6FpM3lJ/+Uc1NBGQ/XYHPloR38EE/JFOzdvZjcksq2ELki5yoPNldeKij1psOA2rOABLCsTvvaCCGGl5yaSbayufLIGzsXU4/iDIzuUUCm+tpIZVuInJCTIWmZBuHqrfjLzsSIhTCN5p68ONnXRirbQuSEnJsOWaZBwxdv4B8zK7nIrsNDuOp9/GO/hGp3df36LvraCCGGl5yaSR4fkA5/8jZE1eHBd9JMwlXvd29GaSaXSJOQFCI35FRIGrHGNgHZyub04jtpJolYqMv3kL42QuSWnDrcdngLO91mc3az37b0tREip+TUTDIzLBQlp/5uESKnSUj2QPIOTkXORwqRQyQke0L62giRcyQke0D62giReyQke0L62giRc+Tb3gPS10aI3CMh2SOy+o8QuUZCsiekr40QOUdCspukr40QuUlCsruksi1ETpKQ7CbLSiQDUpZIEyKnSEh2k/S1ESI3SUh2l2VKZVuIHJRTIRkP1xAJ7uxwmx4O0txY3fmLLaSvjRA5KKdC0uErxojUtQtKPRwkVr8PZ2B0h6+TvjZC5K6cCklFUcgbd1aboNQjdcTq9+EfO7vz2w2lr40QOSunQhLaBmXTvs3EQwfTByTS10aIXJZzIQnJoHQVjCNauxO7Z0TXC1ZIXxshclZOhmRz40GiNdspnn0lZlwjVr8/7fOlr40QuSvnQrI1IPMnfgVFUXEXTSIRC6UPSulrI0TOyqmQ1LXaNgEJyUNvb+k0ErEQ8dChTl4pfW2EyFU59c23e0a0CchWrUGJZbZ7jfS1ESK35VRIpgs6RVE6vi9b+toIkdNy6nC7N6SvjRC5TUKyK9LXRoicJt/8LkhfGyFym4Rkl6SvjRC5TEKyK9LXRoicJiGZhvS1EUJISKZhmQmpbAuR4yQk07FM6WsjRI6TkExD+toIIQbkZJtpmtx5553s3bsXn8/H/fffT2FhIZC87e+ee+5hx44dxONxfvjDH3LBBRcMxLC6ZpkoqixsIUQuG5CQ3LBhA263m/Xr1/PKK6+wZs0a7rjjDgA2btxIJBJh3bp1NDQ0cOmllw6ikATVJofaor1EIoGu69kehugmh8OBrZff5QEJya1btzJv3jwA5s+fz+OPP57adtZZZzF79mwgOatU1cFxBkD62ojOhMNhqqqqWhY/EUOBoiiMHTsWv9/f49cOSAKEw+HU4Hw+H5qmpba53W7cbjeRSIQbb7yR66+/vt/GYRpxUBTUDhbQtSwLKxFHtbtaHjCTq5FL0UYcJ5FIUFVVhdfrpbi4WM5XDwGWZREMBqmqqmLSpEk9nlEOSEj6/f5UMGqaRl5eXpvttbW1/Nu//RsLFy7kkksu6bdxmHGN0IEtFEw8v01QWpaFVr0VV8E4VH9xy2MmqGqfL/8xzAR7QrXsCgUJ6TECDjcTA8WUB4qwy6VFQ46u61iWRXFxMR6PJ9vDEd1UXFxMZWUluq73OCQH5Nh25syZvPXWW0DyHOSsWbNS28LhMFdffTVLlizh8ssv79dx2L0jyBs7h4Zdb2AmkueTWgPSmT8WR0tAAhnpa2OYCTYd3sWbh3ZSrTWgJwyqtQbePLSTTYd3YZiJvn4kkSUygxxa+vL7GpCZ5IIFC9i4cSOLFy/G4XCwevVqVqxYQUVFBW+++SY1NTU89dRTPPXUUwCsXbu21ydZu+LwFaaC0j92DpEjn+MuPAVn3qg2z7OsBKrN26d97QnVsrOxhpEuL+7jZq4xQ2dnYw1jvAVMLhiV5h3EUCZHEcPDgISkzWZj+fLlbR67/fbbAZgyZQpLliwZiGGkJIPySwQ/XI9vzOx2AQmAZfV59Z9doSAqSpuABHDbHahxlV2hoITkMNV6FLGzsablz4Cd6ngDB7R6qiMNzC+dmLGgPP/883njjTd69dr77ruPa6+9luLi4h5tyyU5Wbq1LBPt8Kf4x86hubEK98iTO7g/u+99bUJ6DLet4/dw22yE9Fif3l8MXkPlKOLOO+/s1bZcknMhaVkmjXs24imcgM2dj2p30XTgPfLK5qSCMnlpR9+LNgGHm+p4Q4fbYokEhe6eX44ghob+Oop48cUXefPNNwmFQjQ0NHDHHXekrgw5dOgQ5eXl3HfffVx00UX8/ve/x+/38/DDDzNu3DhCoRB//OMfMU2Tr3/961xzzTVcddVVLF++nHg8zrJly9B1nZEjR7J69Wquu+46li9fTjgcZuXKlZimSSQSYcWKFdTW1vLEE09gt9vZt28fV199NZdeemmmfnyDyuC4KHGAHB+QroIyAGyuPLwlU2k68B6WabQ8MZFcibyPITkxUIyJRcxoe9FxzNAxMZkYyO3DmOGsP48idF1n7dq1/PrXv+a+++5D0zSWLl3K+vXr+fDDD6mrq+PCCy/k1VdfxbIs3njjDRYsWMDLL7/Mfffdx7p169pdL7hy5Up+/OMf8/zzz3PRRRexa9eu1LZdu3axbNky/vu//5tvfOMbvPbaa0DyqpRHHnmEp556iieffLLXn2ewy6mZpKEdbROQreyefLwlU9HDQZyB0Rnra1MeKKI60pA8LxVXcdtsxBIJTEwm55dQHijq0/uLwas/jyLmzp2LoiiMGzeOpqYmioqKGDUqOSstKioiFouxaNEili1bxsknn8wZZ5yB2+3m3nvv5fHHH+fgwYOpmzta7du3L3XVybe//e0220pLS/nVr36Fy+WitraWmTNnAjB58mRUVaW0tJR4PN7rzzPY5VRItrnE5wR2Tz548oFkZVuxufrc18au2phfOpEx3oJUhbPQ7ZcKZw6YGCjmgFZPzNBx29uek+zrUcRnn30GJINt5MiR1NfXt3vOuHHjAHj66ae55pprgOSh+r333ovD4WDhwoVcfPHFqeeXl5fz8ccfM2fOHNauXZsKXUgWcB5++GHGjBnDf/zHf6TuNMqVy6ByKiS7zTRRXZlp2WBXbUwuGDUoTtKLgdOfRxEHDx7ke9/7HpFIhLvuuosbb7yxw+dVVFTw5JNPMn369OSYysu54oor8Pl8zJo1izFjxqSee8stt3DXXXdhWRYFBQWsWrWK//mf/wHgm9/8Jv/6r//KiBEjKCoqyrnbMRVrGH3iJUuW8Nhjj3X7+UYshB4OYnO3vQMo0RzG4SvC7g5keohiiIvFYuzdu5dTTjkFt9ud9rn9cZ3kiy++SHV1NTfccEOXz/39739PU1MT3//+93u1r+GkJ7+3E8lMshPSskH0VTaPIlatWsWHH37YZjEZ0TuSBCewLBNQpGWDGJQWLlzYrefdcsst/TyS3JFTlwB1h/S1EUIcT0LyRNLXRghxHAnJE0hfGyHE8SQkTyR9bUSGWVoj5vuvYWmNWR3H9u3b2bp1a1bHMBRJSJ7IslA7uZ1MiN6w9m/HOrADa//2rI5jw4YNVFZWZnUMQ5GkwXGSfW2UPt+zLUQrS2vEOrAdFAXrwHascaei+PJ7/X4vvvgif/3rX4nFYlRVVXHTTTfx6quvsmrVKizLoqKiggcffJC7776bZ555Bkgupfbcc8/x0ksv4XQ6mTp1KrfeeitlZWUUFhZy7bXXcuedd2KaJg6Hg5///OcA/PSnP6WkpITKykoWLFjAj370o4z8TIYaCcnjSV8bkWHW/u0QbYKiMVBbjbV/O8rUs/r0noZh8Pjjj/Pee+/x6KOPcujQITRN4/PPP2f69Om4XK52rykuLqaiooIxY8YwdepUNE3jxhtvZPLkydxwww1cd911zJs3j02bNrFy5Upuu+02qqqqWLt2LYqicN555+VsSMrh9nEy1ddGCDhuFukNJNcB8AaSs8k+nps89dRTgeTCE7quc+GFF/Laa6/x0ksvsWjRovbj6OSmuokTJwKwe/du5syZAyQXz2hdAWj8+PG43W5cLhc+n69PYx7KJCSPl4G+NkK0Ss0ivS23t3oDEG3q87nJE6+8WLhwIX/+85/Zu3cvM2bMwOVyUVNTg2VZHDlyhNra2tTrTNNMva61fXN5eXmqoLNly5bU4hhyhUdSTh1u6+EgphFrt1QagBFrxIjU4R45IQsjE8NN21lkMmwURcFqnU328dzk8crKytB1na997WtA8tB67ty5LFq0iPLy8lToTZs2jQceeCA1g2x12223sWzZMh5++GEUReHee+/NyLiGi5xa4MKyTBp3/w1P0URcBWWpBS4sTCJHtuEtnoIzMFoWthCd6u5CCea2zVjb3gb/SDh+QmYB4TqUqeeg9vHc5PGuvvpqHnjgAQoLCzP2nsNJXxa4yKnDbUVRyZ/wZaK1u2huOABAormJyJFt5I2dg2KzS9FGZEa0CfJLwGYH9bh/bPbk49GmjOxG0zQqKir40pe+JAHZT3LqcBuOBWXj7r+hOn00NxwgcPLZLZf9SNFGZIY6+x8HZD8+n4+XXnppQPaVq3JqJtlKUVR8o6cTPvAuzsBJKDZ7xvraCCGGl5wMST1ylKaq9yiaeTl6uIZ405GM9bURQgwvOReSeuQoTQfeo2Di+ag2J95R04g3VBFvOgKqvc99bYQQw0tOJYIRqT8uIJM9bBRFwTd2NvFQNUasfUMlIURuy6nCjer0tQnIVoqi4B01Dbsrr5NXCtEzWz8/Qn2o897aIwJuZp+WneZwv/71r1O3J27atIlrr702Y++9efNmXnrpJVasWNGr1//kJz/hoYce6vG2/pRbIWnvfAk0RVGwuXrfC1mI43ncdiqrm/H7HCjHXShpYRHWdE4qyf6ftalTpzJ16tRsD6ONdCGYjYCEHAvJzkhfG5Fp40YHOHC4CUUBn+fYkYsW1fH7nIwb3bsbFlpXAYpEItTV1XHFFVewYcMGqqur+dWvfsWECRO4//77+eCDDzBNk+uvv56vfOUrbNmyhRUrVhAIBEgkElRUVLSZ9a1cuZIdO3bQ2NjIOeecw80338xVV13Fqaeeyvbt2zFNk0cffZT8/GN3Cd1+++0oikJ1dTWGYaRCbNeuXVxzzTXU1tZy6aWXcvbZZ7Ns2TKeffZZAL73ve+xYsUKfvnLX1JVVUVzczM33XQT55xzDueffz5vvPEGf/nLX/jNb36DaZqce+653Hzzzaltf/rTn3juuecwTROPx8Ojjz7KmjVrOHDgAHV1dRw5coRf/OIXnHHGGX34DR6TU+ckOyN9bUSm+TwOykrziET11AITlmURieqUlea1Cc6eisfjPPHEE/zTP/0Tb7/9No8//jiLFi3i9ddf529/+xvBYJB169bx1FNP8eCDDxKPx7nvvvv49a9/zdq1a9sEHUAoFGLUqFE89dRTrFu3rs11l2eddRbPPPMM48eP5//+7//ajWXSpEk8/fTTfOc732HNmjUAJBIJfvOb3/Db3/6W3/3ud5SXl2NZFgcOHKC6uhqHw0FeXh6fffYZjzzyCP/5n/9Jc3Nz6j0Nw2D58uU8+eSTvPTSS9jtdsLhcGp7VVUVTz75JM8++yx2u51PPvkEgPz8fJ544gmuueYaXnjhhV7/fE8kM0mQvjaiX7TOJiMxA5/HQSRm4HE7ej2LbHXaaacBEAgEKC8vByAvL4/Dhw+zc+dOPv74Y6666iogGThHjhyhsbGRMWPGADBr1qw27+fxeDh06BA333wzPp+vTWBNmTIFSK44dPzjrebOnQvAjBkz+POf/wzA5MmTcTgcFBcXp16zcOFC/vCHP6AoCgsXLsTv93PjjTeydOlSYrEYV155Zeo96+vrKSoqoqCgAIB///d/b7PPESNGcMstt+D1ejl48CCGYQBtV0eKx+M9+pmmIzNJpK+N6B8nziYzMYuE9KvzlJeXM3/+fJ555hn+67/+iwsvvJCSkhLy8/PZv38/AJ999lmb12zcuJFQKMSDDz7IddddRzQaTc1+u/pOtL7Xhx9+mFo4o6PXXHTRRWzatIm3336bf/zHf+TIkSPs3LmT3/zmNzz00EMsX7489dzCwkLq6+tTs8ebb76Z6upqAJqamvjtb3/LL3/5S+6++26Abo+1t2QmCdLXRvSb1tlkbUM0I7PIrpx//vls3ryZK6+8knA4zCWXXILL5eLuu+/mJz/5CX6/H6ez7Z/1M844g0ceeYTLL78ct9vNuHHjqKmp6db+Xn/9dV555RUcDgerVq3iiy++6PB5Pp+PCRMm4PV6cTqdlJSUcPDgQRYvXoyqqvzwhz9MPVdVVW6//XZ+8IMfoCgK8+bNS82C/X4/U6dOZeHChbhcLgoKCggGg738aXVPTq0CdKLWVYBQwOkfJdVt0aXerCazbc9RduytY8opI5laPnwWobj99tupqKjgrLO6t5rRbbfdxve///2sVNRlFaA+k742ov+MGx1g3Oi8fp9FDmbf/e53U/11hho53G451JYl0kR/8XkczD6tNNvDyLieXDD+9NNP9+NI+lfOzySlr40QIp2cD0lA+toIITolIQkoNqlsCyE6JiGpqGnv6RZC5LacCknLTLTcp32Moqooig3LsrBMI0sjE6L/bd++PdU6VnRfToWkEa2ncfff2gSlothAUWk6sIXmhqosjk6I/rVhwwYqKyuzPYwhZ0AuATJNkzvvvJO9e/fi8/m4//7723R2e+ihh3j77bdxOp3cd999jB8/vl/G4fAV4SmeTOPuv5E/4csAWCiED36IwzsS98j+2a8YvoxYI6aRwfuE7U7s7s77cbeuAhSLxaiqquKmm27i1VdfZdWqVViWRUVFBQ8++CB33303zzzzDJC8C+e5557jpZdeSl2reOutt1JWVkZhYSHXXnstd955J6Zp4nA4+PnPfw7AT3/6U0pKSqisrGTBggX86Ec/ajOWgwcPctddd2EYBo2Njdxxxx2UlpaydOnSNvt+44032LNnD8uWLUPXdUaOHMnq1avxer0Z+7n1pwEJyQ0bNuB2u1m/fj2vvPIKa9as4Y477gDg008/Zdu2bbzwwgt89NFHrFq1ikceeaTfxuLKT97e1Lj7b/jGzCJW+wXOgrF4S6b02z7F8JTQo9S8/yzJZtqZojDqzKuxOTydPsMwDB5//HHee+89Hn30UQ4dOoSmaXz++edMnz4dl8vV7jXFxcVUVFSkFtvVNI0bb7yRyZMnc8MNN3Ddddcxb948Nm3axMqVK7ntttuoqqpi7dq1KIrCeeed1y4k9+zZw/XXX8/s2bN57bXXePnll1myZEmHY165ciU//vGPOeuss/jDH/7Arl27mD59et9+VANkQEJy69atzJs3D4D58+fz+OOPd7htxowZbN++vd/H0xqUwa3P4h87F1/Jqf2+TzH82BweSr50ZcZnkukCEtqudqPrOhdeeCGvvfYa7777Lpdffnm753d253HrghS7d+9mzpw5QHJVn1/84hcAjB8/PnULn8/nQ9O0VAh+/etf58wzz+Sxxx5j/fr1xGKxDmeGrfvet29favWhb3/72+l/CIPMgIRkOBzG70/eF936wz5+26hRx5axH4hbyS3LormxCk/RJIxYPZZlSgMw0SvpDo37y4mr3SxcuJB77rkHTdOYMWMGwWCQmpoaLMuipqaG2tra1OtM89j5eFVN/pkvLy9n69atnH322WzZsoVx48Z1uB+fz5c6jAb40Y9+xHe/+13OPPNM1qxZw549e3C5XB3uu7y8nI8//pg5c+awdu1aRo0axUUXXZT5H04/GJCQ9Pv9qWDUNI28vLwOt8GxX1x/sSyLpgNbsLsLCIw7i+bG6tQ5SglKMRSVlZWh6zpf+9rXgOSh9dy5c1m0aBHl5eWp0Js2bRoPPPBAagbZ6rbbbmPZsmU8/PDDKIrCvffe2639LliwgGXLljFy5EhKS0tpbGzsdN+33HILd911F5ZlUVBQwKpVqzL4E+hn1gB45ZVXrHvuuceyLMv605/+ZP3sZz9Lbfvoo4+s6667zrIsy/rggw9S/90bXb3WNE2rcd87lnZke5vHYw1VVv0Xb1immej1vkVuiEaj1ueff25Fo9FsD6WN73//+1ZtbW22hzFo9eX3NiAzyQULFrBx40YWL16Mw+Fg9erVrFixgoqKCqZPn86UKVO47LLLANosvplpuhbE7i5oV6RpPUcZq6vEU1jeb/sXItM0TeNf/uVfuOCCC9pcMSIyJ6fXkxSip/qyLqHIHllPUogBNozmFjmhL78vWU9SiB5wOJK9kILBIMXFxdIXaQiwLItgMIiiKDgcPe8vJCEpRA/YbDbGjh1LVVWV3OI3hCiKwtixY7HZer4kooSkED3k9/uZNGkSuq5neyiimxwOR68CEiQkhegVm83W6y+dGFqkcCOEEGlISAohRBrD6jrJb37zm5SVlWV7GEKIIWLUqFHcc889aZ8zrEJSCCEyTQ63hRAiDQlJIYRIQ0JSCCHSkJAUQog0JCSFECINCUkhhEhDQlIIIdKQkBRCiDQkJIUQIg0JSSGESENCUggh0pCQFEKINHIiJE3TZOnSpSxevJgf/OAHHD16tM32hx56iEsvvZQrr7xy0C7Jn+4zWJbF3XffzRVXXMGiRYt4/fXXszjSznX1ewCoq6vjvPPOo6qqKgsjTK+r8b/wwgtcdtllLFy4kOeeey5Lo0yvq89w7733ctlll3HFFVewe/fuLI2ye/7yl7+wdOnSdo9n+vucEyG5YcMG3G4369evZ9GiRaxZsya17dNPP2Xbtm288MIL3HrrraxatSqLI+1cus+wceNGIpEI69at48knn2TFihVZHGnn0n0GSH6Bf/azn+HxeLI0wvTSjb+yspIXX3yR3/3ud6xbt45gMJjFkXYu3WfYvn07O3bs4Pnnn+eGG27gkUceyeJI01u5ciUPPPBAuy6I/fF9zomQ3Lp1K/PmzQNg/vz5bNmypcNtM2bMYPv27VkZY1fSfYazzjqLZcuWAclZpaoOzl9rus8A8PDDD3PxxRdTUlKSjeF1Kd3433nnHaZNm8bNN9/MNddcw7nnnputYaaV7jOUlJTgdDrRdR1N07DbB293l+nTp3P33Xe3e7w/vs+D96eQQeFwGL/fD4DP50PTtDbbRo0alfr/wbq8ZrrP4Ha7cbvdRCIRbrzxRq6//vpsDTOtdJ9h06ZNxGIxLrjgAtauXZulEaaXbvx1dXW8//77PPvssxw9epRrr72WV155ZdC1nE33Gex2O/F4nAsvvJBwOMyjjz6arWF26aKLLmLz5s3tHu+P7/PgnHJkmN/vT/1h0DSNvLy8DrcBg3YWlu4zANTW1nL11VfzjW98g0suuSQLI+xaus/w8ssv88EHH3DVVVexbds2brrpJpqamrI11A6lG39BQQFz587F6/VSVlaG3++nrq4uW0PtVFe/g7KyMjZs2MAf//hHli1bRnNzc7aG2iv98X0enImQYTNnzuStt94CkufvZs2a1Wbb22+/DcCHH37IxIkTszLGrqT7DOFwmKuvvpolS5Zw+eWXZ2uIXUr3GR588EHWrVvHM888w9SpU1m9enW7vwiyLd34Z8+ezebNm9F1nbq6OkKhEAUFBVkaaefSfYZAIIDP50NVVfLz80kkEhiGka2h9kp/fJ9z4nB7wYIFbNy4kcWLF+NwOFi9ejUrVqygoqKC6dOnM2XKFC677DIAli9fnuXRdizdZ3jzzTepqanhqaee4qmnngJg7dq1g67labrPMGXKlGwPr0vpxn/qqady8cUXp/6SWrp06aD7+UP6z/Ctb32Ld999l8WLF5NIJLj++uvx+XzZHnK39Of3WXrcCCFEGjlxuC2EEL0lISmEEGlISAohRBoSkkIIkYaEpBBCpCEhKYQQaUhICiFEGhKSYlgxDIOrrrqKf/7nfyYej2d7OGIYyIk7bkTuqKmpQdd11q9fn+2hiGFC7rgRw8oPf/hDtmzZwplnnkl+fj7V1dUYhsFDDz1EZWUlDzzwAHa7nWuuuYYLLrgg28MVQ4Acboth5Y477mDq1KkEAgEmTZrE008/zXe+853U4rKJRIJ169ZJQIpuk5AUw9bcuXOB5OKr+/btAxi0qzyJwUtCUgxbn332GdB2yazBul6oGLykcCOGrddff51XXnkFh8PBqlWr+OKLL7I9JDEESeFGDEu33347FRUVnHXWWdkeihji5NhDCCHSkJmkEEKkITNJIYRIQ0JSCCHSkJAUQog0JCSFECINCUkhhEhDQlIIIdKQkBRCiDQkJIUQIo3/H4BIcNU0HQ37AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 330.709x330.709 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set(font_scale=3, style=\"white\")\n",
    "marker_size = 30\n",
    "plt.rcParams.update(plt_conf)\n",
    "sns.lmplot(\n",
    "    x=\"fpr\",\n",
    "    y=\"tpr\",\n",
    "    data=doctors,\n",
    "    hue=\"predictor\",\n",
    "    fit_reg=False,\n",
    "    markers=[\"o\", \"^\", \"v\"],\n",
    "    legend=False,\n",
    "    palette=palette[:3],\n",
    "    scatter_kws={\"s\": marker_size, \"alpha\": 0.5},\n",
    "    height=mm_to_inch(height),\n",
    "    aspect=1,\n",
    ")\n",
    "ax = sns.lineplot(\n",
    "    x=\"fpr\",\n",
    "    y=\"tpr\",\n",
    "    data=combined_sub,\n",
    "    hue=\"trial\",\n",
    "    palette=[palette[3]],\n",
    "    errorbar=lambda x: (x.min(), x.max()),\n",
    ")\n",
    "sns.scatterplot(\n",
    "    x=\"fpr\",\n",
    "    y=\"tpr\",\n",
    "    data=combined_sub,\n",
    "    hue=\"trial\",\n",
    "    palette=[palette[3]],\n",
    "    marker=\"x\",\n",
    "    legend=False,\n",
    "    s=marker_size,\n",
    ")\n",
    "plt.rcParams.update(plt_conf)\n",
    "plt.rcParams[\"figure.figsize\"] = (\n",
    "    mm_to_inch(double_col_width / 3 * 2),\n",
    "    mm_to_inch(height / 2),\n",
    ")\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "leg = ax.legend(handles=handles, labels=labels, loc=\"lower right\")\n",
    "plt.savefig(\n",
    "    \"AIP_plots/3a.svg\", bbox_inches=\"tight\", format=\"svg\", dpi=300, transparent=True\n",
    ")\n",
    "plt.savefig(\"AIP_plots/3a.eps\", bbox_inches=\"tight\", format=\"eps\")\n",
    "plt.savefig(\"AIP_plots/3a.pdf\", bbox_inches=\"tight\", format=\"pdf\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  },
  "vscode": {
   "interpreter": {
    "hash": "f2d120223a6515dc0c7c16162fc5a2fa4110b35f32e696807c5ea82b3cb4ddfe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}