p_values_inhospital_mortality.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "task = 'inhospital_mortality'\n",
    "src = Path('results')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ViTiMM C</th>\n",
       "      <th>ViTiMM M</th>\n",
       "      <th>ViTiMM X</th>\n",
       "      <th>ViTiMM E</th>\n",
       "      <th>ViTiMM C|X</th>\n",
       "      <th>ViTiMM C|M|X|E</th>\n",
       "      <th>y_true</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hadm_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30008792</th>\n",
       "      <td>0.193546</td>\n",
       "      <td>0.475056</td>\n",
       "      <td>0.822056</td>\n",
       "      <td>0.476070</td>\n",
       "      <td>0.614302</td>\n",
       "      <td>0.305298</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30009123</th>\n",
       "      <td>0.015755</td>\n",
       "      <td>0.291267</td>\n",
       "      <td>0.103424</td>\n",
       "      <td>0.140598</td>\n",
       "      <td>0.049464</td>\n",
       "      <td>0.013982</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30009597</th>\n",
       "      <td>0.742484</td>\n",
       "      <td>0.500244</td>\n",
       "      <td>0.746755</td>\n",
       "      <td>0.615504</td>\n",
       "      <td>0.620785</td>\n",
       "      <td>0.944517</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30009753</th>\n",
       "      <td>0.747420</td>\n",
       "      <td>0.468689</td>\n",
       "      <td>0.913119</td>\n",
       "      <td>0.607680</td>\n",
       "      <td>0.417776</td>\n",
       "      <td>0.886910</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30022756</th>\n",
       "      <td>0.127118</td>\n",
       "      <td>0.411215</td>\n",
       "      <td>0.176093</td>\n",
       "      <td>0.466201</td>\n",
       "      <td>0.050603</td>\n",
       "      <td>0.149327</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39970416</th>\n",
       "      <td>0.071834</td>\n",
       "      <td>0.700435</td>\n",
       "      <td>0.371966</td>\n",
       "      <td>0.578458</td>\n",
       "      <td>0.070380</td>\n",
       "      <td>0.042467</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39972274</th>\n",
       "      <td>0.072215</td>\n",
       "      <td>0.231143</td>\n",
       "      <td>0.675282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.160729</td>\n",
       "      <td>0.135811</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39972327</th>\n",
       "      <td>0.170641</td>\n",
       "      <td>0.159267</td>\n",
       "      <td>0.105504</td>\n",
       "      <td>0.266358</td>\n",
       "      <td>0.048591</td>\n",
       "      <td>0.011602</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39982305</th>\n",
       "      <td>0.810775</td>\n",
       "      <td>0.494039</td>\n",
       "      <td>0.061346</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.287964</td>\n",
       "      <td>0.085535</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39983674</th>\n",
       "      <td>0.058178</td>\n",
       "      <td>0.538133</td>\n",
       "      <td>0.048904</td>\n",
       "      <td>0.162350</td>\n",
       "      <td>0.042707</td>\n",
       "      <td>0.020068</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1373 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          ViTiMM C  ViTiMM M  ViTiMM X  ViTiMM E  ViTiMM C|X  ViTiMM C|M|X|E  \\\n",
       "hadm_id                                                                        \n",
       "30008792  0.193546  0.475056  0.822056  0.476070    0.614302        0.305298   \n",
       "30009123  0.015755  0.291267  0.103424  0.140598    0.049464        0.013982   \n",
       "30009597  0.742484  0.500244  0.746755  0.615504    0.620785        0.944517   \n",
       "30009753  0.747420  0.468689  0.913119  0.607680    0.417776        0.886910   \n",
       "30022756  0.127118  0.411215  0.176093  0.466201    0.050603        0.149327   \n",
       "...            ...       ...       ...       ...         ...             ...   \n",
       "39970416  0.071834  0.700435  0.371966  0.578458    0.070380        0.042467   \n",
       "39972274  0.072215  0.231143  0.675282       NaN    0.160729        0.135811   \n",
       "39972327  0.170641  0.159267  0.105504  0.266358    0.048591        0.011602   \n",
       "39982305  0.810775  0.494039  0.061346       NaN    0.287964        0.085535   \n",
       "39983674  0.058178  0.538133  0.048904  0.162350    0.042707        0.020068   \n",
       "\n",
       "          y_true  \n",
       "hadm_id           \n",
       "30008792       1  \n",
       "30009123       0  \n",
       "30009597       1  \n",
       "30009753       0  \n",
       "30022756       0  \n",
       "...          ...  \n",
       "39970416       0  \n",
       "39972274       0  \n",
       "39972327       0  \n",
       "39982305       0  \n",
       "39983674       0  \n",
       "\n",
       "[1373 rows x 7 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ViTiMM = pd.read_csv(src / f'results_ViTiMM_{task}.csv')\n",
    "df_ViTiMM.rename(columns={'stay_id': 'hadm_id'}, inplace=True)\n",
    "df_ViTiMM.set_index('hadm_id', inplace=True)\n",
    "df_ViTiMM.rename(columns={\n",
    "    'swin_lab_text': 'C',\n",
    "    'swin_cxr_text': 'X',\n",
    "    'swin_med_text': 'M',\n",
    "    'swin_ecg_text': 'E',\n",
    "    'swin_lab_cxr_text': 'C|X',\n",
    "    'swin_lab_med_cxr_ecg_text_diagnoses': 'C|M|X|E'\n",
    "}, inplace=True)\n",
    "df_ViTiMM.rename(columns={c: f'ViTiMM {c}' for c in df_ViTiMM.columns[:-1]}, inplace=True)\n",
    "df_ViTiMM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ViTiMM C          0.014110\n",
       "ViTiMM M          0.125996\n",
       "ViTiMM X          0.068579\n",
       "ViTiMM E          0.154697\n",
       "ViTiMM C|X        0.033074\n",
       "ViTiMM C|M|X|E    0.012294\n",
       "y_true            1.000000\n",
       "Name: 30596506, dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ViTiMM.loc[30596506]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MeTra C</th>\n",
       "      <th>MeTra X</th>\n",
       "      <th>MeTra C|X</th>\n",
       "      <th>y_true</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hadm_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30008792</th>\n",
       "      <td>0.100261</td>\n",
       "      <td>0.167284</td>\n",
       "      <td>0.229474</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30009123</th>\n",
       "      <td>0.010703</td>\n",
       "      <td>0.022816</td>\n",
       "      <td>0.014702</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30009597</th>\n",
       "      <td>0.292390</td>\n",
       "      <td>0.087597</td>\n",
       "      <td>0.467905</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30009753</th>\n",
       "      <td>0.540065</td>\n",
       "      <td>0.081458</td>\n",
       "      <td>0.683803</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30022756</th>\n",
       "      <td>0.027811</td>\n",
       "      <td>0.023543</td>\n",
       "      <td>0.006235</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39970416</th>\n",
       "      <td>0.064205</td>\n",
       "      <td>0.012789</td>\n",
       "      <td>0.025678</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39972274</th>\n",
       "      <td>0.048572</td>\n",
       "      <td>0.037547</td>\n",
       "      <td>0.016200</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39972327</th>\n",
       "      <td>0.019332</td>\n",
       "      <td>0.016353</td>\n",
       "      <td>0.018670</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39982305</th>\n",
       "      <td>0.601376</td>\n",
       "      <td>0.011006</td>\n",
       "      <td>0.090009</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39983674</th>\n",
       "      <td>0.019771</td>\n",
       "      <td>0.007780</td>\n",
       "      <td>0.004198</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1373 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           MeTra C   MeTra X  MeTra C|X  y_true\n",
       "hadm_id                                        \n",
       "30008792  0.100261  0.167284   0.229474     1.0\n",
       "30009123  0.010703  0.022816   0.014702     0.0\n",
       "30009597  0.292390  0.087597   0.467905     1.0\n",
       "30009753  0.540065  0.081458   0.683803     0.0\n",
       "30022756  0.027811  0.023543   0.006235     0.0\n",
       "...            ...       ...        ...     ...\n",
       "39970416  0.064205  0.012789   0.025678     0.0\n",
       "39972274  0.048572  0.037547   0.016200     0.0\n",
       "39972327  0.019332  0.016353   0.018670     0.0\n",
       "39982305  0.601376  0.011006   0.090009     0.0\n",
       "39983674  0.019771  0.007780   0.004198     0.0\n",
       "\n",
       "[1373 rows x 4 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_MeTra = pd.read_csv(src / f'results_MeTra_{task}.csv')\n",
    "# df_MeTra.set_index('hadm_id', inplace=True)\n",
    "# metra_meta = pd.read_csv('results/MeTra_test_metadata.csv', index_col=0)\n",
    "# metra_meta = metra_meta[['stay_id','y_true']]\n",
    "# metra_meta.rename(columns={'stay_id': 'hadm_id'}, inplace=True)\n",
    "# df_MeTra = df_MeTra.merge(metra_meta, on='hadm_id', how='left')\n",
    "df_MeTra.set_index('hadm_id', inplace=True)\n",
    "df_MeTra.rename(columns={\n",
    "    'EHR': 'C',\n",
    "    'CXR': 'X',\n",
    "    'EHR_CXR': 'C|X'\n",
    "}, inplace=True)\n",
    "df_MeTra.rename(columns={c: f'MeTra {c}' for c in df_MeTra.columns[:-1]}, inplace=True)\n",
    "df_MeTra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_true</th>\n",
       "      <th>MedFuse C</th>\n",
       "      <th>MedFuse C|X</th>\n",
       "      <th>MedFuse X</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hadm_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37510196</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.111955</td>\n",
       "      <td>0.245501</td>\n",
       "      <td>0.283654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32128372</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.046855</td>\n",
       "      <td>0.029993</td>\n",
       "      <td>0.112429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34577403</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.312450</td>\n",
       "      <td>0.358792</td>\n",
       "      <td>0.130637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39880770</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.229408</td>\n",
       "      <td>0.072748</td>\n",
       "      <td>0.097106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33072499</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006346</td>\n",
       "      <td>0.019151</td>\n",
       "      <td>0.105155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37467324</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.015719</td>\n",
       "      <td>0.014003</td>\n",
       "      <td>0.146886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30437043</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.208004</td>\n",
       "      <td>0.014739</td>\n",
       "      <td>0.084373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38232926</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.043289</td>\n",
       "      <td>0.053863</td>\n",
       "      <td>0.170470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35505750</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005538</td>\n",
       "      <td>0.036478</td>\n",
       "      <td>0.117179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36403582</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004418</td>\n",
       "      <td>0.025927</td>\n",
       "      <td>0.080135</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1242 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          y_true  MedFuse C  MedFuse C|X  MedFuse X\n",
       "hadm_id                                            \n",
       "37510196     1.0   0.111955     0.245501   0.283654\n",
       "32128372     0.0   0.046855     0.029993   0.112429\n",
       "34577403     1.0   0.312450     0.358792   0.130637\n",
       "39880770     0.0   0.229408     0.072748   0.097106\n",
       "33072499     0.0   0.006346     0.019151   0.105155\n",
       "...          ...        ...          ...        ...\n",
       "37467324     0.0   0.015719     0.014003   0.146886\n",
       "30437043     0.0   0.208004     0.014739   0.084373\n",
       "38232926     0.0   0.043289     0.053863   0.170470\n",
       "35505750     0.0   0.005538     0.036478   0.117179\n",
       "36403582     0.0   0.004418     0.025927   0.080135\n",
       "\n",
       "[1242 rows x 4 columns]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_MedFuse = pd.read_csv(src / f'results_MedFuse_{task}.csv')\n",
    "df_MedFuse.set_index('hadm_id', inplace=True)\n",
    "# df_MedFuse.drop(columns=['GT'], inplace=True)\n",
    "df_MedFuse.rename(columns={\n",
    "    'uni_ehr_paired': 'C',\n",
    "    'uni_cxr_paired': 'X',\n",
    "    'medFuse_paired': 'C|X'\n",
    "}, inplace=True)\n",
    "df_MedFuse.rename(columns={c: f'MedFuse {c}' for c in df_MedFuse.columns}, inplace=True)\n",
    "df_MedFuse.rename(columns={'MedFuse GT': 'y_true'}, inplace=True)\n",
    "df_MedFuse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>hadm_id</th>\n",
       "      <th>ViTiMM C</th>\n",
       "      <th>ViTiMM M</th>\n",
       "      <th>ViTiMM X</th>\n",
       "      <th>ViTiMM E</th>\n",
       "      <th>ViTiMM C|X</th>\n",
       "      <th>ViTiMM C|M|X|E</th>\n",
       "      <th>y_true</th>\n",
       "      <th>MedFuse C</th>\n",
       "      <th>MedFuse C|X</th>\n",
       "      <th>MedFuse X</th>\n",
       "      <th>MeTra C</th>\n",
       "      <th>MeTra X</th>\n",
       "      <th>MeTra C|X</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>30009123</td>\n",
       "      <td>0.015755</td>\n",
       "      <td>0.291267</td>\n",
       "      <td>0.103424</td>\n",
       "      <td>0.140598</td>\n",
       "      <td>0.049464</td>\n",
       "      <td>0.013982</td>\n",
       "      <td>0</td>\n",
       "      <td>0.022464</td>\n",
       "      <td>0.013993</td>\n",
       "      <td>0.191195</td>\n",
       "      <td>0.010703</td>\n",
       "      <td>0.022816</td>\n",
       "      <td>0.014702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>30009753</td>\n",
       "      <td>0.747420</td>\n",
       "      <td>0.468689</td>\n",
       "      <td>0.913119</td>\n",
       "      <td>0.607680</td>\n",
       "      <td>0.417776</td>\n",
       "      <td>0.886910</td>\n",
       "      <td>0</td>\n",
       "      <td>0.191673</td>\n",
       "      <td>0.093446</td>\n",
       "      <td>0.234384</td>\n",
       "      <td>0.540065</td>\n",
       "      <td>0.081458</td>\n",
       "      <td>0.683803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>30026004</td>\n",
       "      <td>0.171401</td>\n",
       "      <td>0.748546</td>\n",
       "      <td>0.089372</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.037318</td>\n",
       "      <td>0.030151</td>\n",
       "      <td>0</td>\n",
       "      <td>0.036244</td>\n",
       "      <td>0.046869</td>\n",
       "      <td>0.114915</td>\n",
       "      <td>0.020639</td>\n",
       "      <td>0.016869</td>\n",
       "      <td>0.005591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>30055897</td>\n",
       "      <td>0.327163</td>\n",
       "      <td>0.649366</td>\n",
       "      <td>0.727310</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.784216</td>\n",
       "      <td>0.602486</td>\n",
       "      <td>0</td>\n",
       "      <td>0.227690</td>\n",
       "      <td>0.324537</td>\n",
       "      <td>0.091459</td>\n",
       "      <td>0.257603</td>\n",
       "      <td>0.229790</td>\n",
       "      <td>0.707857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>30067545</td>\n",
       "      <td>0.429493</td>\n",
       "      <td>0.324002</td>\n",
       "      <td>0.361437</td>\n",
       "      <td>0.323633</td>\n",
       "      <td>0.333390</td>\n",
       "      <td>0.111332</td>\n",
       "      <td>0</td>\n",
       "      <td>0.031351</td>\n",
       "      <td>0.081943</td>\n",
       "      <td>0.085739</td>\n",
       "      <td>0.160105</td>\n",
       "      <td>0.036511</td>\n",
       "      <td>0.241456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>937</th>\n",
       "      <td>1365</td>\n",
       "      <td>39949367</td>\n",
       "      <td>0.281490</td>\n",
       "      <td>0.422440</td>\n",
       "      <td>0.164513</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.129648</td>\n",
       "      <td>0.019135</td>\n",
       "      <td>0</td>\n",
       "      <td>0.522400</td>\n",
       "      <td>0.410036</td>\n",
       "      <td>0.141685</td>\n",
       "      <td>0.020467</td>\n",
       "      <td>0.019782</td>\n",
       "      <td>0.007048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>938</th>\n",
       "      <td>1366</td>\n",
       "      <td>39950156</td>\n",
       "      <td>0.116626</td>\n",
       "      <td>0.544811</td>\n",
       "      <td>0.099997</td>\n",
       "      <td>0.518058</td>\n",
       "      <td>0.099588</td>\n",
       "      <td>0.053085</td>\n",
       "      <td>0</td>\n",
       "      <td>0.029253</td>\n",
       "      <td>0.025478</td>\n",
       "      <td>0.122701</td>\n",
       "      <td>0.062476</td>\n",
       "      <td>0.046358</td>\n",
       "      <td>0.043506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>939</th>\n",
       "      <td>1367</td>\n",
       "      <td>39951232</td>\n",
       "      <td>0.033991</td>\n",
       "      <td>0.099447</td>\n",
       "      <td>0.078051</td>\n",
       "      <td>0.166932</td>\n",
       "      <td>0.023999</td>\n",
       "      <td>0.005631</td>\n",
       "      <td>0</td>\n",
       "      <td>0.046427</td>\n",
       "      <td>0.144327</td>\n",
       "      <td>0.154643</td>\n",
       "      <td>0.099625</td>\n",
       "      <td>0.022541</td>\n",
       "      <td>0.026754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>1368</td>\n",
       "      <td>39970416</td>\n",
       "      <td>0.071834</td>\n",
       "      <td>0.700435</td>\n",
       "      <td>0.371966</td>\n",
       "      <td>0.578458</td>\n",
       "      <td>0.070380</td>\n",
       "      <td>0.042467</td>\n",
       "      <td>0</td>\n",
       "      <td>0.398944</td>\n",
       "      <td>0.774022</td>\n",
       "      <td>0.152684</td>\n",
       "      <td>0.064205</td>\n",
       "      <td>0.012789</td>\n",
       "      <td>0.025678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>941</th>\n",
       "      <td>1372</td>\n",
       "      <td>39983674</td>\n",
       "      <td>0.058178</td>\n",
       "      <td>0.538133</td>\n",
       "      <td>0.048904</td>\n",
       "      <td>0.162350</td>\n",
       "      <td>0.042707</td>\n",
       "      <td>0.020068</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000427</td>\n",
       "      <td>0.017476</td>\n",
       "      <td>0.061362</td>\n",
       "      <td>0.019771</td>\n",
       "      <td>0.007780</td>\n",
       "      <td>0.004198</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>942 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index   hadm_id  ViTiMM C  ViTiMM M  ViTiMM X  ViTiMM E  ViTiMM C|X  \\\n",
       "0        1  30009123  0.015755  0.291267  0.103424  0.140598    0.049464   \n",
       "1        3  30009753  0.747420  0.468689  0.913119  0.607680    0.417776   \n",
       "2        6  30026004  0.171401  0.748546  0.089372       NaN    0.037318   \n",
       "3        7  30055897  0.327163  0.649366  0.727310       NaN    0.784216   \n",
       "4        8  30067545  0.429493  0.324002  0.361437  0.323633    0.333390   \n",
       "..     ...       ...       ...       ...       ...       ...         ...   \n",
       "937   1365  39949367  0.281490  0.422440  0.164513       NaN    0.129648   \n",
       "938   1366  39950156  0.116626  0.544811  0.099997  0.518058    0.099588   \n",
       "939   1367  39951232  0.033991  0.099447  0.078051  0.166932    0.023999   \n",
       "940   1368  39970416  0.071834  0.700435  0.371966  0.578458    0.070380   \n",
       "941   1372  39983674  0.058178  0.538133  0.048904  0.162350    0.042707   \n",
       "\n",
       "     ViTiMM C|M|X|E  y_true  MedFuse C  MedFuse C|X  MedFuse X   MeTra C  \\\n",
       "0          0.013982       0   0.022464     0.013993   0.191195  0.010703   \n",
       "1          0.886910       0   0.191673     0.093446   0.234384  0.540065   \n",
       "2          0.030151       0   0.036244     0.046869   0.114915  0.020639   \n",
       "3          0.602486       0   0.227690     0.324537   0.091459  0.257603   \n",
       "4          0.111332       0   0.031351     0.081943   0.085739  0.160105   \n",
       "..              ...     ...        ...          ...        ...       ...   \n",
       "937        0.019135       0   0.522400     0.410036   0.141685  0.020467   \n",
       "938        0.053085       0   0.029253     0.025478   0.122701  0.062476   \n",
       "939        0.005631       0   0.046427     0.144327   0.154643  0.099625   \n",
       "940        0.042467       0   0.398944     0.774022   0.152684  0.064205   \n",
       "941        0.020068       0   0.000427     0.017476   0.061362  0.019771   \n",
       "\n",
       "      MeTra X  MeTra C|X  \n",
       "0    0.022816   0.014702  \n",
       "1    0.081458   0.683803  \n",
       "2    0.016869   0.005591  \n",
       "3    0.229790   0.707857  \n",
       "4    0.036511   0.241456  \n",
       "..        ...        ...  \n",
       "937  0.019782   0.007048  \n",
       "938  0.046358   0.043506  \n",
       "939  0.022541   0.026754  \n",
       "940  0.012789   0.025678  \n",
       "941  0.007780   0.004198  \n",
       "\n",
       "[942 rows x 15 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_MeTra = df_MeTra.reset_index()\n",
    "df_MedFuse = df_MedFuse.reset_index()\n",
    "df_ViTiMM = df_ViTiMM.reset_index()\n",
    "df = df_ViTiMM.merge(df_MedFuse, on=['hadm_id','y_true'], how='inner')\n",
    "df = df.merge(df_MeTra, on=['hadm_id','y_true'], how='inner')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index (np.float64(0.39985067940869046), np.float64(0.036642784196048855), np.float64(0.5))\n",
      "hadm_id (np.float64(0.39985067940869046), np.float64(0.036642784196048855), np.float64(0.5))\n",
      "ViTiMM C (np.float64(0.810094071972525), np.float64(0.2022796615116953), np.float64(0.7430640585336719))\n",
      "ViTiMM M (np.float64(0.657339107062864), np.float64(0.09503700329415554), np.float64(0.6134089890995968))\n",
      "ViTiMM X (np.float64(0.8028370912348813), np.float64(0.18873755736866235), np.float64(0.7221143795729431))\n",
      "ViTiMM E (np.float64(0.6987419245154709), np.float64(0.058772561885706696), np.float64(0.6202538818995806))\n",
      "ViTiMM C|X (np.float64(0.8410930267283859), np.float64(0.2841931619954137), np.float64(0.7502911751530537))\n",
      "ViTiMM C|M|X|E (np.float64(0.877198745707033), np.float64(0.4792994434270713), np.float64(0.8145886217709422))\n",
      "MedFuse C (np.float64(0.8192026280424071), np.float64(0.23922295536354093), np.float64(0.5852023294012244))\n",
      "MedFuse C|X (np.float64(0.8164551291623114), np.float64(0.25283698032430885), np.float64(0.7059728236523817))\n",
      "MedFuse X (np.float64(0.616574585635359), np.float64(0.10160419647828176), np.float64(0.5))\n",
      "MeTra C (np.float64(0.7903837539196655), np.float64(0.16830351683973066), np.float64(0.6271016873226818))\n",
      "MeTra X (np.float64(0.7862625055995222), np.float64(0.13736829312660412), np.float64(0.5096461101985964))\n",
      "MeTra C|X (np.float64(0.8314469165297895), np.float64(0.2708694884693218), np.float64(0.6783933104375094))\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from utils import metrics\n",
    "methods = [c for c in df.columns if c != 'y_true']\n",
    "for m in methods:\n",
    "    dfm = df.dropna(subset=[m])\n",
    "    print(m, metrics(torch.tensor(dfm.y_true.tolist()), torch.tensor(dfm[m].tolist())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\\\begin{tabular}{lrrrrrrrrrrrr}\\n\\\\toprule\\n & MeTra C & MeTra X & MeTra C|X & MedFuse C & MedFuse X & MedFuse C|X & ViTiMM C & ViTiMM X & ViTiMM M & ViTiMM E & ViTiMM C|X & ViTiMM C|M|X|E \\\\\\\\\\n\\\\midrule\\nMeTra C & NaN & 0.000000 & 0.000000 & 0.000000 & 0.738000 & 0.261000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 \\\\\\\\\\nMeTra X & 0.000000 & NaN & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 \\\\\\\\\\nMeTra C|X & 0.000000 & 0.000000 & NaN & 0.444000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 \\\\\\\\\\nMedFuse C & 0.000000 & 0.000000 & 0.444000 & NaN & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 \\\\\\\\\\nMedFuse X & 0.738000 & 0.000000 & 0.000000 & 0.000000 & NaN & 0.253000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 \\\\\\\\\\nMedFuse C|X & 0.261000 & 0.000000 & 0.000000 & 0.000000 & 0.253000 & NaN & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 \\\\\\\\\\nViTiMM C & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & NaN & 0.104000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 \\\\\\\\\\nViTiMM X & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.104000 & NaN & 0.000000 & 0.000000 & 0.000000 & 0.000000 \\\\\\\\\\nViTiMM M & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & NaN & 0.143000 & 0.000000 & 0.000000 \\\\\\\\\\nViTiMM E & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.143000 & NaN & 0.000000 & 0.000000 \\\\\\\\\\nViTiMM C|X & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & NaN & 0.000000 \\\\\\\\\\nViTiMM C|M|X|E & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & 0.000000 & NaN \\\\\\\\\\n\\\\bottomrule\\n\\\\end{tabular}\\n'"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.stats import ttest_rel\n",
    "dfp = df.drop(columns=['y_true']).set_index('hadm_id')\n",
    "\n",
    "cols = [\n",
    "    'MeTra C', \n",
    "    'MeTra X', \n",
    "    'MeTra C|X',\n",
    "    'MedFuse C', \n",
    "    'MedFuse X',\n",
    "    'MedFuse C|X', \n",
    "    'ViTiMM C', \n",
    "    'ViTiMM X', \n",
    "    'ViTiMM M', \n",
    "    'ViTiMM E', \n",
    "    'ViTiMM C|X',\n",
    "    'ViTiMM C|M|X|E'\n",
    "]\n",
    "dfp = dfp[cols]\n",
    "methods = dfp.columns\n",
    "p_values = pd.DataFrame(index=methods, columns=methods)\n",
    "\n",
    "for method_1 in methods:\n",
    "    for method_2 in methods:\n",
    "        if method_1 != method_2:\n",
    "            _df = df.dropna(subset=[method_1,method_2])\n",
    "            _, p_val = ttest_rel(_df[method_1], _df[method_2])\n",
    "            p_values.loc[method_1, method_2] = p_val\n",
    "        else:\n",
    "            p_values.loc[method_1, method_2] = None  # No test against itself\n",
    "p_values = p_values.astype(float)\n",
    "p_values.round(3).to_latex()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cfm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}