code/results and statistics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_models_path = [\n",
    "    '../data/Adult_non-aggregated-results.csv',          # Adult\n",
    "    '../data/AOF_non-aggregated-results.csv',            # AOF\n",
    "    '../data/COMPAS_non-aggregated-results.csv',         # COMPAS\n",
    "    '../data/Donors Choose_non-aggregated-results.csv',  # Donors Choose\n",
    "    '../data/all-datasets-with-TPE-tuner_non-aggregated-results.csv',          # OOB Tuners (TPE)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = None\n",
    "\n",
    "for path in top_models_path:#[:4]:\n",
    "    df = pd.read_csv(path)\n",
    "    \n",
    "    perf_metric = df['performance_metric'].unique()[0]\n",
    "    fair_metric = df['fairness_metric'].unique()[0]\n",
    "    \n",
    "    if 'performance_val' not in set(df.columns):\n",
    "        df['performance_val'] = df[f\"validation_{perf_metric}\"]\n",
    "    if 'fairness_val' not in set(df.columns):\n",
    "        df['fairness_val'] = df[f\"validation_{fair_metric}\"]\n",
    "    if 'performance_test' not in set(df.columns):\n",
    "        df['performance_test'] = df[f\"test_{perf_metric}\"]\n",
    "    if 'fairness_test' not in set(df.columns):\n",
    "        df['fairness_test'] = df[f\"test_{fair_metric}\"]\n",
    "    \n",
    "    results_df = df if results_df is None else pd.concat((results_df, df), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = results_df[[\n",
    "    'dataset', 'task_type', 'alpha',\n",
    "    'performance_val', 'fairness_val',\n",
    "    'performance_test', 'fairness_test',\n",
    "    'model_classpath', 'hyperparameters', 'class_ratio',\n",
    "    'model_config_uuid', 'model_uuid',\n",
    "    'si', 'i', 'ni', 'ri', 'target_threshold',\n",
    "]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>task_type</th>\n",
       "      <th>alpha</th>\n",
       "      <th>performance_val</th>\n",
       "      <th>fairness_val</th>\n",
       "      <th>performance_test</th>\n",
       "      <th>fairness_test</th>\n",
       "      <th>model_classpath</th>\n",
       "      <th>hyperparameters</th>\n",
       "      <th>class_ratio</th>\n",
       "      <th>model_config_uuid</th>\n",
       "      <th>model_uuid</th>\n",
       "      <th>si</th>\n",
       "      <th>i</th>\n",
       "      <th>ni</th>\n",
       "      <th>ri</th>\n",
       "      <th>target_threshold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adult</td>\n",
       "      <td>Hyperband</td>\n",
       "      <td>dynamic</td>\n",
       "      <td>0.94100</td>\n",
       "      <td>0.91024</td>\n",
       "      <td>0.93272</td>\n",
       "      <td>0.84107</td>\n",
       "      <td>fairautoml.nn.wrappers.FeedForwardClassifier</td>\n",
       "      <td>{'batch_size': 2048, 'max_epochs': 75, 'optimi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>e90d361bb0bfc5448ae077a02f47afe2</td>\n",
       "      <td>98f05504756f5a66e28f5515f4c00cdc</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>3.704</td>\n",
       "      <td>tpr = 50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Adult</td>\n",
       "      <td>Hyperband</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.91896</td>\n",
       "      <td>0.99948</td>\n",
       "      <td>0.91367</td>\n",
       "      <td>0.94126</td>\n",
       "      <td>fairautoml.nn.wrappers.FeedForwardClassifier</td>\n",
       "      <td>{'batch_size': 2048, 'max_epochs': 25, 'optimi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ffa8b120ec9b1ae9201a6634fd9dffa4</td>\n",
       "      <td>cad876af4952ec67447a5e79422310c2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>1.235</td>\n",
       "      <td>tpr = 50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Adult</td>\n",
       "      <td>Hyperband</td>\n",
       "      <td>dynamic</td>\n",
       "      <td>0.93142</td>\n",
       "      <td>0.93815</td>\n",
       "      <td>0.92774</td>\n",
       "      <td>0.88856</td>\n",
       "      <td>fairautoml.nn.wrappers.FeedForwardClassifier</td>\n",
       "      <td>{'batch_size': 2048, 'max_epochs': 25, 'optimi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>592dde8641d47d278a158df7d45f1ce0</td>\n",
       "      <td>772f5373cdb06dc996a79f8474acef54</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>3.704</td>\n",
       "      <td>tpr = 50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Adult</td>\n",
       "      <td>Hyperband</td>\n",
       "      <td>dynamic</td>\n",
       "      <td>0.94315</td>\n",
       "      <td>0.93658</td>\n",
       "      <td>0.93605</td>\n",
       "      <td>0.88501</td>\n",
       "      <td>fairautoml.nn.wrappers.FeedForwardClassifier</td>\n",
       "      <td>{'batch_size': 2048, 'max_epochs': 25, 'optimi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7e1266b6f483aaa91345b090acfa8e87</td>\n",
       "      <td>8a3f2ed18191d2b036bf6d656fb63891</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>11.111</td>\n",
       "      <td>tpr = 50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Adult</td>\n",
       "      <td>Hyperband</td>\n",
       "      <td>dynamic</td>\n",
       "      <td>0.92067</td>\n",
       "      <td>0.99373</td>\n",
       "      <td>0.91778</td>\n",
       "      <td>0.93524</td>\n",
       "      <td>fairautoml.nn.wrappers.FeedForwardClassifier</td>\n",
       "      <td>{'batch_size': 2048, 'max_epochs': 25, 'optimi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25b7ae7ed998f0b159e77417ec8aab68</td>\n",
       "      <td>cbbaad53880e7dace0db138a34868a0c</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>1.235</td>\n",
       "      <td>tpr = 50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  dataset  task_type    alpha  performance_val  fairness_val  \\\n",
       "0   Adult  Hyperband  dynamic          0.94100       0.91024   \n",
       "1   Adult  Hyperband      0.5          0.91896       0.99948   \n",
       "2   Adult  Hyperband  dynamic          0.93142       0.93815   \n",
       "3   Adult  Hyperband  dynamic          0.94315       0.93658   \n",
       "4   Adult  Hyperband  dynamic          0.92067       0.99373   \n",
       "\n",
       "   performance_test  fairness_test  \\\n",
       "0           0.93272        0.84107   \n",
       "1           0.91367        0.94126   \n",
       "2           0.92774        0.88856   \n",
       "3           0.93605        0.88501   \n",
       "4           0.91778        0.93524   \n",
       "\n",
       "                                model_classpath  \\\n",
       "0  fairautoml.nn.wrappers.FeedForwardClassifier   \n",
       "1  fairautoml.nn.wrappers.FeedForwardClassifier   \n",
       "2  fairautoml.nn.wrappers.FeedForwardClassifier   \n",
       "3  fairautoml.nn.wrappers.FeedForwardClassifier   \n",
       "4  fairautoml.nn.wrappers.FeedForwardClassifier   \n",
       "\n",
       "                                     hyperparameters  class_ratio  \\\n",
       "0  {'batch_size': 2048, 'max_epochs': 75, 'optimi...          NaN   \n",
       "1  {'batch_size': 2048, 'max_epochs': 25, 'optimi...          NaN   \n",
       "2  {'batch_size': 2048, 'max_epochs': 25, 'optimi...          NaN   \n",
       "3  {'batch_size': 2048, 'max_epochs': 25, 'optimi...          NaN   \n",
       "4  {'batch_size': 2048, 'max_epochs': 25, 'optimi...          NaN   \n",
       "\n",
       "                  model_config_uuid                        model_uuid   si  \\\n",
       "0  e90d361bb0bfc5448ae077a02f47afe2  98f05504756f5a66e28f5515f4c00cdc  3.0   \n",
       "1  ffa8b120ec9b1ae9201a6634fd9dffa4  cad876af4952ec67447a5e79422310c2  4.0   \n",
       "2  592dde8641d47d278a158df7d45f1ce0  772f5373cdb06dc996a79f8474acef54  3.0   \n",
       "3  7e1266b6f483aaa91345b090acfa8e87  8a3f2ed18191d2b036bf6d656fb63891  3.0   \n",
       "4  25b7ae7ed998f0b159e77417ec8aab68  cbbaad53880e7dace0db138a34868a0c  4.0   \n",
       "\n",
       "     i    ni      ri target_threshold  \n",
       "0  0.0  34.0   3.704         tpr = 50  \n",
       "1  0.0  81.0   1.235         tpr = 50  \n",
       "2  0.0  34.0   3.704         tpr = 50  \n",
       "3  1.0  11.0  11.111         tpr = 50  \n",
       "4  0.0  81.0   1.235         tpr = 50  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dataset\n",
       "AOF              135\n",
       "Adult            135\n",
       "COMPAS           135\n",
       "Donors Choose    135\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df.value_counts('dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = results_df[results_df['task_type'] != 'RandomSampler']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FB-auto    60\n",
       "FB         60\n",
       "RS         60\n",
       "HB         60\n",
       "FairRS     60\n",
       "FairTPE    60\n",
       "TPE        60\n",
       "Name: tuner, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df['tuner'] = results_df.apply(\n",
    "    lambda row: f\"{row.task_type} {row.alpha}\",\n",
    "    axis=1,\n",
    ")\n",
    "\n",
    "# Changing to naming convention used on the paper\n",
    "tuner_rename = {\n",
    "    'RandomSearch 0.5': 'FairRS',\n",
    "    'RandomSearch 1': 'RS',\n",
    "    'Hyperband dynamic': 'FB-auto',\n",
    "    'Hyperband 0.5': 'FB',\n",
    "    'Hyperband 1': 'HB',\n",
    "    'TPESampler 0.5': 'FairTPE',\n",
    "    'TPESampler 1.0': 'TPE',\n",
    "}\n",
    "\n",
    "results_df['tuner'] = results_df['tuner'].apply(\n",
    "    lambda tuner: tuner_rename[tuner] if tuner in tuner_rename else tuner\n",
    ")\n",
    "results_df['tuner'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">performance_val</th>\n",
       "      <th colspan=\"2\" halign=\"left\">fairness_val</th>\n",
       "      <th colspan=\"2\" halign=\"left\">performance_test</th>\n",
       "      <th colspan=\"2\" halign=\"left\">fairness_test</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>tuner</th>\n",
       "      <th>alpha</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">AOF</th>\n",
       "      <th>FB</th>\n",
       "      <th>0.5</th>\n",
       "      <td>50.7</td>\n",
       "      <td>15.8</td>\n",
       "      <td>76.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>52.6</td>\n",
       "      <td>14.6</td>\n",
       "      <td>81.3</td>\n",
       "      <td>12.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FB-auto</th>\n",
       "      <th>dynamic</th>\n",
       "      <td>61.7</td>\n",
       "      <td>6.5</td>\n",
       "      <td>68.1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>5.1</td>\n",
       "      <td>74.2</td>\n",
       "      <td>12.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FairRS</th>\n",
       "      <th>0.5</th>\n",
       "      <td>60.4</td>\n",
       "      <td>9.2</td>\n",
       "      <td>64.1</td>\n",
       "      <td>15.6</td>\n",
       "      <td>62.6</td>\n",
       "      <td>10.0</td>\n",
       "      <td>68.6</td>\n",
       "      <td>15.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FairTPE</th>\n",
       "      <th>0.5</th>\n",
       "      <td>55.7</td>\n",
       "      <td>7.1</td>\n",
       "      <td>76.9</td>\n",
       "      <td>17.0</td>\n",
       "      <td>59.2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>14.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HB</th>\n",
       "      <th>1</th>\n",
       "      <td>68.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>32.3</td>\n",
       "      <td>6.2</td>\n",
       "      <td>68.4</td>\n",
       "      <td>1.4</td>\n",
       "      <td>35.2</td>\n",
       "      <td>6.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RS</th>\n",
       "      <th>1</th>\n",
       "      <td>67.8</td>\n",
       "      <td>0.6</td>\n",
       "      <td>42.2</td>\n",
       "      <td>10.7</td>\n",
       "      <td>68.1</td>\n",
       "      <td>1.6</td>\n",
       "      <td>45.0</td>\n",
       "      <td>11.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TPE</th>\n",
       "      <th>1.0</th>\n",
       "      <td>68.7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>30.5</td>\n",
       "      <td>4.8</td>\n",
       "      <td>68.5</td>\n",
       "      <td>1.5</td>\n",
       "      <td>33.7</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">Adult</th>\n",
       "      <th>FB</th>\n",
       "      <th>0.5</th>\n",
       "      <td>92.7</td>\n",
       "      <td>3.4</td>\n",
       "      <td>94.0</td>\n",
       "      <td>5.4</td>\n",
       "      <td>92.3</td>\n",
       "      <td>3.3</td>\n",
       "      <td>89.5</td>\n",
       "      <td>6.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FB-auto</th>\n",
       "      <th>dynamic</th>\n",
       "      <td>92.0</td>\n",
       "      <td>4.4</td>\n",
       "      <td>94.7</td>\n",
       "      <td>2.8</td>\n",
       "      <td>91.6</td>\n",
       "      <td>4.2</td>\n",
       "      <td>90.9</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FairRS</th>\n",
       "      <th>0.5</th>\n",
       "      <td>93.6</td>\n",
       "      <td>6.5</td>\n",
       "      <td>79.4</td>\n",
       "      <td>8.1</td>\n",
       "      <td>93.8</td>\n",
       "      <td>6.3</td>\n",
       "      <td>78.6</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FairTPE</th>\n",
       "      <th>0.5</th>\n",
       "      <td>93.3</td>\n",
       "      <td>3.8</td>\n",
       "      <td>82.2</td>\n",
       "      <td>5.7</td>\n",
       "      <td>93.5</td>\n",
       "      <td>3.7</td>\n",
       "      <td>80.7</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HB</th>\n",
       "      <th>1</th>\n",
       "      <td>99.4</td>\n",
       "      <td>0.1</td>\n",
       "      <td>53.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>99.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>54.1</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RS</th>\n",
       "      <th>1</th>\n",
       "      <td>99.4</td>\n",
       "      <td>0.1</td>\n",
       "      <td>55.7</td>\n",
       "      <td>2.9</td>\n",
       "      <td>99.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>56.6</td>\n",
       "      <td>3.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TPE</th>\n",
       "      <th>1.0</th>\n",
       "      <td>99.4</td>\n",
       "      <td>0.1</td>\n",
       "      <td>54.9</td>\n",
       "      <td>2.1</td>\n",
       "      <td>99.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>55.6</td>\n",
       "      <td>2.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">COMPAS</th>\n",
       "      <th>FB</th>\n",
       "      <th>0.5</th>\n",
       "      <td>71.2</td>\n",
       "      <td>5.1</td>\n",
       "      <td>95.5</td>\n",
       "      <td>2.7</td>\n",
       "      <td>67.6</td>\n",
       "      <td>4.8</td>\n",
       "      <td>80.7</td>\n",
       "      <td>14.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FB-auto</th>\n",
       "      <th>dynamic</th>\n",
       "      <td>74.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>95.8</td>\n",
       "      <td>2.9</td>\n",
       "      <td>70.1</td>\n",
       "      <td>4.3</td>\n",
       "      <td>90.0</td>\n",
       "      <td>6.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FairRS</th>\n",
       "      <th>0.5</th>\n",
       "      <td>67.4</td>\n",
       "      <td>7.6</td>\n",
       "      <td>77.4</td>\n",
       "      <td>13.6</td>\n",
       "      <td>64.2</td>\n",
       "      <td>6.7</td>\n",
       "      <td>67.8</td>\n",
       "      <td>11.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FairTPE</th>\n",
       "      <th>0.5</th>\n",
       "      <td>67.1</td>\n",
       "      <td>7.6</td>\n",
       "      <td>81.8</td>\n",
       "      <td>12.4</td>\n",
       "      <td>63.9</td>\n",
       "      <td>6.7</td>\n",
       "      <td>69.5</td>\n",
       "      <td>14.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HB</th>\n",
       "      <th>1</th>\n",
       "      <td>78.1</td>\n",
       "      <td>0.3</td>\n",
       "      <td>45.4</td>\n",
       "      <td>6.8</td>\n",
       "      <td>73.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>51.2</td>\n",
       "      <td>7.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RS</th>\n",
       "      <th>1</th>\n",
       "      <td>77.7</td>\n",
       "      <td>0.3</td>\n",
       "      <td>43.8</td>\n",
       "      <td>6.9</td>\n",
       "      <td>73.2</td>\n",
       "      <td>0.9</td>\n",
       "      <td>43.4</td>\n",
       "      <td>5.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TPE</th>\n",
       "      <th>1.0</th>\n",
       "      <td>78.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>42.8</td>\n",
       "      <td>7.8</td>\n",
       "      <td>73.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>46.6</td>\n",
       "      <td>8.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">Donors Choose</th>\n",
       "      <th>FB</th>\n",
       "      <th>0.5</th>\n",
       "      <td>54.2</td>\n",
       "      <td>1.4</td>\n",
       "      <td>97.7</td>\n",
       "      <td>2.1</td>\n",
       "      <td>50.4</td>\n",
       "      <td>1.1</td>\n",
       "      <td>85.5</td>\n",
       "      <td>7.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FB-auto</th>\n",
       "      <th>dynamic</th>\n",
       "      <td>54.2</td>\n",
       "      <td>1.6</td>\n",
       "      <td>98.2</td>\n",
       "      <td>1.7</td>\n",
       "      <td>50.7</td>\n",
       "      <td>1.1</td>\n",
       "      <td>86.5</td>\n",
       "      <td>6.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FairRS</th>\n",
       "      <th>0.5</th>\n",
       "      <td>51.7</td>\n",
       "      <td>2.2</td>\n",
       "      <td>97.0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>50.4</td>\n",
       "      <td>0.7</td>\n",
       "      <td>79.5</td>\n",
       "      <td>4.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FairTPE</th>\n",
       "      <th>0.5</th>\n",
       "      <td>52.3</td>\n",
       "      <td>2.2</td>\n",
       "      <td>96.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>50.6</td>\n",
       "      <td>1.3</td>\n",
       "      <td>79.1</td>\n",
       "      <td>4.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HB</th>\n",
       "      <th>1</th>\n",
       "      <td>60.9</td>\n",
       "      <td>0.6</td>\n",
       "      <td>28.7</td>\n",
       "      <td>3.4</td>\n",
       "      <td>53.6</td>\n",
       "      <td>0.6</td>\n",
       "      <td>35.0</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RS</th>\n",
       "      <th>1</th>\n",
       "      <td>59.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>24.9</td>\n",
       "      <td>2.9</td>\n",
       "      <td>53.4</td>\n",
       "      <td>0.6</td>\n",
       "      <td>32.4</td>\n",
       "      <td>4.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TPE</th>\n",
       "      <th>1.0</th>\n",
       "      <td>61.0</td>\n",
       "      <td>0.7</td>\n",
       "      <td>27.1</td>\n",
       "      <td>2.1</td>\n",
       "      <td>53.3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>33.4</td>\n",
       "      <td>3.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              performance_val       fairness_val        \\\n",
       "                                         mean   std         mean   std   \n",
       "dataset       tuner   alpha                                              \n",
       "AOF           FB      0.5                50.7  15.8         76.0  15.0   \n",
       "              FB-auto dynamic            61.7   6.5         68.1  14.0   \n",
       "              FairRS  0.5                60.4   9.2         64.1  15.6   \n",
       "              FairTPE 0.5                55.7   7.1         76.9  17.0   \n",
       "              HB      1                  68.4   1.0         32.3   6.2   \n",
       "              RS      1                  67.8   0.6         42.2  10.7   \n",
       "              TPE     1.0                68.7   1.0         30.5   4.8   \n",
       "Adult         FB      0.5                92.7   3.4         94.0   5.4   \n",
       "              FB-auto dynamic            92.0   4.4         94.7   2.8   \n",
       "              FairRS  0.5                93.6   6.5         79.4   8.1   \n",
       "              FairTPE 0.5                93.3   3.8         82.2   5.7   \n",
       "              HB      1                  99.4   0.1         53.5   1.0   \n",
       "              RS      1                  99.4   0.1         55.7   2.9   \n",
       "              TPE     1.0                99.4   0.1         54.9   2.1   \n",
       "COMPAS        FB      0.5                71.2   5.1         95.5   2.7   \n",
       "              FB-auto dynamic            74.0   3.5         95.8   2.9   \n",
       "              FairRS  0.5                67.4   7.6         77.4  13.6   \n",
       "              FairTPE 0.5                67.1   7.6         81.8  12.4   \n",
       "              HB      1                  78.1   0.3         45.4   6.8   \n",
       "              RS      1                  77.7   0.3         43.8   6.9   \n",
       "              TPE     1.0                78.0   0.3         42.8   7.8   \n",
       "Donors Choose FB      0.5                54.2   1.4         97.7   2.1   \n",
       "              FB-auto dynamic            54.2   1.6         98.2   1.7   \n",
       "              FairRS  0.5                51.7   2.2         97.0   2.3   \n",
       "              FairTPE 0.5                52.3   2.2         96.3   3.0   \n",
       "              HB      1                  60.9   0.6         28.7   3.4   \n",
       "              RS      1                  59.9   0.9         24.9   2.9   \n",
       "              TPE     1.0                61.0   0.7         27.1   2.1   \n",
       "\n",
       "                              performance_test       fairness_test        \n",
       "                                          mean   std          mean   std  \n",
       "dataset       tuner   alpha                                               \n",
       "AOF           FB      0.5                 52.6  14.6          81.3  12.3  \n",
       "              FB-auto dynamic             64.0   5.1          74.2  12.9  \n",
       "              FairRS  0.5                 62.6  10.0          68.6  15.6  \n",
       "              FairTPE 0.5                 59.2   9.0          80.0  14.6  \n",
       "              HB      1                   68.4   1.4          35.2   6.7  \n",
       "              RS      1                   68.1   1.6          45.0  11.5  \n",
       "              TPE     1.0                 68.5   1.5          33.7   5.2  \n",
       "Adult         FB      0.5                 92.3   3.3          89.5   6.1  \n",
       "              FB-auto dynamic             91.6   4.2          90.9   5.2  \n",
       "              FairRS  0.5                 93.8   6.3          78.6   8.0  \n",
       "              FairTPE 0.5                 93.5   3.7          80.7   6.0  \n",
       "              HB      1                   99.0   0.1          54.1   1.2  \n",
       "              RS      1                   99.1   0.1          56.6   3.2  \n",
       "              TPE     1.0                 99.1   0.1          55.6   2.4  \n",
       "COMPAS        FB      0.5                 67.6   4.8          80.7  14.8  \n",
       "              FB-auto dynamic             70.1   4.3          90.0   6.5  \n",
       "              FairRS  0.5                 64.2   6.7          67.8  11.9  \n",
       "              FairTPE 0.5                 63.9   6.7          69.5  14.5  \n",
       "              HB      1                   73.6   1.4          51.2   7.7  \n",
       "              RS      1                   73.2   0.9          43.4   5.8  \n",
       "              TPE     1.0                 73.5   1.4          46.6   8.3  \n",
       "Donors Choose FB      0.5                 50.4   1.1          85.5   7.5  \n",
       "              FB-auto dynamic             50.7   1.1          86.5   6.1  \n",
       "              FairRS  0.5                 50.4   0.7          79.5   4.1  \n",
       "              FairTPE 0.5                 50.6   1.3          79.1   4.1  \n",
       "              HB      1                   53.6   0.6          35.0   4.5  \n",
       "              RS      1                   53.4   0.6          32.4   4.6  \n",
       "              TPE     1.0                 53.3   0.5          33.4   3.7  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "key_columns = ['dataset', 'tuner', 'alpha']\n",
    "metric_columns = ['performance_val', 'fairness_val', 'performance_test', 'fairness_test']\n",
    "\n",
    "results_df_metrics = results_df[key_columns + metric_columns]\n",
    "\n",
    "agg_results = results_df_metrics.groupby(['dataset', 'tuner', 'alpha']).agg(['mean', 'std'])\n",
    "agg_results.round(decimals=3) * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute statistical differences between every candidate pair of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "\n",
    "results_indexed = results_df_metrics.set_index(['dataset', 'tuner'])\n",
    "\n",
    "all_statistical_tests = dict()\n",
    "\n",
    "for metric_col in metric_columns:\n",
    "    metric_tests = dict()\n",
    "\n",
    "    for dataset in results_df_metrics['dataset'].unique():\n",
    "        dataset_results = results_indexed.loc[dataset]\n",
    "        dataset_tests = dict()\n",
    "\n",
    "        for tuner_type_outer in results_df_metrics['tuner'].unique():\n",
    "            outer_tuner_tests = dict()\n",
    "\n",
    "            outer_values = dataset_results.loc[tuner_type_outer]\n",
    "            outer_values = outer_values[metric_col]\n",
    "\n",
    "            for tuner_type_inner in results_df_metrics['tuner'].unique():\n",
    "\n",
    "                inner_values = dataset_results.loc[tuner_type_inner]\n",
    "                inner_values = inner_values[metric_col]\n",
    "\n",
    "                # Run statistical test\n",
    "                stats_val, p_val = stats.ks_2samp(outer_values, inner_values)\n",
    "\n",
    "                outer_tuner_tests[tuner_type_inner] = p_val  # Extract p-value\n",
    "\n",
    "            dataset_tests[tuner_type_outer] = outer_tuner_tests\n",
    "            \n",
    "        metric_tests[dataset] = dataset_tests\n",
    "\n",
    "    all_statistical_tests[metric_col] = metric_tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Aggregate results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_tests_dfs = dict()\n",
    "\n",
    "for metric in all_statistical_tests.keys():\n",
    "    for dataset in all_statistical_tests[metric].keys():\n",
    "        dataset_tests = all_statistical_tests[metric][dataset]\n",
    "        tests_df = pd.DataFrame(dataset_tests)\n",
    "        stats_tests_dfs[f\"{dataset}_{metric}\"] = tests_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show p-values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "### Adult_performance_val"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_186f7_row0_col0, #T_186f7_row0_col1, #T_186f7_row1_col0, #T_186f7_row1_col1, #T_186f7_row1_col5, #T_186f7_row2_col2, #T_186f7_row2_col3, #T_186f7_row2_col6, #T_186f7_row3_col2, #T_186f7_row3_col3, #T_186f7_row3_col6, #T_186f7_row4_col4, #T_186f7_row5_col1, #T_186f7_row5_col5, #T_186f7_row6_col2, #T_186f7_row6_col3, #T_186f7_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_186f7_row0_col2, #T_186f7_row0_col3, #T_186f7_row0_col4, #T_186f7_row0_col6, #T_186f7_row1_col2, #T_186f7_row1_col3, #T_186f7_row1_col4, #T_186f7_row1_col6, #T_186f7_row2_col0, #T_186f7_row2_col1, #T_186f7_row2_col4, #T_186f7_row2_col5, #T_186f7_row3_col0, #T_186f7_row3_col1, #T_186f7_row3_col4, #T_186f7_row3_col5, #T_186f7_row4_col0, #T_186f7_row4_col1, #T_186f7_row4_col2, #T_186f7_row4_col3, #T_186f7_row4_col6, #T_186f7_row5_col2, #T_186f7_row5_col3, #T_186f7_row5_col6, #T_186f7_row6_col0, #T_186f7_row6_col1, #T_186f7_row6_col4, #T_186f7_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_186f7_row0_col5, #T_186f7_row4_col5, #T_186f7_row5_col0, #T_186f7_row5_col4 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_186f7_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_186f7_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_186f7_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_186f7_row0_col1\" class=\"data row0 col1\" >0.678138</td>\n",
       "      <td id=\"T_186f7_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row0_col4\" class=\"data row0 col4\" >0.001837</td>\n",
       "      <td id=\"T_186f7_row0_col5\" class=\"data row0 col5\" >0.075464</td>\n",
       "      <td id=\"T_186f7_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_186f7_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_186f7_row1_col0\" class=\"data row1 col0\" >0.678138</td>\n",
       "      <td id=\"T_186f7_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_186f7_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row1_col4\" class=\"data row1 col4\" >0.001837</td>\n",
       "      <td id=\"T_186f7_row1_col5\" class=\"data row1 col5\" >0.385547</td>\n",
       "      <td id=\"T_186f7_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_186f7_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_186f7_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_186f7_row2_col3\" class=\"data row2 col3\" >0.938331</td>\n",
       "      <td id=\"T_186f7_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row2_col6\" class=\"data row2 col6\" >0.184416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_186f7_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_186f7_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row3_col2\" class=\"data row3 col2\" >0.938331</td>\n",
       "      <td id=\"T_186f7_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_186f7_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row3_col6\" class=\"data row3 col6\" >0.184416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_186f7_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_186f7_row4_col0\" class=\"data row4 col0\" >0.001837</td>\n",
       "      <td id=\"T_186f7_row4_col1\" class=\"data row4 col1\" >0.001837</td>\n",
       "      <td id=\"T_186f7_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_186f7_row4_col5\" class=\"data row4 col5\" >0.075464</td>\n",
       "      <td id=\"T_186f7_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_186f7_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_186f7_row5_col0\" class=\"data row5 col0\" >0.075464</td>\n",
       "      <td id=\"T_186f7_row5_col1\" class=\"data row5 col1\" >0.385547</td>\n",
       "      <td id=\"T_186f7_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row5_col4\" class=\"data row5 col4\" >0.075464</td>\n",
       "      <td id=\"T_186f7_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_186f7_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_186f7_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_186f7_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row6_col2\" class=\"data row6 col2\" >0.184416</td>\n",
       "      <td id=\"T_186f7_row6_col3\" class=\"data row6 col3\" >0.184416</td>\n",
       "      <td id=\"T_186f7_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_186f7_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8808c3e130>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### AOF_performance_val"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_f22b3_row0_col0, #T_f22b3_row0_col4, #T_f22b3_row1_col1, #T_f22b3_row1_col4, #T_f22b3_row1_col5, #T_f22b3_row2_col2, #T_f22b3_row2_col3, #T_f22b3_row3_col2, #T_f22b3_row3_col3, #T_f22b3_row3_col6, #T_f22b3_row4_col0, #T_f22b3_row4_col1, #T_f22b3_row4_col4, #T_f22b3_row5_col1, #T_f22b3_row5_col5, #T_f22b3_row6_col3, #T_f22b3_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_f22b3_row0_col1, #T_f22b3_row0_col2, #T_f22b3_row0_col3, #T_f22b3_row0_col6, #T_f22b3_row1_col0, #T_f22b3_row1_col2, #T_f22b3_row1_col3, #T_f22b3_row1_col6, #T_f22b3_row2_col0, #T_f22b3_row2_col1, #T_f22b3_row2_col4, #T_f22b3_row2_col5, #T_f22b3_row2_col6, #T_f22b3_row3_col0, #T_f22b3_row3_col1, #T_f22b3_row3_col4, #T_f22b3_row3_col5, #T_f22b3_row4_col2, #T_f22b3_row4_col3, #T_f22b3_row4_col6, #T_f22b3_row5_col2, #T_f22b3_row5_col3, #T_f22b3_row5_col6, #T_f22b3_row6_col0, #T_f22b3_row6_col1, #T_f22b3_row6_col2, #T_f22b3_row6_col4, #T_f22b3_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_f22b3_row0_col5, #T_f22b3_row5_col0 {\n",
       "  background-color: #fcba03;\n",
       "}\n",
       "#T_f22b3_row4_col5, #T_f22b3_row5_col4 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_f22b3_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_f22b3_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_f22b3_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_f22b3_row0_col1\" class=\"data row0 col1\" >0.007656</td>\n",
       "      <td id=\"T_f22b3_row0_col2\" class=\"data row0 col2\" >0.000006</td>\n",
       "      <td id=\"T_f22b3_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row0_col4\" class=\"data row0 col4\" >0.678138</td>\n",
       "      <td id=\"T_f22b3_row0_col5\" class=\"data row0 col5\" >0.026248</td>\n",
       "      <td id=\"T_f22b3_row0_col6\" class=\"data row0 col6\" >0.000006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f22b3_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_f22b3_row1_col0\" class=\"data row1 col0\" >0.007656</td>\n",
       "      <td id=\"T_f22b3_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_f22b3_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row1_col4\" class=\"data row1 col4\" >0.184416</td>\n",
       "      <td id=\"T_f22b3_row1_col5\" class=\"data row1 col5\" >0.678138</td>\n",
       "      <td id=\"T_f22b3_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f22b3_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_f22b3_row2_col0\" class=\"data row2 col0\" >0.000006</td>\n",
       "      <td id=\"T_f22b3_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_f22b3_row2_col3\" class=\"data row2 col3\" >0.184416</td>\n",
       "      <td id=\"T_f22b3_row2_col4\" class=\"data row2 col4\" >0.000052</td>\n",
       "      <td id=\"T_f22b3_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row2_col6\" class=\"data row2 col6\" >0.007656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f22b3_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_f22b3_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row3_col2\" class=\"data row3 col2\" >0.184416</td>\n",
       "      <td id=\"T_f22b3_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_f22b3_row3_col4\" class=\"data row3 col4\" >0.000353</td>\n",
       "      <td id=\"T_f22b3_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row3_col6\" class=\"data row3 col6\" >0.678138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f22b3_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_f22b3_row4_col0\" class=\"data row4 col0\" >0.678138</td>\n",
       "      <td id=\"T_f22b3_row4_col1\" class=\"data row4 col1\" >0.184416</td>\n",
       "      <td id=\"T_f22b3_row4_col2\" class=\"data row4 col2\" >0.000052</td>\n",
       "      <td id=\"T_f22b3_row4_col3\" class=\"data row4 col3\" >0.000353</td>\n",
       "      <td id=\"T_f22b3_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_f22b3_row4_col5\" class=\"data row4 col5\" >0.075464</td>\n",
       "      <td id=\"T_f22b3_row4_col6\" class=\"data row4 col6\" >0.000353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f22b3_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_f22b3_row5_col0\" class=\"data row5 col0\" >0.026248</td>\n",
       "      <td id=\"T_f22b3_row5_col1\" class=\"data row5 col1\" >0.678138</td>\n",
       "      <td id=\"T_f22b3_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row5_col4\" class=\"data row5 col4\" >0.075464</td>\n",
       "      <td id=\"T_f22b3_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_f22b3_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f22b3_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_f22b3_row6_col0\" class=\"data row6 col0\" >0.000006</td>\n",
       "      <td id=\"T_f22b3_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row6_col2\" class=\"data row6 col2\" >0.007656</td>\n",
       "      <td id=\"T_f22b3_row6_col3\" class=\"data row6 col3\" >0.678138</td>\n",
       "      <td id=\"T_f22b3_row6_col4\" class=\"data row6 col4\" >0.000353</td>\n",
       "      <td id=\"T_f22b3_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_f22b3_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a71839a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### COMPAS_performance_val"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_55d77_row0_col0, #T_55d77_row0_col1, #T_55d77_row1_col0, #T_55d77_row1_col1, #T_55d77_row1_col4, #T_55d77_row2_col2, #T_55d77_row2_col3, #T_55d77_row3_col2, #T_55d77_row3_col3, #T_55d77_row3_col6, #T_55d77_row4_col1, #T_55d77_row4_col4, #T_55d77_row4_col5, #T_55d77_row5_col4, #T_55d77_row5_col5, #T_55d77_row6_col3, #T_55d77_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_55d77_row0_col2, #T_55d77_row0_col3, #T_55d77_row0_col4, #T_55d77_row0_col5, #T_55d77_row0_col6, #T_55d77_row1_col2, #T_55d77_row1_col3, #T_55d77_row1_col6, #T_55d77_row2_col0, #T_55d77_row2_col1, #T_55d77_row2_col4, #T_55d77_row2_col5, #T_55d77_row3_col0, #T_55d77_row3_col1, #T_55d77_row3_col4, #T_55d77_row3_col5, #T_55d77_row4_col0, #T_55d77_row4_col2, #T_55d77_row4_col3, #T_55d77_row4_col6, #T_55d77_row5_col0, #T_55d77_row5_col2, #T_55d77_row5_col3, #T_55d77_row5_col6, #T_55d77_row6_col0, #T_55d77_row6_col1, #T_55d77_row6_col4, #T_55d77_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_55d77_row1_col5, #T_55d77_row2_col6, #T_55d77_row5_col1, #T_55d77_row6_col2 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_55d77_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_55d77_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_55d77_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_55d77_row0_col1\" class=\"data row0 col1\" >0.184416</td>\n",
       "      <td id=\"T_55d77_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row0_col4\" class=\"data row0 col4\" >0.001837</td>\n",
       "      <td id=\"T_55d77_row0_col5\" class=\"data row0 col5\" >0.001837</td>\n",
       "      <td id=\"T_55d77_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_55d77_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_55d77_row1_col0\" class=\"data row1 col0\" >0.184416</td>\n",
       "      <td id=\"T_55d77_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_55d77_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row1_col4\" class=\"data row1 col4\" >0.184416</td>\n",
       "      <td id=\"T_55d77_row1_col5\" class=\"data row1 col5\" >0.075464</td>\n",
       "      <td id=\"T_55d77_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_55d77_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_55d77_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_55d77_row2_col3\" class=\"data row2 col3\" >0.184416</td>\n",
       "      <td id=\"T_55d77_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row2_col6\" class=\"data row2 col6\" >0.075464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_55d77_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_55d77_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row3_col2\" class=\"data row3 col2\" >0.184416</td>\n",
       "      <td id=\"T_55d77_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_55d77_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row3_col6\" class=\"data row3 col6\" >0.938331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_55d77_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_55d77_row4_col0\" class=\"data row4 col0\" >0.001837</td>\n",
       "      <td id=\"T_55d77_row4_col1\" class=\"data row4 col1\" >0.184416</td>\n",
       "      <td id=\"T_55d77_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_55d77_row4_col5\" class=\"data row4 col5\" >0.999789</td>\n",
       "      <td id=\"T_55d77_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_55d77_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_55d77_row5_col0\" class=\"data row5 col0\" >0.001837</td>\n",
       "      <td id=\"T_55d77_row5_col1\" class=\"data row5 col1\" >0.075464</td>\n",
       "      <td id=\"T_55d77_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row5_col4\" class=\"data row5 col4\" >0.999789</td>\n",
       "      <td id=\"T_55d77_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_55d77_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_55d77_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_55d77_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row6_col2\" class=\"data row6 col2\" >0.075464</td>\n",
       "      <td id=\"T_55d77_row6_col3\" class=\"data row6 col3\" >0.938331</td>\n",
       "      <td id=\"T_55d77_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_55d77_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8804426be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### Donors Choose_performance_val"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_f2967_row0_col0, #T_f2967_row0_col1, #T_f2967_row1_col0, #T_f2967_row1_col1, #T_f2967_row2_col2, #T_f2967_row3_col3, #T_f2967_row3_col6, #T_f2967_row4_col4, #T_f2967_row4_col5, #T_f2967_row5_col4, #T_f2967_row5_col5, #T_f2967_row6_col3, #T_f2967_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_f2967_row0_col2, #T_f2967_row0_col3, #T_f2967_row0_col4, #T_f2967_row0_col5, #T_f2967_row0_col6, #T_f2967_row1_col2, #T_f2967_row1_col3, #T_f2967_row1_col4, #T_f2967_row1_col5, #T_f2967_row1_col6, #T_f2967_row2_col0, #T_f2967_row2_col1, #T_f2967_row2_col4, #T_f2967_row2_col5, #T_f2967_row2_col6, #T_f2967_row3_col0, #T_f2967_row3_col1, #T_f2967_row3_col4, #T_f2967_row3_col5, #T_f2967_row4_col0, #T_f2967_row4_col1, #T_f2967_row4_col2, #T_f2967_row4_col3, #T_f2967_row4_col6, #T_f2967_row5_col0, #T_f2967_row5_col1, #T_f2967_row5_col2, #T_f2967_row5_col3, #T_f2967_row5_col6, #T_f2967_row6_col0, #T_f2967_row6_col1, #T_f2967_row6_col2, #T_f2967_row6_col4, #T_f2967_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_f2967_row2_col3, #T_f2967_row3_col2 {\n",
       "  background-color: #fcba03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_f2967_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_f2967_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_f2967_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_f2967_row0_col1\" class=\"data row0 col1\" >0.678138</td>\n",
       "      <td id=\"T_f2967_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row0_col4\" class=\"data row0 col4\" >0.000353</td>\n",
       "      <td id=\"T_f2967_row0_col5\" class=\"data row0 col5\" >0.000353</td>\n",
       "      <td id=\"T_f2967_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f2967_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_f2967_row1_col0\" class=\"data row1 col0\" >0.678138</td>\n",
       "      <td id=\"T_f2967_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_f2967_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row1_col4\" class=\"data row1 col4\" >0.000353</td>\n",
       "      <td id=\"T_f2967_row1_col5\" class=\"data row1 col5\" >0.000353</td>\n",
       "      <td id=\"T_f2967_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f2967_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_f2967_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_f2967_row2_col3\" class=\"data row2 col3\" >0.026248</td>\n",
       "      <td id=\"T_f2967_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row2_col6\" class=\"data row2 col6\" >0.007656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f2967_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_f2967_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row3_col2\" class=\"data row3 col2\" >0.026248</td>\n",
       "      <td id=\"T_f2967_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_f2967_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row3_col6\" class=\"data row3 col6\" >0.678138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f2967_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_f2967_row4_col0\" class=\"data row4 col0\" >0.000353</td>\n",
       "      <td id=\"T_f2967_row4_col1\" class=\"data row4 col1\" >0.000353</td>\n",
       "      <td id=\"T_f2967_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_f2967_row4_col5\" class=\"data row4 col5\" >0.678138</td>\n",
       "      <td id=\"T_f2967_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f2967_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_f2967_row5_col0\" class=\"data row5 col0\" >0.000353</td>\n",
       "      <td id=\"T_f2967_row5_col1\" class=\"data row5 col1\" >0.000353</td>\n",
       "      <td id=\"T_f2967_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row5_col4\" class=\"data row5 col4\" >0.678138</td>\n",
       "      <td id=\"T_f2967_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_f2967_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_f2967_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_f2967_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row6_col2\" class=\"data row6 col2\" >0.007656</td>\n",
       "      <td id=\"T_f2967_row6_col3\" class=\"data row6 col3\" >0.678138</td>\n",
       "      <td id=\"T_f2967_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_f2967_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a870a5e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### Adult_fairness_val"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_08661_row0_col0, #T_08661_row0_col1, #T_08661_row1_col0, #T_08661_row1_col1, #T_08661_row2_col2, #T_08661_row2_col6, #T_08661_row3_col3, #T_08661_row4_col4, #T_08661_row4_col5, #T_08661_row5_col4, #T_08661_row5_col5, #T_08661_row6_col2, #T_08661_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_08661_row0_col2, #T_08661_row0_col3, #T_08661_row0_col4, #T_08661_row0_col5, #T_08661_row0_col6, #T_08661_row1_col2, #T_08661_row1_col3, #T_08661_row1_col4, #T_08661_row1_col5, #T_08661_row1_col6, #T_08661_row2_col0, #T_08661_row2_col1, #T_08661_row2_col3, #T_08661_row2_col4, #T_08661_row2_col5, #T_08661_row3_col0, #T_08661_row3_col1, #T_08661_row3_col2, #T_08661_row3_col4, #T_08661_row3_col5, #T_08661_row4_col0, #T_08661_row4_col1, #T_08661_row4_col2, #T_08661_row4_col3, #T_08661_row4_col6, #T_08661_row5_col0, #T_08661_row5_col1, #T_08661_row5_col2, #T_08661_row5_col3, #T_08661_row5_col6, #T_08661_row6_col0, #T_08661_row6_col1, #T_08661_row6_col4, #T_08661_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_08661_row3_col6, #T_08661_row6_col3 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_08661_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_08661_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_08661_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_08661_row0_col1\" class=\"data row0 col1\" >0.678138</td>\n",
       "      <td id=\"T_08661_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_08661_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_08661_row0_col4\" class=\"data row0 col4\" >0.000000</td>\n",
       "      <td id=\"T_08661_row0_col5\" class=\"data row0 col5\" >0.000000</td>\n",
       "      <td id=\"T_08661_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_08661_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_08661_row1_col0\" class=\"data row1 col0\" >0.678138</td>\n",
       "      <td id=\"T_08661_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_08661_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_08661_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_08661_row1_col4\" class=\"data row1 col4\" >0.000052</td>\n",
       "      <td id=\"T_08661_row1_col5\" class=\"data row1 col5\" >0.000052</td>\n",
       "      <td id=\"T_08661_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_08661_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_08661_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_08661_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_08661_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_08661_row2_col3\" class=\"data row2 col3\" >0.007656</td>\n",
       "      <td id=\"T_08661_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_08661_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_08661_row2_col6\" class=\"data row2 col6\" >0.938331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_08661_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_08661_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_08661_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_08661_row3_col2\" class=\"data row3 col2\" >0.007656</td>\n",
       "      <td id=\"T_08661_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_08661_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_08661_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_08661_row3_col6\" class=\"data row3 col6\" >0.075464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_08661_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_08661_row4_col0\" class=\"data row4 col0\" >0.000000</td>\n",
       "      <td id=\"T_08661_row4_col1\" class=\"data row4 col1\" >0.000052</td>\n",
       "      <td id=\"T_08661_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_08661_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_08661_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_08661_row4_col5\" class=\"data row4 col5\" >0.678138</td>\n",
       "      <td id=\"T_08661_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_08661_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_08661_row5_col0\" class=\"data row5 col0\" >0.000000</td>\n",
       "      <td id=\"T_08661_row5_col1\" class=\"data row5 col1\" >0.000052</td>\n",
       "      <td id=\"T_08661_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_08661_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_08661_row5_col4\" class=\"data row5 col4\" >0.678138</td>\n",
       "      <td id=\"T_08661_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_08661_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_08661_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_08661_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_08661_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_08661_row6_col2\" class=\"data row6 col2\" >0.938331</td>\n",
       "      <td id=\"T_08661_row6_col3\" class=\"data row6 col3\" >0.075464</td>\n",
       "      <td id=\"T_08661_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_08661_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_08661_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a71838e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### AOF_fairness_val"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_6178e_row0_col0, #T_6178e_row0_col1, #T_6178e_row0_col4, #T_6178e_row0_col5, #T_6178e_row1_col0, #T_6178e_row1_col1, #T_6178e_row1_col4, #T_6178e_row1_col5, #T_6178e_row2_col2, #T_6178e_row3_col3, #T_6178e_row3_col6, #T_6178e_row4_col0, #T_6178e_row4_col1, #T_6178e_row4_col4, #T_6178e_row5_col0, #T_6178e_row5_col1, #T_6178e_row5_col5, #T_6178e_row6_col3, #T_6178e_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_6178e_row0_col2, #T_6178e_row0_col3, #T_6178e_row0_col6, #T_6178e_row1_col2, #T_6178e_row1_col3, #T_6178e_row1_col6, #T_6178e_row2_col0, #T_6178e_row2_col1, #T_6178e_row2_col3, #T_6178e_row2_col4, #T_6178e_row2_col5, #T_6178e_row2_col6, #T_6178e_row3_col0, #T_6178e_row3_col1, #T_6178e_row3_col2, #T_6178e_row3_col4, #T_6178e_row3_col5, #T_6178e_row4_col2, #T_6178e_row4_col3, #T_6178e_row4_col6, #T_6178e_row5_col2, #T_6178e_row5_col3, #T_6178e_row5_col6, #T_6178e_row6_col0, #T_6178e_row6_col1, #T_6178e_row6_col2, #T_6178e_row6_col4, #T_6178e_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_6178e_row4_col5, #T_6178e_row5_col4 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_6178e_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_6178e_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_6178e_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_6178e_row0_col1\" class=\"data row0 col1\" >0.184416</td>\n",
       "      <td id=\"T_6178e_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row0_col4\" class=\"data row0 col4\" >0.184416</td>\n",
       "      <td id=\"T_6178e_row0_col5\" class=\"data row0 col5\" >0.184416</td>\n",
       "      <td id=\"T_6178e_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6178e_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_6178e_row1_col0\" class=\"data row1 col0\" >0.184416</td>\n",
       "      <td id=\"T_6178e_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_6178e_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row1_col4\" class=\"data row1 col4\" >0.184416</td>\n",
       "      <td id=\"T_6178e_row1_col5\" class=\"data row1 col5\" >0.678138</td>\n",
       "      <td id=\"T_6178e_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6178e_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_6178e_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_6178e_row2_col3\" class=\"data row2 col3\" >0.007656</td>\n",
       "      <td id=\"T_6178e_row2_col4\" class=\"data row2 col4\" >0.000353</td>\n",
       "      <td id=\"T_6178e_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row2_col6\" class=\"data row2 col6\" >0.001837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6178e_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_6178e_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row3_col2\" class=\"data row3 col2\" >0.007656</td>\n",
       "      <td id=\"T_6178e_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_6178e_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row3_col6\" class=\"data row3 col6\" >0.938331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6178e_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_6178e_row4_col0\" class=\"data row4 col0\" >0.184416</td>\n",
       "      <td id=\"T_6178e_row4_col1\" class=\"data row4 col1\" >0.184416</td>\n",
       "      <td id=\"T_6178e_row4_col2\" class=\"data row4 col2\" >0.000353</td>\n",
       "      <td id=\"T_6178e_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_6178e_row4_col5\" class=\"data row4 col5\" >0.075464</td>\n",
       "      <td id=\"T_6178e_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6178e_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_6178e_row5_col0\" class=\"data row5 col0\" >0.184416</td>\n",
       "      <td id=\"T_6178e_row5_col1\" class=\"data row5 col1\" >0.678138</td>\n",
       "      <td id=\"T_6178e_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row5_col4\" class=\"data row5 col4\" >0.075464</td>\n",
       "      <td id=\"T_6178e_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_6178e_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6178e_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_6178e_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row6_col2\" class=\"data row6 col2\" >0.001837</td>\n",
       "      <td id=\"T_6178e_row6_col3\" class=\"data row6 col3\" >0.938331</td>\n",
       "      <td id=\"T_6178e_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_6178e_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8804426bb0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### COMPAS_fairness_val"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_29c26_row0_col0, #T_29c26_row0_col1, #T_29c26_row1_col0, #T_29c26_row1_col1, #T_29c26_row2_col2, #T_29c26_row2_col3, #T_29c26_row2_col6, #T_29c26_row3_col2, #T_29c26_row3_col3, #T_29c26_row3_col6, #T_29c26_row4_col4, #T_29c26_row4_col5, #T_29c26_row5_col4, #T_29c26_row5_col5, #T_29c26_row6_col2, #T_29c26_row6_col3, #T_29c26_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_29c26_row0_col2, #T_29c26_row0_col3, #T_29c26_row0_col4, #T_29c26_row0_col5, #T_29c26_row0_col6, #T_29c26_row1_col2, #T_29c26_row1_col3, #T_29c26_row1_col4, #T_29c26_row1_col5, #T_29c26_row1_col6, #T_29c26_row2_col0, #T_29c26_row2_col1, #T_29c26_row2_col4, #T_29c26_row2_col5, #T_29c26_row3_col0, #T_29c26_row3_col1, #T_29c26_row3_col4, #T_29c26_row3_col5, #T_29c26_row4_col0, #T_29c26_row4_col1, #T_29c26_row4_col2, #T_29c26_row4_col3, #T_29c26_row4_col6, #T_29c26_row5_col0, #T_29c26_row5_col1, #T_29c26_row5_col2, #T_29c26_row5_col3, #T_29c26_row5_col6, #T_29c26_row6_col0, #T_29c26_row6_col1, #T_29c26_row6_col4, #T_29c26_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_29c26_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_29c26_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_29c26_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_29c26_row0_col1\" class=\"data row0 col1\" >0.999789</td>\n",
       "      <td id=\"T_29c26_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row0_col4\" class=\"data row0 col4\" >0.000353</td>\n",
       "      <td id=\"T_29c26_row0_col5\" class=\"data row0 col5\" >0.007656</td>\n",
       "      <td id=\"T_29c26_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_29c26_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_29c26_row1_col0\" class=\"data row1 col0\" >0.999789</td>\n",
       "      <td id=\"T_29c26_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_29c26_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row1_col4\" class=\"data row1 col4\" >0.000353</td>\n",
       "      <td id=\"T_29c26_row1_col5\" class=\"data row1 col5\" >0.007656</td>\n",
       "      <td id=\"T_29c26_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_29c26_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_29c26_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_29c26_row2_col3\" class=\"data row2 col3\" >0.999789</td>\n",
       "      <td id=\"T_29c26_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row2_col6\" class=\"data row2 col6\" >0.678138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_29c26_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_29c26_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row3_col2\" class=\"data row3 col2\" >0.999789</td>\n",
       "      <td id=\"T_29c26_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_29c26_row3_col4\" class=\"data row3 col4\" >0.000006</td>\n",
       "      <td id=\"T_29c26_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row3_col6\" class=\"data row3 col6\" >0.938331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_29c26_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_29c26_row4_col0\" class=\"data row4 col0\" >0.000353</td>\n",
       "      <td id=\"T_29c26_row4_col1\" class=\"data row4 col1\" >0.000353</td>\n",
       "      <td id=\"T_29c26_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row4_col3\" class=\"data row4 col3\" >0.000006</td>\n",
       "      <td id=\"T_29c26_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_29c26_row4_col5\" class=\"data row4 col5\" >0.938331</td>\n",
       "      <td id=\"T_29c26_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_29c26_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_29c26_row5_col0\" class=\"data row5 col0\" >0.007656</td>\n",
       "      <td id=\"T_29c26_row5_col1\" class=\"data row5 col1\" >0.007656</td>\n",
       "      <td id=\"T_29c26_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row5_col4\" class=\"data row5 col4\" >0.938331</td>\n",
       "      <td id=\"T_29c26_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_29c26_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_29c26_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_29c26_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row6_col2\" class=\"data row6 col2\" >0.678138</td>\n",
       "      <td id=\"T_29c26_row6_col3\" class=\"data row6 col3\" >0.938331</td>\n",
       "      <td id=\"T_29c26_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_29c26_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a6588f10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### Donors Choose_fairness_val"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_a6496_row0_col0, #T_a6496_row0_col1, #T_a6496_row1_col0, #T_a6496_row1_col1, #T_a6496_row1_col4, #T_a6496_row1_col5, #T_a6496_row2_col2, #T_a6496_row3_col3, #T_a6496_row3_col6, #T_a6496_row4_col1, #T_a6496_row4_col4, #T_a6496_row4_col5, #T_a6496_row5_col1, #T_a6496_row5_col4, #T_a6496_row5_col5, #T_a6496_row6_col3, #T_a6496_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_a6496_row0_col2, #T_a6496_row0_col3, #T_a6496_row0_col6, #T_a6496_row1_col2, #T_a6496_row1_col3, #T_a6496_row1_col6, #T_a6496_row2_col0, #T_a6496_row2_col1, #T_a6496_row2_col3, #T_a6496_row2_col4, #T_a6496_row2_col5, #T_a6496_row3_col0, #T_a6496_row3_col1, #T_a6496_row3_col2, #T_a6496_row3_col4, #T_a6496_row3_col5, #T_a6496_row4_col2, #T_a6496_row4_col3, #T_a6496_row4_col6, #T_a6496_row5_col2, #T_a6496_row5_col3, #T_a6496_row5_col6, #T_a6496_row6_col0, #T_a6496_row6_col1, #T_a6496_row6_col4, #T_a6496_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_a6496_row0_col4, #T_a6496_row2_col6, #T_a6496_row4_col0, #T_a6496_row6_col2 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "#T_a6496_row0_col5, #T_a6496_row5_col0 {\n",
       "  background-color: #fcba03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_a6496_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_a6496_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_a6496_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_a6496_row0_col1\" class=\"data row0 col1\" >0.385547</td>\n",
       "      <td id=\"T_a6496_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row0_col4\" class=\"data row0 col4\" >0.075464</td>\n",
       "      <td id=\"T_a6496_row0_col5\" class=\"data row0 col5\" >0.026248</td>\n",
       "      <td id=\"T_a6496_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a6496_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_a6496_row1_col0\" class=\"data row1 col0\" >0.385547</td>\n",
       "      <td id=\"T_a6496_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_a6496_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row1_col4\" class=\"data row1 col4\" >0.678138</td>\n",
       "      <td id=\"T_a6496_row1_col5\" class=\"data row1 col5\" >0.385547</td>\n",
       "      <td id=\"T_a6496_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a6496_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_a6496_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_a6496_row2_col3\" class=\"data row2 col3\" >0.001837</td>\n",
       "      <td id=\"T_a6496_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row2_col6\" class=\"data row2 col6\" >0.075464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a6496_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_a6496_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row3_col2\" class=\"data row3 col2\" >0.001837</td>\n",
       "      <td id=\"T_a6496_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_a6496_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row3_col6\" class=\"data row3 col6\" >0.678138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a6496_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_a6496_row4_col0\" class=\"data row4 col0\" >0.075464</td>\n",
       "      <td id=\"T_a6496_row4_col1\" class=\"data row4 col1\" >0.678138</td>\n",
       "      <td id=\"T_a6496_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_a6496_row4_col5\" class=\"data row4 col5\" >0.938331</td>\n",
       "      <td id=\"T_a6496_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a6496_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_a6496_row5_col0\" class=\"data row5 col0\" >0.026248</td>\n",
       "      <td id=\"T_a6496_row5_col1\" class=\"data row5 col1\" >0.385547</td>\n",
       "      <td id=\"T_a6496_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row5_col4\" class=\"data row5 col4\" >0.938331</td>\n",
       "      <td id=\"T_a6496_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_a6496_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a6496_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_a6496_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row6_col2\" class=\"data row6 col2\" >0.075464</td>\n",
       "      <td id=\"T_a6496_row6_col3\" class=\"data row6 col3\" >0.678138</td>\n",
       "      <td id=\"T_a6496_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_a6496_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8808c3e130>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### Adult_performance_test"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_5dbb2_row0_col0, #T_5dbb2_row0_col1, #T_5dbb2_row0_col5, #T_5dbb2_row1_col0, #T_5dbb2_row1_col1, #T_5dbb2_row1_col5, #T_5dbb2_row2_col2, #T_5dbb2_row2_col6, #T_5dbb2_row3_col3, #T_5dbb2_row3_col6, #T_5dbb2_row4_col4, #T_5dbb2_row5_col0, #T_5dbb2_row5_col1, #T_5dbb2_row5_col5, #T_5dbb2_row6_col2, #T_5dbb2_row6_col3, #T_5dbb2_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_5dbb2_row0_col2, #T_5dbb2_row0_col3, #T_5dbb2_row0_col4, #T_5dbb2_row0_col6, #T_5dbb2_row1_col2, #T_5dbb2_row1_col3, #T_5dbb2_row1_col4, #T_5dbb2_row1_col6, #T_5dbb2_row2_col0, #T_5dbb2_row2_col1, #T_5dbb2_row2_col3, #T_5dbb2_row2_col4, #T_5dbb2_row2_col5, #T_5dbb2_row3_col0, #T_5dbb2_row3_col1, #T_5dbb2_row3_col2, #T_5dbb2_row3_col4, #T_5dbb2_row3_col5, #T_5dbb2_row4_col0, #T_5dbb2_row4_col1, #T_5dbb2_row4_col2, #T_5dbb2_row4_col3, #T_5dbb2_row4_col6, #T_5dbb2_row5_col2, #T_5dbb2_row5_col3, #T_5dbb2_row5_col6, #T_5dbb2_row6_col0, #T_5dbb2_row6_col1, #T_5dbb2_row6_col4, #T_5dbb2_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_5dbb2_row4_col5, #T_5dbb2_row5_col4 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_5dbb2_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_5dbb2_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_5dbb2_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_5dbb2_row0_col1\" class=\"data row0 col1\" >0.678138</td>\n",
       "      <td id=\"T_5dbb2_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row0_col4\" class=\"data row0 col4\" >0.007656</td>\n",
       "      <td id=\"T_5dbb2_row0_col5\" class=\"data row0 col5\" >0.184416</td>\n",
       "      <td id=\"T_5dbb2_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5dbb2_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_5dbb2_row1_col0\" class=\"data row1 col0\" >0.678138</td>\n",
       "      <td id=\"T_5dbb2_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_5dbb2_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row1_col4\" class=\"data row1 col4\" >0.007656</td>\n",
       "      <td id=\"T_5dbb2_row1_col5\" class=\"data row1 col5\" >0.385547</td>\n",
       "      <td id=\"T_5dbb2_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5dbb2_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_5dbb2_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_5dbb2_row2_col3\" class=\"data row2 col3\" >0.007656</td>\n",
       "      <td id=\"T_5dbb2_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row2_col6\" class=\"data row2 col6\" >0.385547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5dbb2_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_5dbb2_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row3_col2\" class=\"data row3 col2\" >0.007656</td>\n",
       "      <td id=\"T_5dbb2_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_5dbb2_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row3_col6\" class=\"data row3 col6\" >0.184416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5dbb2_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_5dbb2_row4_col0\" class=\"data row4 col0\" >0.007656</td>\n",
       "      <td id=\"T_5dbb2_row4_col1\" class=\"data row4 col1\" >0.007656</td>\n",
       "      <td id=\"T_5dbb2_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_5dbb2_row4_col5\" class=\"data row4 col5\" >0.075464</td>\n",
       "      <td id=\"T_5dbb2_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5dbb2_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_5dbb2_row5_col0\" class=\"data row5 col0\" >0.184416</td>\n",
       "      <td id=\"T_5dbb2_row5_col1\" class=\"data row5 col1\" >0.385547</td>\n",
       "      <td id=\"T_5dbb2_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row5_col4\" class=\"data row5 col4\" >0.075464</td>\n",
       "      <td id=\"T_5dbb2_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_5dbb2_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5dbb2_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_5dbb2_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row6_col2\" class=\"data row6 col2\" >0.385547</td>\n",
       "      <td id=\"T_5dbb2_row6_col3\" class=\"data row6 col3\" >0.184416</td>\n",
       "      <td id=\"T_5dbb2_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_5dbb2_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a65743d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### AOF_performance_test"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_6a87d_row0_col0, #T_6a87d_row0_col4, #T_6a87d_row0_col5, #T_6a87d_row1_col1, #T_6a87d_row1_col5, #T_6a87d_row2_col2, #T_6a87d_row2_col3, #T_6a87d_row2_col4, #T_6a87d_row2_col6, #T_6a87d_row3_col2, #T_6a87d_row3_col3, #T_6a87d_row3_col4, #T_6a87d_row3_col6, #T_6a87d_row4_col0, #T_6a87d_row4_col2, #T_6a87d_row4_col3, #T_6a87d_row4_col4, #T_6a87d_row4_col5, #T_6a87d_row5_col0, #T_6a87d_row5_col1, #T_6a87d_row5_col4, #T_6a87d_row5_col5, #T_6a87d_row6_col2, #T_6a87d_row6_col3, #T_6a87d_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_6a87d_row0_col1, #T_6a87d_row0_col2, #T_6a87d_row1_col0, #T_6a87d_row2_col0 {\n",
       "  background-color: #fcba03;\n",
       "}\n",
       "#T_6a87d_row0_col3, #T_6a87d_row0_col6, #T_6a87d_row1_col2, #T_6a87d_row1_col3, #T_6a87d_row1_col4, #T_6a87d_row1_col6, #T_6a87d_row2_col1, #T_6a87d_row2_col5, #T_6a87d_row3_col0, #T_6a87d_row3_col1, #T_6a87d_row3_col5, #T_6a87d_row4_col1, #T_6a87d_row5_col2, #T_6a87d_row5_col3, #T_6a87d_row5_col6, #T_6a87d_row6_col0, #T_6a87d_row6_col1, #T_6a87d_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_6a87d_row4_col6, #T_6a87d_row6_col4 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_6a87d_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_6a87d_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_6a87d_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_6a87d_row0_col1\" class=\"data row0 col1\" >0.026248</td>\n",
       "      <td id=\"T_6a87d_row0_col2\" class=\"data row0 col2\" >0.026248</td>\n",
       "      <td id=\"T_6a87d_row0_col3\" class=\"data row0 col3\" >0.001837</td>\n",
       "      <td id=\"T_6a87d_row0_col4\" class=\"data row0 col4\" >0.678138</td>\n",
       "      <td id=\"T_6a87d_row0_col5\" class=\"data row0 col5\" >0.184416</td>\n",
       "      <td id=\"T_6a87d_row0_col6\" class=\"data row0 col6\" >0.007656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6a87d_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_6a87d_row1_col0\" class=\"data row1 col0\" >0.026248</td>\n",
       "      <td id=\"T_6a87d_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_6a87d_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_6a87d_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_6a87d_row1_col4\" class=\"data row1 col4\" >0.001837</td>\n",
       "      <td id=\"T_6a87d_row1_col5\" class=\"data row1 col5\" >0.184416</td>\n",
       "      <td id=\"T_6a87d_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6a87d_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_6a87d_row2_col0\" class=\"data row2 col0\" >0.026248</td>\n",
       "      <td id=\"T_6a87d_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_6a87d_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_6a87d_row2_col3\" class=\"data row2 col3\" >0.678138</td>\n",
       "      <td id=\"T_6a87d_row2_col4\" class=\"data row2 col4\" >0.184416</td>\n",
       "      <td id=\"T_6a87d_row2_col5\" class=\"data row2 col5\" >0.007656</td>\n",
       "      <td id=\"T_6a87d_row2_col6\" class=\"data row2 col6\" >0.938331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6a87d_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_6a87d_row3_col0\" class=\"data row3 col0\" >0.001837</td>\n",
       "      <td id=\"T_6a87d_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_6a87d_row3_col2\" class=\"data row3 col2\" >0.678138</td>\n",
       "      <td id=\"T_6a87d_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_6a87d_row3_col4\" class=\"data row3 col4\" >0.184416</td>\n",
       "      <td id=\"T_6a87d_row3_col5\" class=\"data row3 col5\" >0.007656</td>\n",
       "      <td id=\"T_6a87d_row3_col6\" class=\"data row3 col6\" >0.999789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6a87d_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_6a87d_row4_col0\" class=\"data row4 col0\" >0.678138</td>\n",
       "      <td id=\"T_6a87d_row4_col1\" class=\"data row4 col1\" >0.001837</td>\n",
       "      <td id=\"T_6a87d_row4_col2\" class=\"data row4 col2\" >0.184416</td>\n",
       "      <td id=\"T_6a87d_row4_col3\" class=\"data row4 col3\" >0.184416</td>\n",
       "      <td id=\"T_6a87d_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_6a87d_row4_col5\" class=\"data row4 col5\" >0.385547</td>\n",
       "      <td id=\"T_6a87d_row4_col6\" class=\"data row4 col6\" >0.075464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6a87d_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_6a87d_row5_col0\" class=\"data row5 col0\" >0.184416</td>\n",
       "      <td id=\"T_6a87d_row5_col1\" class=\"data row5 col1\" >0.184416</td>\n",
       "      <td id=\"T_6a87d_row5_col2\" class=\"data row5 col2\" >0.007656</td>\n",
       "      <td id=\"T_6a87d_row5_col3\" class=\"data row5 col3\" >0.007656</td>\n",
       "      <td id=\"T_6a87d_row5_col4\" class=\"data row5 col4\" >0.385547</td>\n",
       "      <td id=\"T_6a87d_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_6a87d_row5_col6\" class=\"data row5 col6\" >0.007656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_6a87d_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_6a87d_row6_col0\" class=\"data row6 col0\" >0.007656</td>\n",
       "      <td id=\"T_6a87d_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_6a87d_row6_col2\" class=\"data row6 col2\" >0.938331</td>\n",
       "      <td id=\"T_6a87d_row6_col3\" class=\"data row6 col3\" >0.999789</td>\n",
       "      <td id=\"T_6a87d_row6_col4\" class=\"data row6 col4\" >0.075464</td>\n",
       "      <td id=\"T_6a87d_row6_col5\" class=\"data row6 col5\" >0.007656</td>\n",
       "      <td id=\"T_6a87d_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a870a5e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### COMPAS_performance_test"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_1db2d_row0_col0, #T_1db2d_row0_col1, #T_1db2d_row1_col0, #T_1db2d_row1_col1, #T_1db2d_row2_col2, #T_1db2d_row2_col3, #T_1db2d_row2_col6, #T_1db2d_row3_col2, #T_1db2d_row3_col3, #T_1db2d_row3_col6, #T_1db2d_row4_col4, #T_1db2d_row4_col5, #T_1db2d_row5_col4, #T_1db2d_row5_col5, #T_1db2d_row6_col2, #T_1db2d_row6_col3, #T_1db2d_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_1db2d_row0_col2, #T_1db2d_row0_col4, #T_1db2d_row0_col5, #T_1db2d_row0_col6, #T_1db2d_row1_col2, #T_1db2d_row1_col3, #T_1db2d_row1_col6, #T_1db2d_row2_col0, #T_1db2d_row2_col1, #T_1db2d_row2_col4, #T_1db2d_row2_col5, #T_1db2d_row3_col1, #T_1db2d_row3_col4, #T_1db2d_row3_col5, #T_1db2d_row4_col0, #T_1db2d_row4_col2, #T_1db2d_row4_col3, #T_1db2d_row4_col6, #T_1db2d_row5_col0, #T_1db2d_row5_col2, #T_1db2d_row5_col3, #T_1db2d_row5_col6, #T_1db2d_row6_col0, #T_1db2d_row6_col1, #T_1db2d_row6_col4, #T_1db2d_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_1db2d_row0_col3, #T_1db2d_row3_col0 {\n",
       "  background-color: #fcba03;\n",
       "}\n",
       "#T_1db2d_row1_col4, #T_1db2d_row1_col5, #T_1db2d_row4_col1, #T_1db2d_row5_col1 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_1db2d_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_1db2d_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_1db2d_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_1db2d_row0_col1\" class=\"data row0 col1\" >0.385547</td>\n",
       "      <td id=\"T_1db2d_row0_col2\" class=\"data row0 col2\" >0.001837</td>\n",
       "      <td id=\"T_1db2d_row0_col3\" class=\"data row0 col3\" >0.026248</td>\n",
       "      <td id=\"T_1db2d_row0_col4\" class=\"data row0 col4\" >0.007656</td>\n",
       "      <td id=\"T_1db2d_row0_col5\" class=\"data row0 col5\" >0.007656</td>\n",
       "      <td id=\"T_1db2d_row0_col6\" class=\"data row0 col6\" >0.007656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_1db2d_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_1db2d_row1_col0\" class=\"data row1 col0\" >0.385547</td>\n",
       "      <td id=\"T_1db2d_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_1db2d_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row1_col3\" class=\"data row1 col3\" >0.000006</td>\n",
       "      <td id=\"T_1db2d_row1_col4\" class=\"data row1 col4\" >0.075464</td>\n",
       "      <td id=\"T_1db2d_row1_col5\" class=\"data row1 col5\" >0.075464</td>\n",
       "      <td id=\"T_1db2d_row1_col6\" class=\"data row1 col6\" >0.000006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_1db2d_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_1db2d_row2_col0\" class=\"data row2 col0\" >0.001837</td>\n",
       "      <td id=\"T_1db2d_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_1db2d_row2_col3\" class=\"data row2 col3\" >0.385547</td>\n",
       "      <td id=\"T_1db2d_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row2_col6\" class=\"data row2 col6\" >0.184416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_1db2d_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_1db2d_row3_col0\" class=\"data row3 col0\" >0.026248</td>\n",
       "      <td id=\"T_1db2d_row3_col1\" class=\"data row3 col1\" >0.000006</td>\n",
       "      <td id=\"T_1db2d_row3_col2\" class=\"data row3 col2\" >0.385547</td>\n",
       "      <td id=\"T_1db2d_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_1db2d_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row3_col6\" class=\"data row3 col6\" >0.385547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_1db2d_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_1db2d_row4_col0\" class=\"data row4 col0\" >0.007656</td>\n",
       "      <td id=\"T_1db2d_row4_col1\" class=\"data row4 col1\" >0.075464</td>\n",
       "      <td id=\"T_1db2d_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_1db2d_row4_col5\" class=\"data row4 col5\" >0.999789</td>\n",
       "      <td id=\"T_1db2d_row4_col6\" class=\"data row4 col6\" >0.000006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_1db2d_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_1db2d_row5_col0\" class=\"data row5 col0\" >0.007656</td>\n",
       "      <td id=\"T_1db2d_row5_col1\" class=\"data row5 col1\" >0.075464</td>\n",
       "      <td id=\"T_1db2d_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row5_col4\" class=\"data row5 col4\" >0.999789</td>\n",
       "      <td id=\"T_1db2d_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_1db2d_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_1db2d_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_1db2d_row6_col0\" class=\"data row6 col0\" >0.007656</td>\n",
       "      <td id=\"T_1db2d_row6_col1\" class=\"data row6 col1\" >0.000006</td>\n",
       "      <td id=\"T_1db2d_row6_col2\" class=\"data row6 col2\" >0.184416</td>\n",
       "      <td id=\"T_1db2d_row6_col3\" class=\"data row6 col3\" >0.385547</td>\n",
       "      <td id=\"T_1db2d_row6_col4\" class=\"data row6 col4\" >0.000006</td>\n",
       "      <td id=\"T_1db2d_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_1db2d_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a6580070>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### Donors Choose_performance_test"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_982eb_row0_col0, #T_982eb_row0_col1, #T_982eb_row0_col4, #T_982eb_row0_col5, #T_982eb_row1_col0, #T_982eb_row1_col1, #T_982eb_row1_col4, #T_982eb_row1_col5, #T_982eb_row2_col2, #T_982eb_row2_col3, #T_982eb_row2_col6, #T_982eb_row3_col2, #T_982eb_row3_col3, #T_982eb_row3_col6, #T_982eb_row4_col0, #T_982eb_row4_col1, #T_982eb_row4_col4, #T_982eb_row4_col5, #T_982eb_row5_col0, #T_982eb_row5_col1, #T_982eb_row5_col4, #T_982eb_row5_col5, #T_982eb_row6_col2, #T_982eb_row6_col3, #T_982eb_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_982eb_row0_col2, #T_982eb_row0_col3, #T_982eb_row0_col6, #T_982eb_row1_col2, #T_982eb_row1_col3, #T_982eb_row1_col6, #T_982eb_row2_col0, #T_982eb_row2_col1, #T_982eb_row2_col4, #T_982eb_row2_col5, #T_982eb_row3_col0, #T_982eb_row3_col1, #T_982eb_row3_col4, #T_982eb_row3_col5, #T_982eb_row4_col2, #T_982eb_row4_col3, #T_982eb_row4_col6, #T_982eb_row5_col2, #T_982eb_row5_col3, #T_982eb_row5_col6, #T_982eb_row6_col0, #T_982eb_row6_col1, #T_982eb_row6_col4, #T_982eb_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_982eb_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_982eb_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_982eb_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_982eb_row0_col1\" class=\"data row0 col1\" >0.938331</td>\n",
       "      <td id=\"T_982eb_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row0_col4\" class=\"data row0 col4\" >0.184416</td>\n",
       "      <td id=\"T_982eb_row0_col5\" class=\"data row0 col5\" >0.678138</td>\n",
       "      <td id=\"T_982eb_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_982eb_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_982eb_row1_col0\" class=\"data row1 col0\" >0.938331</td>\n",
       "      <td id=\"T_982eb_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_982eb_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row1_col4\" class=\"data row1 col4\" >0.385547</td>\n",
       "      <td id=\"T_982eb_row1_col5\" class=\"data row1 col5\" >0.184416</td>\n",
       "      <td id=\"T_982eb_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_982eb_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_982eb_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_982eb_row2_col3\" class=\"data row2 col3\" >0.385547</td>\n",
       "      <td id=\"T_982eb_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row2_col6\" class=\"data row2 col6\" >0.999789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_982eb_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_982eb_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row3_col2\" class=\"data row3 col2\" >0.385547</td>\n",
       "      <td id=\"T_982eb_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_982eb_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row3_col6\" class=\"data row3 col6\" >0.184416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_982eb_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_982eb_row4_col0\" class=\"data row4 col0\" >0.184416</td>\n",
       "      <td id=\"T_982eb_row4_col1\" class=\"data row4 col1\" >0.385547</td>\n",
       "      <td id=\"T_982eb_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_982eb_row4_col5\" class=\"data row4 col5\" >0.678138</td>\n",
       "      <td id=\"T_982eb_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_982eb_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_982eb_row5_col0\" class=\"data row5 col0\" >0.678138</td>\n",
       "      <td id=\"T_982eb_row5_col1\" class=\"data row5 col1\" >0.184416</td>\n",
       "      <td id=\"T_982eb_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row5_col4\" class=\"data row5 col4\" >0.678138</td>\n",
       "      <td id=\"T_982eb_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_982eb_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_982eb_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_982eb_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row6_col2\" class=\"data row6 col2\" >0.999789</td>\n",
       "      <td id=\"T_982eb_row6_col3\" class=\"data row6 col3\" >0.184416</td>\n",
       "      <td id=\"T_982eb_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_982eb_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8804426bb0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### Adult_fairness_test"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_81f2e_row0_col0, #T_81f2e_row0_col1, #T_81f2e_row1_col0, #T_81f2e_row1_col1, #T_81f2e_row2_col2, #T_81f2e_row2_col6, #T_81f2e_row3_col3, #T_81f2e_row4_col4, #T_81f2e_row4_col5, #T_81f2e_row5_col4, #T_81f2e_row5_col5, #T_81f2e_row6_col2, #T_81f2e_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_81f2e_row0_col2, #T_81f2e_row0_col3, #T_81f2e_row0_col4, #T_81f2e_row0_col5, #T_81f2e_row0_col6, #T_81f2e_row1_col2, #T_81f2e_row1_col3, #T_81f2e_row1_col4, #T_81f2e_row1_col5, #T_81f2e_row1_col6, #T_81f2e_row2_col0, #T_81f2e_row2_col1, #T_81f2e_row2_col4, #T_81f2e_row2_col5, #T_81f2e_row3_col0, #T_81f2e_row3_col1, #T_81f2e_row3_col4, #T_81f2e_row3_col5, #T_81f2e_row4_col0, #T_81f2e_row4_col1, #T_81f2e_row4_col2, #T_81f2e_row4_col3, #T_81f2e_row4_col6, #T_81f2e_row5_col0, #T_81f2e_row5_col1, #T_81f2e_row5_col2, #T_81f2e_row5_col3, #T_81f2e_row5_col6, #T_81f2e_row6_col0, #T_81f2e_row6_col1, #T_81f2e_row6_col4, #T_81f2e_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_81f2e_row2_col3, #T_81f2e_row3_col2 {\n",
       "  background-color: #fcba03;\n",
       "}\n",
       "#T_81f2e_row3_col6, #T_81f2e_row6_col3 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_81f2e_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_81f2e_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_81f2e_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_81f2e_row0_col1\" class=\"data row0 col1\" >0.938331</td>\n",
       "      <td id=\"T_81f2e_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row0_col4\" class=\"data row0 col4\" >0.000052</td>\n",
       "      <td id=\"T_81f2e_row0_col5\" class=\"data row0 col5\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_81f2e_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_81f2e_row1_col0\" class=\"data row1 col0\" >0.938331</td>\n",
       "      <td id=\"T_81f2e_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_81f2e_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row1_col4\" class=\"data row1 col4\" >0.001837</td>\n",
       "      <td id=\"T_81f2e_row1_col5\" class=\"data row1 col5\" >0.000353</td>\n",
       "      <td id=\"T_81f2e_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_81f2e_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_81f2e_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_81f2e_row2_col3\" class=\"data row2 col3\" >0.026248</td>\n",
       "      <td id=\"T_81f2e_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row2_col6\" class=\"data row2 col6\" >0.938331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_81f2e_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_81f2e_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row3_col2\" class=\"data row3 col2\" >0.026248</td>\n",
       "      <td id=\"T_81f2e_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_81f2e_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row3_col6\" class=\"data row3 col6\" >0.075464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_81f2e_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_81f2e_row4_col0\" class=\"data row4 col0\" >0.000052</td>\n",
       "      <td id=\"T_81f2e_row4_col1\" class=\"data row4 col1\" >0.001837</td>\n",
       "      <td id=\"T_81f2e_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_81f2e_row4_col5\" class=\"data row4 col5\" >0.184416</td>\n",
       "      <td id=\"T_81f2e_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_81f2e_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_81f2e_row5_col0\" class=\"data row5 col0\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row5_col1\" class=\"data row5 col1\" >0.000353</td>\n",
       "      <td id=\"T_81f2e_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row5_col4\" class=\"data row5 col4\" >0.184416</td>\n",
       "      <td id=\"T_81f2e_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_81f2e_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_81f2e_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_81f2e_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row6_col2\" class=\"data row6 col2\" >0.938331</td>\n",
       "      <td id=\"T_81f2e_row6_col3\" class=\"data row6 col3\" >0.075464</td>\n",
       "      <td id=\"T_81f2e_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_81f2e_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a65950a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### AOF_fairness_test"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_008df_row0_col0, #T_008df_row0_col1, #T_008df_row0_col4, #T_008df_row0_col5, #T_008df_row1_col0, #T_008df_row1_col1, #T_008df_row1_col5, #T_008df_row2_col2, #T_008df_row3_col3, #T_008df_row3_col6, #T_008df_row4_col0, #T_008df_row4_col4, #T_008df_row5_col0, #T_008df_row5_col1, #T_008df_row5_col5, #T_008df_row6_col3, #T_008df_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_008df_row0_col2, #T_008df_row0_col3, #T_008df_row0_col6, #T_008df_row1_col2, #T_008df_row1_col3, #T_008df_row1_col6, #T_008df_row2_col0, #T_008df_row2_col1, #T_008df_row2_col4, #T_008df_row2_col5, #T_008df_row2_col6, #T_008df_row3_col0, #T_008df_row3_col1, #T_008df_row3_col4, #T_008df_row3_col5, #T_008df_row4_col2, #T_008df_row4_col3, #T_008df_row4_col6, #T_008df_row5_col2, #T_008df_row5_col3, #T_008df_row5_col6, #T_008df_row6_col0, #T_008df_row6_col1, #T_008df_row6_col2, #T_008df_row6_col4, #T_008df_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_008df_row1_col4, #T_008df_row4_col1, #T_008df_row4_col5, #T_008df_row5_col4 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "#T_008df_row2_col3, #T_008df_row3_col2 {\n",
       "  background-color: #fcba03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_008df_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_008df_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_008df_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_008df_row0_col1\" class=\"data row0 col1\" >0.184416</td>\n",
       "      <td id=\"T_008df_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_008df_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_008df_row0_col4\" class=\"data row0 col4\" >0.385547</td>\n",
       "      <td id=\"T_008df_row0_col5\" class=\"data row0 col5\" >0.385547</td>\n",
       "      <td id=\"T_008df_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_008df_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_008df_row1_col0\" class=\"data row1 col0\" >0.184416</td>\n",
       "      <td id=\"T_008df_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_008df_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_008df_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_008df_row1_col4\" class=\"data row1 col4\" >0.075464</td>\n",
       "      <td id=\"T_008df_row1_col5\" class=\"data row1 col5\" >0.938331</td>\n",
       "      <td id=\"T_008df_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_008df_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_008df_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_008df_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_008df_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_008df_row2_col3\" class=\"data row2 col3\" >0.026248</td>\n",
       "      <td id=\"T_008df_row2_col4\" class=\"data row2 col4\" >0.000353</td>\n",
       "      <td id=\"T_008df_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_008df_row2_col6\" class=\"data row2 col6\" >0.001837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_008df_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_008df_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_008df_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_008df_row3_col2\" class=\"data row3 col2\" >0.026248</td>\n",
       "      <td id=\"T_008df_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_008df_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_008df_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_008df_row3_col6\" class=\"data row3 col6\" >0.938331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_008df_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_008df_row4_col0\" class=\"data row4 col0\" >0.385547</td>\n",
       "      <td id=\"T_008df_row4_col1\" class=\"data row4 col1\" >0.075464</td>\n",
       "      <td id=\"T_008df_row4_col2\" class=\"data row4 col2\" >0.000353</td>\n",
       "      <td id=\"T_008df_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_008df_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_008df_row4_col5\" class=\"data row4 col5\" >0.075464</td>\n",
       "      <td id=\"T_008df_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_008df_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_008df_row5_col0\" class=\"data row5 col0\" >0.385547</td>\n",
       "      <td id=\"T_008df_row5_col1\" class=\"data row5 col1\" >0.938331</td>\n",
       "      <td id=\"T_008df_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_008df_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_008df_row5_col4\" class=\"data row5 col4\" >0.075464</td>\n",
       "      <td id=\"T_008df_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_008df_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_008df_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_008df_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_008df_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_008df_row6_col2\" class=\"data row6 col2\" >0.001837</td>\n",
       "      <td id=\"T_008df_row6_col3\" class=\"data row6 col3\" >0.938331</td>\n",
       "      <td id=\"T_008df_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_008df_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_008df_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8804426be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### COMPAS_fairness_test"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_4e33e_row0_col0, #T_4e33e_row0_col1, #T_4e33e_row1_col0, #T_4e33e_row1_col1, #T_4e33e_row2_col2, #T_4e33e_row2_col6, #T_4e33e_row3_col3, #T_4e33e_row3_col6, #T_4e33e_row4_col4, #T_4e33e_row4_col5, #T_4e33e_row5_col4, #T_4e33e_row5_col5, #T_4e33e_row6_col2, #T_4e33e_row6_col3, #T_4e33e_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_4e33e_row0_col2, #T_4e33e_row0_col3, #T_4e33e_row0_col4, #T_4e33e_row0_col5, #T_4e33e_row0_col6, #T_4e33e_row1_col2, #T_4e33e_row1_col3, #T_4e33e_row1_col4, #T_4e33e_row1_col5, #T_4e33e_row1_col6, #T_4e33e_row2_col0, #T_4e33e_row2_col1, #T_4e33e_row2_col4, #T_4e33e_row2_col5, #T_4e33e_row3_col0, #T_4e33e_row3_col1, #T_4e33e_row3_col4, #T_4e33e_row3_col5, #T_4e33e_row4_col0, #T_4e33e_row4_col1, #T_4e33e_row4_col2, #T_4e33e_row4_col3, #T_4e33e_row4_col6, #T_4e33e_row5_col0, #T_4e33e_row5_col1, #T_4e33e_row5_col2, #T_4e33e_row5_col3, #T_4e33e_row5_col6, #T_4e33e_row6_col0, #T_4e33e_row6_col1, #T_4e33e_row6_col4, #T_4e33e_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "#T_4e33e_row2_col3, #T_4e33e_row3_col2 {\n",
       "  background-color: #dffc03;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_4e33e_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_4e33e_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_4e33e_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_4e33e_row0_col1\" class=\"data row0 col1\" >0.184416</td>\n",
       "      <td id=\"T_4e33e_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_4e33e_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_4e33e_row0_col4\" class=\"data row0 col4\" >0.000006</td>\n",
       "      <td id=\"T_4e33e_row0_col5\" class=\"data row0 col5\" >0.000052</td>\n",
       "      <td id=\"T_4e33e_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4e33e_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_4e33e_row1_col0\" class=\"data row1 col0\" >0.184416</td>\n",
       "      <td id=\"T_4e33e_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_4e33e_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_4e33e_row1_col3\" class=\"data row1 col3\" >0.000052</td>\n",
       "      <td id=\"T_4e33e_row1_col4\" class=\"data row1 col4\" >0.001837</td>\n",
       "      <td id=\"T_4e33e_row1_col5\" class=\"data row1 col5\" >0.007656</td>\n",
       "      <td id=\"T_4e33e_row1_col6\" class=\"data row1 col6\" >0.000052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4e33e_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_4e33e_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_4e33e_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_4e33e_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_4e33e_row2_col3\" class=\"data row2 col3\" >0.075464</td>\n",
       "      <td id=\"T_4e33e_row2_col4\" class=\"data row2 col4\" >0.000006</td>\n",
       "      <td id=\"T_4e33e_row2_col5\" class=\"data row2 col5\" >0.000006</td>\n",
       "      <td id=\"T_4e33e_row2_col6\" class=\"data row2 col6\" >0.385547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4e33e_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_4e33e_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_4e33e_row3_col1\" class=\"data row3 col1\" >0.000052</td>\n",
       "      <td id=\"T_4e33e_row3_col2\" class=\"data row3 col2\" >0.075464</td>\n",
       "      <td id=\"T_4e33e_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_4e33e_row3_col4\" class=\"data row3 col4\" >0.007656</td>\n",
       "      <td id=\"T_4e33e_row3_col5\" class=\"data row3 col5\" >0.007656</td>\n",
       "      <td id=\"T_4e33e_row3_col6\" class=\"data row3 col6\" >0.678138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4e33e_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_4e33e_row4_col0\" class=\"data row4 col0\" >0.000006</td>\n",
       "      <td id=\"T_4e33e_row4_col1\" class=\"data row4 col1\" >0.001837</td>\n",
       "      <td id=\"T_4e33e_row4_col2\" class=\"data row4 col2\" >0.000006</td>\n",
       "      <td id=\"T_4e33e_row4_col3\" class=\"data row4 col3\" >0.007656</td>\n",
       "      <td id=\"T_4e33e_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_4e33e_row4_col5\" class=\"data row4 col5\" >0.938331</td>\n",
       "      <td id=\"T_4e33e_row4_col6\" class=\"data row4 col6\" >0.000052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4e33e_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_4e33e_row5_col0\" class=\"data row5 col0\" >0.000052</td>\n",
       "      <td id=\"T_4e33e_row5_col1\" class=\"data row5 col1\" >0.007656</td>\n",
       "      <td id=\"T_4e33e_row5_col2\" class=\"data row5 col2\" >0.000006</td>\n",
       "      <td id=\"T_4e33e_row5_col3\" class=\"data row5 col3\" >0.007656</td>\n",
       "      <td id=\"T_4e33e_row5_col4\" class=\"data row5 col4\" >0.938331</td>\n",
       "      <td id=\"T_4e33e_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_4e33e_row5_col6\" class=\"data row5 col6\" >0.000052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4e33e_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_4e33e_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_4e33e_row6_col1\" class=\"data row6 col1\" >0.000052</td>\n",
       "      <td id=\"T_4e33e_row6_col2\" class=\"data row6 col2\" >0.385547</td>\n",
       "      <td id=\"T_4e33e_row6_col3\" class=\"data row6 col3\" >0.678138</td>\n",
       "      <td id=\"T_4e33e_row6_col4\" class=\"data row6 col4\" >0.000052</td>\n",
       "      <td id=\"T_4e33e_row6_col5\" class=\"data row6 col5\" >0.000052</td>\n",
       "      <td id=\"T_4e33e_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a6588970>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### Donors Choose_fairness_test"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_cf3a8_row0_col0, #T_cf3a8_row0_col1, #T_cf3a8_row1_col0, #T_cf3a8_row1_col1, #T_cf3a8_row2_col2, #T_cf3a8_row2_col3, #T_cf3a8_row2_col6, #T_cf3a8_row3_col2, #T_cf3a8_row3_col3, #T_cf3a8_row3_col6, #T_cf3a8_row4_col4, #T_cf3a8_row4_col5, #T_cf3a8_row5_col4, #T_cf3a8_row5_col5, #T_cf3a8_row6_col2, #T_cf3a8_row6_col3, #T_cf3a8_row6_col6 {\n",
       "  background-color: white;\n",
       "}\n",
       "#T_cf3a8_row0_col2, #T_cf3a8_row0_col3, #T_cf3a8_row0_col4, #T_cf3a8_row0_col5, #T_cf3a8_row0_col6, #T_cf3a8_row1_col2, #T_cf3a8_row1_col3, #T_cf3a8_row1_col4, #T_cf3a8_row1_col5, #T_cf3a8_row1_col6, #T_cf3a8_row2_col0, #T_cf3a8_row2_col1, #T_cf3a8_row2_col4, #T_cf3a8_row2_col5, #T_cf3a8_row3_col0, #T_cf3a8_row3_col1, #T_cf3a8_row3_col4, #T_cf3a8_row3_col5, #T_cf3a8_row4_col0, #T_cf3a8_row4_col1, #T_cf3a8_row4_col2, #T_cf3a8_row4_col3, #T_cf3a8_row4_col6, #T_cf3a8_row5_col0, #T_cf3a8_row5_col1, #T_cf3a8_row5_col2, #T_cf3a8_row5_col3, #T_cf3a8_row5_col6, #T_cf3a8_row6_col0, #T_cf3a8_row6_col1, #T_cf3a8_row6_col4, #T_cf3a8_row6_col5 {\n",
       "  background-color: #fc6203;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_cf3a8_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >FB-auto</th>\n",
       "      <th class=\"col_heading level0 col1\" >FB</th>\n",
       "      <th class=\"col_heading level0 col2\" >RS</th>\n",
       "      <th class=\"col_heading level0 col3\" >HB</th>\n",
       "      <th class=\"col_heading level0 col4\" >FairRS</th>\n",
       "      <th class=\"col_heading level0 col5\" >FairTPE</th>\n",
       "      <th class=\"col_heading level0 col6\" >TPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_cf3a8_level0_row0\" class=\"row_heading level0 row0\" >FB-auto</th>\n",
       "      <td id=\"T_cf3a8_row0_col0\" class=\"data row0 col0\" >1.000000</td>\n",
       "      <td id=\"T_cf3a8_row0_col1\" class=\"data row0 col1\" >0.938331</td>\n",
       "      <td id=\"T_cf3a8_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row0_col3\" class=\"data row0 col3\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row0_col4\" class=\"data row0 col4\" >0.000353</td>\n",
       "      <td id=\"T_cf3a8_row0_col5\" class=\"data row0 col5\" >0.000353</td>\n",
       "      <td id=\"T_cf3a8_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_cf3a8_level0_row1\" class=\"row_heading level0 row1\" >FB</th>\n",
       "      <td id=\"T_cf3a8_row1_col0\" class=\"data row1 col0\" >0.938331</td>\n",
       "      <td id=\"T_cf3a8_row1_col1\" class=\"data row1 col1\" >1.000000</td>\n",
       "      <td id=\"T_cf3a8_row1_col2\" class=\"data row1 col2\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row1_col3\" class=\"data row1 col3\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row1_col4\" class=\"data row1 col4\" >0.001837</td>\n",
       "      <td id=\"T_cf3a8_row1_col5\" class=\"data row1 col5\" >0.000353</td>\n",
       "      <td id=\"T_cf3a8_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_cf3a8_level0_row2\" class=\"row_heading level0 row2\" >RS</th>\n",
       "      <td id=\"T_cf3a8_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row2_col2\" class=\"data row2 col2\" >1.000000</td>\n",
       "      <td id=\"T_cf3a8_row2_col3\" class=\"data row2 col3\" >0.385547</td>\n",
       "      <td id=\"T_cf3a8_row2_col4\" class=\"data row2 col4\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row2_col5\" class=\"data row2 col5\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row2_col6\" class=\"data row2 col6\" >0.678138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_cf3a8_level0_row3\" class=\"row_heading level0 row3\" >HB</th>\n",
       "      <td id=\"T_cf3a8_row3_col0\" class=\"data row3 col0\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row3_col2\" class=\"data row3 col2\" >0.385547</td>\n",
       "      <td id=\"T_cf3a8_row3_col3\" class=\"data row3 col3\" >1.000000</td>\n",
       "      <td id=\"T_cf3a8_row3_col4\" class=\"data row3 col4\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row3_col5\" class=\"data row3 col5\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row3_col6\" class=\"data row3 col6\" >0.938331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_cf3a8_level0_row4\" class=\"row_heading level0 row4\" >FairRS</th>\n",
       "      <td id=\"T_cf3a8_row4_col0\" class=\"data row4 col0\" >0.000353</td>\n",
       "      <td id=\"T_cf3a8_row4_col1\" class=\"data row4 col1\" >0.001837</td>\n",
       "      <td id=\"T_cf3a8_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row4_col4\" class=\"data row4 col4\" >1.000000</td>\n",
       "      <td id=\"T_cf3a8_row4_col5\" class=\"data row4 col5\" >0.385547</td>\n",
       "      <td id=\"T_cf3a8_row4_col6\" class=\"data row4 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_cf3a8_level0_row5\" class=\"row_heading level0 row5\" >FairTPE</th>\n",
       "      <td id=\"T_cf3a8_row5_col0\" class=\"data row5 col0\" >0.000353</td>\n",
       "      <td id=\"T_cf3a8_row5_col1\" class=\"data row5 col1\" >0.000353</td>\n",
       "      <td id=\"T_cf3a8_row5_col2\" class=\"data row5 col2\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row5_col4\" class=\"data row5 col4\" >0.385547</td>\n",
       "      <td id=\"T_cf3a8_row5_col5\" class=\"data row5 col5\" >1.000000</td>\n",
       "      <td id=\"T_cf3a8_row5_col6\" class=\"data row5 col6\" >0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_cf3a8_level0_row6\" class=\"row_heading level0 row6\" >TPE</th>\n",
       "      <td id=\"T_cf3a8_row6_col0\" class=\"data row6 col0\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row6_col1\" class=\"data row6 col1\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row6_col2\" class=\"data row6 col2\" >0.678138</td>\n",
       "      <td id=\"T_cf3a8_row6_col3\" class=\"data row6 col3\" >0.938331</td>\n",
       "      <td id=\"T_cf3a8_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_cf3a8_row6_col6\" class=\"data row6 col6\" >1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f87a71838e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import Markdown, display\n",
    "\n",
    "for key, df in stats_tests_dfs.items():\n",
    "    styled_df = df.style.applymap(\n",
    "        lambda x: \"background-color: #fc6203\" if x<1e-2\n",
    "            else \"background-color: #fcba03\" if x<5e-2\n",
    "            else \"background-color: #dffc03\" if x<1e-1\n",
    "            else \"background-color: white\"\n",
    "    )\n",
    "\n",
    "    display(Markdown(f\"### {key}\"))\n",
    "    display(styled_df)\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}