slides_ensemble_study.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "small_sklearn_kernel",
      "language": "python",
      "name": "small_sklearn_kernel"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.10"
    },
    "colab": {
      "name": "slides_ensemble_study.ipynb",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/stepthom/sandbox/blob/master/slides_ensemble_study.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "71iHiiyxntxt",
        "colab_type": "text"
      },
      "source": [
        "# Playground for Ensemble Slides - Study on Many Datasets\n",
        "\n",
        "- Stephen W. Thomas\n",
        "- Used for MMA 869, MMAI 869, and GMMA 869\n",
        "\n",
        "This code is going to evaluate lots of ensemble classifiers (as well as some regular ones) on a lot of datasets. BOOM.\n",
        "\n",
        "Also, we'll let autosklearn crank for 5 minutes per dataset to see how it fairs as well."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8G1gDrGYntxu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "55985d7e-0a02-44f7-dc5c-6eb13314e3dd"
      },
      "source": [
        "import datetime\n",
        "print(datetime.datetime.now())"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-07-13 20:27:23.956403\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eto1SoZZntxz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "4fe16caa-f726-48fd-ecd5-a866e0417fa4"
      },
      "source": [
        "import pandas as pd\n",
        "pd.show_versions(as_json=False)\n",
        "\n",
        "import sklearn\n",
        "sklearn.__version__"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/psycopg2/__init__.py:144: UserWarning: The psycopg2 wheel package will be renamed from release 2.8; in order to keep installing from binary please use \"pip install psycopg2-binary\" instead. For details see: <http://initd.org/psycopg/docs/install.html#binary-install-from-pypi>.\n",
            "  \"\"\")\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "INSTALLED VERSIONS\n",
            "------------------\n",
            "commit           : None\n",
            "python           : 3.6.9.final.0\n",
            "python-bits      : 64\n",
            "OS               : Linux\n",
            "OS-release       : 4.19.104+\n",
            "machine          : x86_64\n",
            "processor        : x86_64\n",
            "byteorder        : little\n",
            "LC_ALL           : None\n",
            "LANG             : en_US.UTF-8\n",
            "LOCALE           : en_US.UTF-8\n",
            "\n",
            "pandas           : 0.25.3\n",
            "numpy            : 1.18.5\n",
            "pytz             : 2018.9\n",
            "dateutil         : 2.8.1\n",
            "pip              : 19.3.1\n",
            "setuptools       : 49.1.0\n",
            "Cython           : 0.29.20\n",
            "pytest           : 5.4.3\n",
            "hypothesis       : None\n",
            "sphinx           : 1.8.5\n",
            "blosc            : None\n",
            "feather          : 0.4.1\n",
            "xlsxwriter       : None\n",
            "lxml.etree       : 4.2.6\n",
            "html5lib         : 1.0.1\n",
            "pymysql          : None\n",
            "psycopg2         : 2.7.6.1 (dt dec pq3 ext lo64)\n",
            "jinja2           : 2.11.2\n",
            "IPython          : 5.5.0\n",
            "pandas_datareader: 0.8.1\n",
            "bs4              : 4.6.3\n",
            "bottleneck       : 1.3.2\n",
            "fastparquet      : None\n",
            "gcsfs            : None\n",
            "lxml.etree       : 4.2.6\n",
            "matplotlib       : 3.2.2\n",
            "numexpr          : 2.7.1\n",
            "odfpy            : None\n",
            "openpyxl         : 2.5.9\n",
            "pandas_gbq       : 0.11.0\n",
            "pyarrow          : 0.14.1\n",
            "pytables         : None\n",
            "s3fs             : 0.4.2\n",
            "scipy            : 1.4.1\n",
            "sqlalchemy       : 1.3.18\n",
            "tables           : 3.4.4\n",
            "xarray           : 0.15.1\n",
            "xlrd             : 1.1.0\n",
            "xlwt             : 1.3.0\n",
            "xlsxwriter       : None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic": {
              "type": "string"
            },
            "text/plain": [
              "'0.22.2.post1'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hMj4KFFEPKGk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 751
        },
        "outputId": "90cf18e4-9444-4a5c-d4af-ec5c325a9883"
      },
      "source": [
        "!apt-get install swig -y\n",
        "!pip install Cython numpy\n",
        "\n",
        "# sometimes you have to run the next command twice on colab\n",
        "# I haven't figured out why\n",
        "!pip install auto-sklearn"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "swig is already the newest version (3.0.12-1).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-440\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 35 not upgraded.\n",
            "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (0.29.20)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (1.18.5)\n",
            "Requirement already satisfied: auto-sklearn in /usr/local/lib/python3.6/dist-packages (0.8.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.16.0)\n",
            "Requirement already satisfied: pandas<1.0 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.25.3)\n",
            "Requirement already satisfied: scipy>=0.14.1 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (1.4.1)\n",
            "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.29.20)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (3.13)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (49.1.0)\n",
            "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (1.18.5)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (5.4.8)\n",
            "Requirement already satisfied: liac-arff in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (2.4.0)\n",
            "Requirement already satisfied: pyrfr<0.9,>=0.7 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.8.0)\n",
            "Requirement already satisfied: scikit-learn<0.23,>=0.22.0 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.22.2.post1)\n",
            "Requirement already satisfied: pytest>=4.6 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (5.4.3)\n",
            "Requirement already satisfied: smac>=0.12 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.12.2)\n",
            "Requirement already satisfied: pynisher>=0.4.2 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.5.0)\n",
            "Requirement already satisfied: ConfigSpace<0.5,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.4.13)\n",
            "Requirement already satisfied: lockfile in /usr/local/lib/python3.6/dist-packages (from auto-sklearn) (0.12.2)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas<1.0->auto-sklearn) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas<1.0->auto-sklearn) (2018.9)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from pytest>=4.6->auto-sklearn) (0.2.5)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=4.6->auto-sklearn) (8.4.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from pytest>=4.6->auto-sklearn) (20.4)\n",
            "Requirement already satisfied: importlib-metadata>=0.12; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from pytest>=4.6->auto-sklearn) (1.7.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=4.6->auto-sklearn) (1.9.0)\n",
            "Requirement already satisfied: pluggy<1.0,>=0.12 in /usr/local/lib/python3.6/dist-packages (from pytest>=4.6->auto-sklearn) (0.13.1)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=4.6->auto-sklearn) (19.3.0)\n",
            "Requirement already satisfied: sobol-seq in /usr/local/lib/python3.6/dist-packages (from smac>=0.12->auto-sklearn) (0.2.0)\n",
            "Requirement already satisfied: lazy-import in /usr/local/lib/python3.6/dist-packages (from smac>=0.12->auto-sklearn) (0.2.2)\n",
            "Requirement already satisfied: docutils>=0.3 in /usr/local/lib/python3.6/dist-packages (from pynisher>=0.4.2->auto-sklearn) (0.15.2)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.6/dist-packages (from ConfigSpace<0.5,>=0.4.0->auto-sklearn) (2.4.7)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.6.1->pandas<1.0->auto-sklearn) (1.12.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.12; python_version < \"3.8\"->pytest>=4.6->auto-sklearn) (3.1.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OdoDWfSGntx3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from mpl_toolkits.mplot3d import Axes3D\n",
        "import seaborn as sns \n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import itertools\n",
        "\n",
        "import scipy\n",
        "\n",
        "from IPython.core.interactiveshell import InteractiveShell\n",
        "InteractiveShell.ast_node_interactivity = \"all\""
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wXY0CHmAxdsm",
        "colab_type": "text"
      },
      "source": [
        "# Uncle Steve's Amazing Do-All function\n",
        "\n",
        "It slices! It dices!\n",
        "\n",
        "In order to streamline the evaluation of each dataset, let's create an function that takes in a dataset, the name of the target column, and the name of any columsn to drop (because that's decided by the human), and then automate the rest:\n",
        "\n",
        "- Converting datatypes of the target column if necessary\n",
        "- OHE any categorical features\n",
        "- Splitting data into training and testing\n",
        "- Training and evaluating all the models/ensembles\n",
        "- Returning a list of the performance of all the models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aghz7iPYntx7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "from sklearn.ensemble import RandomForestClassifier, StackingClassifier, VotingClassifier, BaggingClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier\n",
        "\n",
        "from sklearn.experimental import enable_hist_gradient_boosting  # noqa\n",
        "\n",
        "from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, recall_score, precision_score, roc_auc_score\n",
        "\n",
        "import autosklearn.classification\n",
        "\n",
        "import time\n",
        "\n",
        "# Helper function\n",
        "def do_all_for_dataset(dataset_name, df, target_col, drop_cols=[]):\n",
        "\n",
        "    # If target_col is an object, convert to numbers\n",
        "    if df[target_col].dtype == 'object':\n",
        "      df[target_col] =  df[target_col].astype('category').cat.codes\n",
        "\n",
        "    # OHE all categorical columns\n",
        "    cat_cols = list(df.select_dtypes(include=['object']).columns) \n",
        "    if target_col in cat_cols: cat_cols.remove(targe_col)\n",
        "    if len(cat_cols) > 0:\n",
        "      df = pd.concat([df,pd.get_dummies(df[cat_cols])],axis=1)\n",
        "\n",
        "    # Split into X and y\n",
        "    X = df.drop(drop_cols + cat_cols + [target_col], axis=1)\n",
        "    y = df[target_col]\n",
        "\n",
        "    # Split into training and testing\n",
        "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)\n",
        "\n",
        "    print('Y (train) counts:')\n",
        "    print(y_train.value_counts())\n",
        "    print('Y (test) counts:')\n",
        "    print(y_test.value_counts())\n",
        "    \n",
        "    nb = GaussianNB()   \n",
        "    lr = LogisticRegression(random_state=42, solver='lbfgs', max_iter=5000)\n",
        "    dt = DecisionTreeClassifier(random_state=42)\n",
        "    knn = KNeighborsClassifier(n_neighbors=7)\n",
        "\n",
        "    rf = RandomForestClassifier(random_state=42, n_estimators=200)\n",
        "    ada = AdaBoostClassifier(random_state=42, n_estimators=200)\n",
        "\n",
        "    scorer = autosklearn.metrics.make_scorer(\n",
        "        'f1_score',\n",
        "        sklearn.metrics.f1_score\n",
        "    )    \n",
        "    automl = autosklearn.classification.AutoSklearnClassifier(\n",
        "          time_left_for_this_task=100, # run auto-sklearn for at most X secs\n",
        "          per_run_time_limit=15, # spend at most 60 sec for each model training\n",
        "          metric=scorer\n",
        "          )\n",
        "\n",
        "\n",
        "    est_list = [('DT', dt), ('LR', lr), ('NB', nb), ('RF', rf), ('ADA', ada)]\n",
        "       \n",
        "    dict_classifiers = {\n",
        "        \"LR\": lr, \n",
        "        \"NB\": nb,\n",
        "        \"DT\": dt,\n",
        "        \"KNN\": knn,\n",
        "        \"Voting\": VotingClassifier(estimators = est_list, voting='soft'),\n",
        "        \"Bagging\": BaggingClassifier(DecisionTreeClassifier(), n_estimators=200, random_state=42),\n",
        "        \"RF\": rf,\n",
        "        \"ExtraTrees\": ExtraTreesClassifier(random_state=42, n_estimators=200),\n",
        "        \"Adaboost\": ada,\n",
        "        \"GBC\": GradientBoostingClassifier(random_state=42, n_estimators=200),\n",
        "        \"Stacking\": StackingClassifier(estimators=est_list, final_estimator=LogisticRegression()),\n",
        "        \"automl\": automl,\n",
        "    }\n",
        "    \n",
        "    model_results = list()\n",
        "    \n",
        "    for model_name, model in dict_classifiers.items():\n",
        "        start = time.time()\n",
        "        y_pred = model.fit(X_train, y_train).predict(X_test)\n",
        "        end = time.time()\n",
        "        total = end - start\n",
        "        \n",
        "        accuracy       = accuracy_score(y_test, y_pred)\n",
        "        f1             = f1_score(y_test, y_pred)\n",
        "        recall         = recall_score(y_test, y_pred)\n",
        "        precision      = precision_score(y_test, y_pred)\n",
        "        roc_auc        = roc_auc_score(y_test, y_pred)\n",
        "    \n",
        "        df = pd.DataFrame({\"Dataset\"   : [dataset_name],\n",
        "                           \"Method\"    : [model_name],\n",
        "                           \"Time\"      : [total],\n",
        "                           \"Accuracy\"  : [accuracy],\n",
        "                           \"Recall\"    : [recall],\n",
        "                           \"Precision\" : [precision],\n",
        "                           \"F1\"        : [f1],\n",
        "                           \"AUC\"       : [roc_auc],\n",
        "                          })\n",
        "        model_results.append(df)\n",
        "   \n",
        "\n",
        "    dataset_results = pd.concat([m for m in model_results], axis = 0).reset_index()\n",
        "\n",
        "    dataset_results = dataset_results.drop(columns = \"index\",axis =1)\n",
        "    dataset_results = dataset_results.sort_values(by=['F1'], ascending=False)\n",
        "    dataset_results['Rank'] = range(1, len(dataset_results)+1)\n",
        "    \n",
        "    return dataset_results"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KuVrDWMv4HMc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We're going to save the results of each dataset into a big list, to analyze later.\n",
        "results = list()"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1ATmTd-d0Li0",
        "colab_type": "text"
      },
      "source": [
        "# Diabetes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dDguHFy7ntyB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "b88762de-c14e-45dc-be1d-24590dae3d6d"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/diabetes_orig.csv')\n",
        "r = do_all_for_dataset( 'Diabetes', df, target_col='diabetes', drop_cols=['Id'])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "0    400\n",
            "1    214\n",
            "Name: diabetes, dtype: int64\n",
            "Y (test) counts:\n",
            "0    100\n",
            "1     54\n",
            "Name: diabetes, dtype: int64\n",
            "[WARNING] [2020-07-13 20:27:43,104:AutoMLSMBO(1)::5ad84c85ebefd51fd3f129fa76a785d6] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>0.409679</td>\n",
              "      <td>0.772727</td>\n",
              "      <td>0.648148</td>\n",
              "      <td>0.686275</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>0.744074</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>GBC</td>\n",
              "      <td>0.321875</td>\n",
              "      <td>0.753247</td>\n",
              "      <td>0.629630</td>\n",
              "      <td>0.653846</td>\n",
              "      <td>0.641509</td>\n",
              "      <td>0.724815</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>Voting</td>\n",
              "      <td>1.009843</td>\n",
              "      <td>0.746753</td>\n",
              "      <td>0.592593</td>\n",
              "      <td>0.653061</td>\n",
              "      <td>0.621359</td>\n",
              "      <td>0.711296</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>RF</td>\n",
              "      <td>0.407973</td>\n",
              "      <td>0.746753</td>\n",
              "      <td>0.592593</td>\n",
              "      <td>0.653061</td>\n",
              "      <td>0.621359</td>\n",
              "      <td>0.711296</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>0.682466</td>\n",
              "      <td>0.746753</td>\n",
              "      <td>0.574074</td>\n",
              "      <td>0.659574</td>\n",
              "      <td>0.613861</td>\n",
              "      <td>0.707037</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.002595</td>\n",
              "      <td>0.707792</td>\n",
              "      <td>0.648148</td>\n",
              "      <td>0.573770</td>\n",
              "      <td>0.608696</td>\n",
              "      <td>0.694074</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>automl</td>\n",
              "      <td>94.601545</td>\n",
              "      <td>0.740260</td>\n",
              "      <td>0.555556</td>\n",
              "      <td>0.652174</td>\n",
              "      <td>0.600000</td>\n",
              "      <td>0.697778</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>4.753475</td>\n",
              "      <td>0.733766</td>\n",
              "      <td>0.555556</td>\n",
              "      <td>0.638298</td>\n",
              "      <td>0.594059</td>\n",
              "      <td>0.692778</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.307972</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.555556</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.588235</td>\n",
              "      <td>0.687778</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.005634</td>\n",
              "      <td>0.727273</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.642857</td>\n",
              "      <td>0.562500</td>\n",
              "      <td>0.675000</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>LR</td>\n",
              "      <td>0.040350</td>\n",
              "      <td>0.714286</td>\n",
              "      <td>0.518519</td>\n",
              "      <td>0.608696</td>\n",
              "      <td>0.560000</td>\n",
              "      <td>0.669259</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Diabetes</td>\n",
              "      <td>KNN</td>\n",
              "      <td>0.010482</td>\n",
              "      <td>0.675325</td>\n",
              "      <td>0.481481</td>\n",
              "      <td>0.541667</td>\n",
              "      <td>0.509804</td>\n",
              "      <td>0.630741</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "8   Diabetes    Adaboost   0.409679  ...  0.666667  0.744074     1\n",
              "9   Diabetes         GBC   0.321875  ...  0.641509  0.724815     2\n",
              "4   Diabetes      Voting   1.009843  ...  0.621359  0.711296     3\n",
              "6   Diabetes          RF   0.407973  ...  0.621359  0.711296     4\n",
              "5   Diabetes     Bagging   0.682466  ...  0.613861  0.707037     5\n",
              "1   Diabetes          NB   0.002595  ...  0.608696  0.694074     6\n",
              "11  Diabetes      automl  94.601545  ...  0.600000  0.697778     7\n",
              "10  Diabetes    Stacking   4.753475  ...  0.594059  0.692778     8\n",
              "7   Diabetes  ExtraTrees   0.307972  ...  0.588235  0.687778     9\n",
              "2   Diabetes          DT   0.005634  ...  0.562500  0.675000    10\n",
              "0   Diabetes          LR   0.040350  ...  0.560000  0.669259    11\n",
              "3   Diabetes         KNN   0.010482  ...  0.509804  0.630741    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "llyxc8cI0Oyz",
        "colab_type": "text"
      },
      "source": [
        "# German Credit"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r-Jss6WdntyE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "8e2035f8-c8ed-4ffe-a849-264eefcd637a"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/GermanCredit.csv')\n",
        "r = do_all_for_dataset('GermanCredit', df, target_col='Class', drop_cols=[])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "1    560\n",
            "0    240\n",
            "Name: Class, dtype: int64\n",
            "Y (test) counts:\n",
            "1    140\n",
            "0     60\n",
            "Name: Class, dtype: int64\n",
            "[WARNING] [2020-07-13 20:29:29,180:AutoMLSMBO(1)::8767103aa82c12d7e1a0e04d3ea95c6e] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>GBC</td>\n",
              "      <td>0.452885</td>\n",
              "      <td>0.760</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>0.810811</td>\n",
              "      <td>0.833333</td>\n",
              "      <td>0.695238</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>automl</td>\n",
              "      <td>95.494524</td>\n",
              "      <td>0.745</td>\n",
              "      <td>0.878571</td>\n",
              "      <td>0.783439</td>\n",
              "      <td>0.828283</td>\n",
              "      <td>0.655952</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.381405</td>\n",
              "      <td>0.745</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>0.794702</td>\n",
              "      <td>0.824742</td>\n",
              "      <td>0.670238</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>RF</td>\n",
              "      <td>0.424692</td>\n",
              "      <td>0.735</td>\n",
              "      <td>0.885714</td>\n",
              "      <td>0.770186</td>\n",
              "      <td>0.823920</td>\n",
              "      <td>0.634524</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>6.789234</td>\n",
              "      <td>0.725</td>\n",
              "      <td>0.828571</td>\n",
              "      <td>0.789116</td>\n",
              "      <td>0.808362</td>\n",
              "      <td>0.655952</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>1.137309</td>\n",
              "      <td>0.720</td>\n",
              "      <td>0.828571</td>\n",
              "      <td>0.783784</td>\n",
              "      <td>0.805556</td>\n",
              "      <td>0.647619</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>KNN</td>\n",
              "      <td>0.028221</td>\n",
              "      <td>0.690</td>\n",
              "      <td>0.850000</td>\n",
              "      <td>0.743750</td>\n",
              "      <td>0.793333</td>\n",
              "      <td>0.583333</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>Voting</td>\n",
              "      <td>1.199796</td>\n",
              "      <td>0.710</td>\n",
              "      <td>0.785714</td>\n",
              "      <td>0.797101</td>\n",
              "      <td>0.791367</td>\n",
              "      <td>0.659524</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>LR</td>\n",
              "      <td>0.137589</td>\n",
              "      <td>0.700</td>\n",
              "      <td>0.792857</td>\n",
              "      <td>0.781690</td>\n",
              "      <td>0.787234</td>\n",
              "      <td>0.638095</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>0.493485</td>\n",
              "      <td>0.695</td>\n",
              "      <td>0.778571</td>\n",
              "      <td>0.784173</td>\n",
              "      <td>0.781362</td>\n",
              "      <td>0.639286</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.012444</td>\n",
              "      <td>0.675</td>\n",
              "      <td>0.742857</td>\n",
              "      <td>0.781955</td>\n",
              "      <td>0.761905</td>\n",
              "      <td>0.629762</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>GermanCredit</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.005122</td>\n",
              "      <td>0.685</td>\n",
              "      <td>0.714286</td>\n",
              "      <td>0.813008</td>\n",
              "      <td>0.760456</td>\n",
              "      <td>0.665476</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "9   GermanCredit         GBC   0.452885  ...  0.833333  0.695238     1\n",
              "11  GermanCredit      automl  95.494524  ...  0.828283  0.655952     2\n",
              "7   GermanCredit  ExtraTrees   0.381405  ...  0.824742  0.670238     3\n",
              "6   GermanCredit          RF   0.424692  ...  0.823920  0.634524     4\n",
              "10  GermanCredit    Stacking   6.789234  ...  0.808362  0.655952     5\n",
              "5   GermanCredit     Bagging   1.137309  ...  0.805556  0.647619     6\n",
              "3   GermanCredit         KNN   0.028221  ...  0.793333  0.583333     7\n",
              "4   GermanCredit      Voting   1.199796  ...  0.791367  0.659524     8\n",
              "0   GermanCredit          LR   0.137589  ...  0.787234  0.638095     9\n",
              "8   GermanCredit    Adaboost   0.493485  ...  0.781362  0.639286    10\n",
              "2   GermanCredit          DT   0.012444  ...  0.761905  0.629762    11\n",
              "1   GermanCredit          NB   0.005122  ...  0.760456  0.665476    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u1XBXNEj0QfT",
        "colab_type": "text"
      },
      "source": [
        "# Los Angeles Heart Disease"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fNi7qgcMntyI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "363a5001-3e64-4f2b-a6f0-c190d313c7c9"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/laheart.csv')\n",
        "r = do_all_for_dataset('LA Heart', df, target_col='DEATH', drop_cols=['ID', 'DEATH_YR'])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "0    109\n",
            "1     51\n",
            "Name: DEATH, dtype: int64\n",
            "Y (test) counts:\n",
            "0    27\n",
            "1    13\n",
            "Name: DEATH, dtype: int64\n",
            "[WARNING] [2020-07-13 20:31:12,292:AutoMLSMBO(1)::2aca45a7767db578456dde2dd6ff9132] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>0.321829</td>\n",
              "      <td>0.800</td>\n",
              "      <td>0.692308</td>\n",
              "      <td>0.692308</td>\n",
              "      <td>0.692308</td>\n",
              "      <td>0.772080</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>automl</td>\n",
              "      <td>95.261597</td>\n",
              "      <td>0.750</td>\n",
              "      <td>0.384615</td>\n",
              "      <td>0.714286</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.655271</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.003416</td>\n",
              "      <td>0.725</td>\n",
              "      <td>0.384615</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.476190</td>\n",
              "      <td>0.636752</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>0.384849</td>\n",
              "      <td>0.700</td>\n",
              "      <td>0.384615</td>\n",
              "      <td>0.555556</td>\n",
              "      <td>0.454545</td>\n",
              "      <td>0.618234</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>GBC</td>\n",
              "      <td>0.177967</td>\n",
              "      <td>0.675</td>\n",
              "      <td>0.384615</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.434783</td>\n",
              "      <td>0.599715</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>RF</td>\n",
              "      <td>0.316920</td>\n",
              "      <td>0.725</td>\n",
              "      <td>0.307692</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>0.421053</td>\n",
              "      <td>0.616809</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>Voting</td>\n",
              "      <td>0.904772</td>\n",
              "      <td>0.700</td>\n",
              "      <td>0.307692</td>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.400000</td>\n",
              "      <td>0.598291</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.004762</td>\n",
              "      <td>0.600</td>\n",
              "      <td>0.384615</td>\n",
              "      <td>0.384615</td>\n",
              "      <td>0.384615</td>\n",
              "      <td>0.544160</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.216772</td>\n",
              "      <td>0.725</td>\n",
              "      <td>0.230769</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.352941</td>\n",
              "      <td>0.596866</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>LR</td>\n",
              "      <td>0.253523</td>\n",
              "      <td>0.675</td>\n",
              "      <td>0.230769</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.315789</td>\n",
              "      <td>0.559829</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>4.681041</td>\n",
              "      <td>0.700</td>\n",
              "      <td>0.153846</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>0.250000</td>\n",
              "      <td>0.558405</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>LA Heart</td>\n",
              "      <td>KNN</td>\n",
              "      <td>0.004526</td>\n",
              "      <td>0.650</td>\n",
              "      <td>0.076923</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.125000</td>\n",
              "      <td>0.501425</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "8   LA Heart    Adaboost   0.321829  ...  0.692308  0.772080     1\n",
              "11  LA Heart      automl  95.261597  ...  0.500000  0.655271     2\n",
              "1   LA Heart          NB   0.003416  ...  0.476190  0.636752     3\n",
              "5   LA Heart     Bagging   0.384849  ...  0.454545  0.618234     4\n",
              "9   LA Heart         GBC   0.177967  ...  0.434783  0.599715     5\n",
              "6   LA Heart          RF   0.316920  ...  0.421053  0.616809     6\n",
              "4   LA Heart      Voting   0.904772  ...  0.400000  0.598291     7\n",
              "2   LA Heart          DT   0.004762  ...  0.384615  0.544160     8\n",
              "7   LA Heart  ExtraTrees   0.216772  ...  0.352941  0.596866     9\n",
              "0   LA Heart          LR   0.253523  ...  0.315789  0.559829    10\n",
              "10  LA Heart    Stacking   4.681041  ...  0.250000  0.558405    11\n",
              "3   LA Heart         KNN   0.004526  ...  0.125000  0.501425    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4w6OV5mB0Ssb",
        "colab_type": "text"
      },
      "source": [
        "# HR Churn"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oBQzLPm2ntyL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "f65db4b1-4b7c-450a-d874-a83bd9eb1a3e"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/HR_comma_sep.csv')\n",
        "r = do_all_for_dataset('HR Churn', df, target_col='left', drop_cols=[])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "0    9142\n",
            "1    2857\n",
            "Name: left, dtype: int64\n",
            "Y (test) counts:\n",
            "0    2286\n",
            "1     714\n",
            "Name: left, dtype: int64\n",
            "[WARNING] [2020-07-13 20:33:35,819:AutoMLSMBO(1)::5f62562ade5dbc7e3eebfd0909a1b28e] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>26.474826</td>\n",
              "      <td>0.990333</td>\n",
              "      <td>0.966387</td>\n",
              "      <td>0.992806</td>\n",
              "      <td>0.979418</td>\n",
              "      <td>0.982100</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>RF</td>\n",
              "      <td>1.948336</td>\n",
              "      <td>0.990000</td>\n",
              "      <td>0.963585</td>\n",
              "      <td>0.994220</td>\n",
              "      <td>0.978663</td>\n",
              "      <td>0.980918</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>1.726488</td>\n",
              "      <td>0.988000</td>\n",
              "      <td>0.963585</td>\n",
              "      <td>0.985673</td>\n",
              "      <td>0.974504</td>\n",
              "      <td>0.979605</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>6.435993</td>\n",
              "      <td>0.987667</td>\n",
              "      <td>0.967787</td>\n",
              "      <td>0.980142</td>\n",
              "      <td>0.973925</td>\n",
              "      <td>0.980831</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>automl</td>\n",
              "      <td>94.908843</td>\n",
              "      <td>0.986000</td>\n",
              "      <td>0.950980</td>\n",
              "      <td>0.989796</td>\n",
              "      <td>0.970000</td>\n",
              "      <td>0.973959</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>Voting</td>\n",
              "      <td>5.131726</td>\n",
              "      <td>0.978333</td>\n",
              "      <td>0.948179</td>\n",
              "      <td>0.960284</td>\n",
              "      <td>0.954193</td>\n",
              "      <td>0.967965</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>GBC</td>\n",
              "      <td>2.744781</td>\n",
              "      <td>0.976667</td>\n",
              "      <td>0.938375</td>\n",
              "      <td>0.962644</td>\n",
              "      <td>0.950355</td>\n",
              "      <td>0.963501</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.066668</td>\n",
              "      <td>0.973333</td>\n",
              "      <td>0.964986</td>\n",
              "      <td>0.926075</td>\n",
              "      <td>0.945130</td>\n",
              "      <td>0.970463</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>2.103505</td>\n",
              "      <td>0.960333</td>\n",
              "      <td>0.917367</td>\n",
              "      <td>0.916084</td>\n",
              "      <td>0.916725</td>\n",
              "      <td>0.945560</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>KNN</td>\n",
              "      <td>0.170436</td>\n",
              "      <td>0.946333</td>\n",
              "      <td>0.932773</td>\n",
              "      <td>0.854942</td>\n",
              "      <td>0.892163</td>\n",
              "      <td>0.941671</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.011730</td>\n",
              "      <td>0.680667</td>\n",
              "      <td>0.829132</td>\n",
              "      <td>0.414566</td>\n",
              "      <td>0.552754</td>\n",
              "      <td>0.731714</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>HR Churn</td>\n",
              "      <td>LR</td>\n",
              "      <td>0.924317</td>\n",
              "      <td>0.791667</td>\n",
              "      <td>0.343137</td>\n",
              "      <td>0.610973</td>\n",
              "      <td>0.439462</td>\n",
              "      <td>0.637448</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "10  HR Churn    Stacking  26.474826  ...  0.979418  0.982100     1\n",
              "6   HR Churn          RF   1.948336  ...  0.978663  0.980918     2\n",
              "7   HR Churn  ExtraTrees   1.726488  ...  0.974504  0.979605     3\n",
              "5   HR Churn     Bagging   6.435993  ...  0.973925  0.980831     4\n",
              "11  HR Churn      automl  94.908843  ...  0.970000  0.973959     5\n",
              "4   HR Churn      Voting   5.131726  ...  0.954193  0.967965     6\n",
              "9   HR Churn         GBC   2.744781  ...  0.950355  0.963501     7\n",
              "2   HR Churn          DT   0.066668  ...  0.945130  0.970463     8\n",
              "8   HR Churn    Adaboost   2.103505  ...  0.916725  0.945560     9\n",
              "3   HR Churn         KNN   0.170436  ...  0.892163  0.941671    10\n",
              "1   HR Churn          NB   0.011730  ...  0.552754  0.731714    11\n",
              "0   HR Churn          LR   0.924317  ...  0.439462  0.637448    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rXMf267z0T9t",
        "colab_type": "text"
      },
      "source": [
        "# US Census Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mb-OVppbrkUb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "e0bfbf7d-2747-4dd3-b48d-886fd6419769"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/adult.csv')\n",
        "r = do_all_for_dataset('Adult', df, target_col='high_salary', drop_cols=['education'])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "0    19775\n",
            "1     6273\n",
            "Name: high_salary, dtype: int64\n",
            "Y (test) counts:\n",
            "0    4945\n",
            "1    1568\n",
            "Name: high_salary, dtype: int64\n",
            "[WARNING] [2020-07-13 20:38:53,976:AutoMLSMBO(1)::ca10de356c0b63d6b727dc557714d1b4] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Adult</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>9.223710</td>\n",
              "      <td>0.871488</td>\n",
              "      <td>0.652423</td>\n",
              "      <td>0.777947</td>\n",
              "      <td>0.709677</td>\n",
              "      <td>0.796687</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Adult</td>\n",
              "      <td>GBC</td>\n",
              "      <td>13.845171</td>\n",
              "      <td>0.872563</td>\n",
              "      <td>0.642857</td>\n",
              "      <td>0.788732</td>\n",
              "      <td>0.708363</td>\n",
              "      <td>0.794128</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Adult</td>\n",
              "      <td>RF</td>\n",
              "      <td>9.886277</td>\n",
              "      <td>0.856902</td>\n",
              "      <td>0.636480</td>\n",
              "      <td>0.733824</td>\n",
              "      <td>0.681694</td>\n",
              "      <td>0.781637</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Adult</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>50.486441</td>\n",
              "      <td>0.854445</td>\n",
              "      <td>0.637755</td>\n",
              "      <td>0.724638</td>\n",
              "      <td>0.678426</td>\n",
              "      <td>0.780455</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Adult</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>102.303363</td>\n",
              "      <td>0.857362</td>\n",
              "      <td>0.605230</td>\n",
              "      <td>0.753773</td>\n",
              "      <td>0.671383</td>\n",
              "      <td>0.771270</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Adult</td>\n",
              "      <td>automl</td>\n",
              "      <td>95.460762</td>\n",
              "      <td>0.818517</td>\n",
              "      <td>0.720663</td>\n",
              "      <td>0.602988</td>\n",
              "      <td>0.656595</td>\n",
              "      <td>0.785104</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Adult</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>12.199482</td>\n",
              "      <td>0.836634</td>\n",
              "      <td>0.616709</td>\n",
              "      <td>0.676224</td>\n",
              "      <td>0.645097</td>\n",
              "      <td>0.761540</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Adult</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.380956</td>\n",
              "      <td>0.814525</td>\n",
              "      <td>0.635204</td>\n",
              "      <td>0.610294</td>\n",
              "      <td>0.622500</td>\n",
              "      <td>0.753295</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Adult</td>\n",
              "      <td>Voting</td>\n",
              "      <td>21.091315</td>\n",
              "      <td>0.838170</td>\n",
              "      <td>0.417092</td>\n",
              "      <td>0.823678</td>\n",
              "      <td>0.553768</td>\n",
              "      <td>0.694390</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Adult</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.107367</td>\n",
              "      <td>0.799324</td>\n",
              "      <td>0.317602</td>\n",
              "      <td>0.677551</td>\n",
              "      <td>0.432479</td>\n",
              "      <td>0.634837</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Adult</td>\n",
              "      <td>KNN</td>\n",
              "      <td>1.855113</td>\n",
              "      <td>0.793951</td>\n",
              "      <td>0.304847</td>\n",
              "      <td>0.654795</td>\n",
              "      <td>0.416014</td>\n",
              "      <td>0.626943</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Adult</td>\n",
              "      <td>LR</td>\n",
              "      <td>0.530645</td>\n",
              "      <td>0.799478</td>\n",
              "      <td>0.274235</td>\n",
              "      <td>0.719064</td>\n",
              "      <td>0.397045</td>\n",
              "      <td>0.620130</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Dataset      Method        Time  ...        F1       AUC  Rank\n",
              "8    Adult    Adaboost    9.223710  ...  0.709677  0.796687     1\n",
              "9    Adult         GBC   13.845171  ...  0.708363  0.794128     2\n",
              "6    Adult          RF    9.886277  ...  0.681694  0.781637     3\n",
              "5    Adult     Bagging   50.486441  ...  0.678426  0.780455     4\n",
              "10   Adult    Stacking  102.303363  ...  0.671383  0.771270     5\n",
              "11   Adult      automl   95.460762  ...  0.656595  0.785104     6\n",
              "7    Adult  ExtraTrees   12.199482  ...  0.645097  0.761540     7\n",
              "2    Adult          DT    0.380956  ...  0.622500  0.753295     8\n",
              "4    Adult      Voting   21.091315  ...  0.553768  0.694390     9\n",
              "1    Adult          NB    0.107367  ...  0.432479  0.634837    10\n",
              "3    Adult         KNN    1.855113  ...  0.416014  0.626943    11\n",
              "0    Adult          LR    0.530645  ...  0.397045  0.620130    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lOqRv3st0dhS",
        "colab_type": "text"
      },
      "source": [
        "# Orange Juice Sales"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WF6jfmiSyTXL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "9a9ace0c-1e7e-470a-cd0f-7e75c7095632"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/OJ.csv')\n",
        "r = do_all_for_dataset('OJ', df, target_col='Purchase', drop_cols=['ID'])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "0    522\n",
            "1    334\n",
            "Name: Purchase, dtype: int64\n",
            "Y (test) counts:\n",
            "0    131\n",
            "1     83\n",
            "Name: Purchase, dtype: int64\n",
            "[WARNING] [2020-07-13 20:40:39,304:AutoMLSMBO(1)::da412f258d362c1cc9abca59cca5959e] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>OJ</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>0.412627</td>\n",
              "      <td>0.855140</td>\n",
              "      <td>0.771084</td>\n",
              "      <td>0.842105</td>\n",
              "      <td>0.805031</td>\n",
              "      <td>0.839741</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>OJ</td>\n",
              "      <td>GBC</td>\n",
              "      <td>0.341138</td>\n",
              "      <td>0.845794</td>\n",
              "      <td>0.746988</td>\n",
              "      <td>0.837838</td>\n",
              "      <td>0.789809</td>\n",
              "      <td>0.827692</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>OJ</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>6.078939</td>\n",
              "      <td>0.841121</td>\n",
              "      <td>0.746988</td>\n",
              "      <td>0.826667</td>\n",
              "      <td>0.784810</td>\n",
              "      <td>0.823876</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>OJ</td>\n",
              "      <td>Voting</td>\n",
              "      <td>1.154752</td>\n",
              "      <td>0.831776</td>\n",
              "      <td>0.759036</td>\n",
              "      <td>0.797468</td>\n",
              "      <td>0.777778</td>\n",
              "      <td>0.818449</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>OJ</td>\n",
              "      <td>LR</td>\n",
              "      <td>0.190480</td>\n",
              "      <td>0.831776</td>\n",
              "      <td>0.722892</td>\n",
              "      <td>0.821918</td>\n",
              "      <td>0.769231</td>\n",
              "      <td>0.811827</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>OJ</td>\n",
              "      <td>automl</td>\n",
              "      <td>94.765673</td>\n",
              "      <td>0.822430</td>\n",
              "      <td>0.759036</td>\n",
              "      <td>0.777778</td>\n",
              "      <td>0.768293</td>\n",
              "      <td>0.810816</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>OJ</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.336726</td>\n",
              "      <td>0.808411</td>\n",
              "      <td>0.722892</td>\n",
              "      <td>0.769231</td>\n",
              "      <td>0.745342</td>\n",
              "      <td>0.792743</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>OJ</td>\n",
              "      <td>RF</td>\n",
              "      <td>0.426854</td>\n",
              "      <td>0.808411</td>\n",
              "      <td>0.698795</td>\n",
              "      <td>0.783784</td>\n",
              "      <td>0.738854</td>\n",
              "      <td>0.788329</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>OJ</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>0.745412</td>\n",
              "      <td>0.799065</td>\n",
              "      <td>0.722892</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.736196</td>\n",
              "      <td>0.785110</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>OJ</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.008439</td>\n",
              "      <td>0.789720</td>\n",
              "      <td>0.698795</td>\n",
              "      <td>0.743590</td>\n",
              "      <td>0.720497</td>\n",
              "      <td>0.773062</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>OJ</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.010538</td>\n",
              "      <td>0.738318</td>\n",
              "      <td>0.795181</td>\n",
              "      <td>0.628571</td>\n",
              "      <td>0.702128</td>\n",
              "      <td>0.748735</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>OJ</td>\n",
              "      <td>KNN</td>\n",
              "      <td>0.015966</td>\n",
              "      <td>0.738318</td>\n",
              "      <td>0.638554</td>\n",
              "      <td>0.670886</td>\n",
              "      <td>0.654321</td>\n",
              "      <td>0.720040</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "8       OJ    Adaboost   0.412627  ...  0.805031  0.839741     1\n",
              "9       OJ         GBC   0.341138  ...  0.789809  0.827692     2\n",
              "10      OJ    Stacking   6.078939  ...  0.784810  0.823876     3\n",
              "4       OJ      Voting   1.154752  ...  0.777778  0.818449     4\n",
              "0       OJ          LR   0.190480  ...  0.769231  0.811827     5\n",
              "11      OJ      automl  94.765673  ...  0.768293  0.810816     6\n",
              "7       OJ  ExtraTrees   0.336726  ...  0.745342  0.792743     7\n",
              "6       OJ          RF   0.426854  ...  0.738854  0.788329     8\n",
              "5       OJ     Bagging   0.745412  ...  0.736196  0.785110     9\n",
              "2       OJ          DT   0.008439  ...  0.720497  0.773062    10\n",
              "1       OJ          NB   0.010538  ...  0.702128  0.748735    11\n",
              "3       OJ         KNN   0.015966  ...  0.654321  0.720040    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-FU5XB4y0fq3",
        "colab_type": "text"
      },
      "source": [
        "# Bank"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2NUWhNQIy2Ze",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "bfc635a2-a8ab-4bb9-a0da-3f49cf1e2f2c"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/bank.csv')\n",
        "r = do_all_for_dataset('Bank', df, target_col='y', drop_cols=[])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "0    3199\n",
            "1     417\n",
            "Name: y, dtype: int64\n",
            "Y (test) counts:\n",
            "0    801\n",
            "1    104\n",
            "Name: y, dtype: int64\n",
            "[WARNING] [2020-07-13 20:43:03,232:AutoMLSMBO(1)::b7e952f6cd300aecd7375886d42fde2f] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Bank</td>\n",
              "      <td>Voting</td>\n",
              "      <td>8.002571</td>\n",
              "      <td>0.895028</td>\n",
              "      <td>0.413462</td>\n",
              "      <td>0.558442</td>\n",
              "      <td>0.475138</td>\n",
              "      <td>0.685507</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Bank</td>\n",
              "      <td>automl</td>\n",
              "      <td>94.526651</td>\n",
              "      <td>0.793370</td>\n",
              "      <td>0.788462</td>\n",
              "      <td>0.331984</td>\n",
              "      <td>0.467236</td>\n",
              "      <td>0.791235</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Bank</td>\n",
              "      <td>GBC</td>\n",
              "      <td>1.650500</td>\n",
              "      <td>0.889503</td>\n",
              "      <td>0.394231</td>\n",
              "      <td>0.525641</td>\n",
              "      <td>0.450549</td>\n",
              "      <td>0.674019</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Bank</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>5.624545</td>\n",
              "      <td>0.887293</td>\n",
              "      <td>0.394231</td>\n",
              "      <td>0.512500</td>\n",
              "      <td>0.445652</td>\n",
              "      <td>0.672771</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Bank</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>26.435482</td>\n",
              "      <td>0.895028</td>\n",
              "      <td>0.365385</td>\n",
              "      <td>0.567164</td>\n",
              "      <td>0.444444</td>\n",
              "      <td>0.664590</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Bank</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.053082</td>\n",
              "      <td>0.857459</td>\n",
              "      <td>0.480769</td>\n",
              "      <td>0.400000</td>\n",
              "      <td>0.436681</td>\n",
              "      <td>0.693568</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Bank</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>1.236406</td>\n",
              "      <td>0.887293</td>\n",
              "      <td>0.336538</td>\n",
              "      <td>0.514706</td>\n",
              "      <td>0.406977</td>\n",
              "      <td>0.647670</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Bank</td>\n",
              "      <td>LR</td>\n",
              "      <td>3.510823</td>\n",
              "      <td>0.893923</td>\n",
              "      <td>0.307692</td>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.400000</td>\n",
              "      <td>0.638865</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Bank</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.011552</td>\n",
              "      <td>0.817680</td>\n",
              "      <td>0.480769</td>\n",
              "      <td>0.310559</td>\n",
              "      <td>0.377358</td>\n",
              "      <td>0.671096</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Bank</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.964086</td>\n",
              "      <td>0.890608</td>\n",
              "      <td>0.240385</td>\n",
              "      <td>0.555556</td>\n",
              "      <td>0.335570</td>\n",
              "      <td>0.607708</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Bank</td>\n",
              "      <td>RF</td>\n",
              "      <td>1.077555</td>\n",
              "      <td>0.889503</td>\n",
              "      <td>0.240385</td>\n",
              "      <td>0.543478</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.607084</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Bank</td>\n",
              "      <td>KNN</td>\n",
              "      <td>0.073459</td>\n",
              "      <td>0.881768</td>\n",
              "      <td>0.173077</td>\n",
              "      <td>0.461538</td>\n",
              "      <td>0.251748</td>\n",
              "      <td>0.573430</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "4     Bank      Voting   8.002571  ...  0.475138  0.685507     1\n",
              "11    Bank      automl  94.526651  ...  0.467236  0.791235     2\n",
              "9     Bank         GBC   1.650500  ...  0.450549  0.674019     3\n",
              "5     Bank     Bagging   5.624545  ...  0.445652  0.672771     4\n",
              "10    Bank    Stacking  26.435482  ...  0.444444  0.664590     5\n",
              "2     Bank          DT   0.053082  ...  0.436681  0.693568     6\n",
              "8     Bank    Adaboost   1.236406  ...  0.406977  0.647670     7\n",
              "0     Bank          LR   3.510823  ...  0.400000  0.638865     8\n",
              "1     Bank          NB   0.011552  ...  0.377358  0.671096     9\n",
              "7     Bank  ExtraTrees   0.964086  ...  0.335570  0.607708    10\n",
              "6     Bank          RF   1.077555  ...  0.333333  0.607084    11\n",
              "3     Bank         KNN   0.073459  ...  0.251748  0.573430    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rI0wetEX0sEi",
        "colab_type": "text"
      },
      "source": [
        "# Credit Data (from ISLR)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NGQ5kfUGzGJj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "c05677c8-03a1-46e0-bfd6-c2f0eeff7661"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/islr_credit.csv')\n",
        "r = do_all_for_dataset('Credit', df, target_col='default', drop_cols=['ID'])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "0    7734\n",
            "1     266\n",
            "Name: default, dtype: int64\n",
            "Y (test) counts:\n",
            "0    1933\n",
            "1      67\n",
            "Name: default, dtype: int64\n",
            "[WARNING] [2020-07-13 20:45:05,265:AutoMLSMBO(1)::7144191554c89502dea94b3b1e15d55d] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Credit</td>\n",
              "      <td>automl</td>\n",
              "      <td>95.552348</td>\n",
              "      <td>0.9710</td>\n",
              "      <td>0.358209</td>\n",
              "      <td>0.615385</td>\n",
              "      <td>0.452830</td>\n",
              "      <td>0.675224</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Credit</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>15.910119</td>\n",
              "      <td>0.9705</td>\n",
              "      <td>0.313433</td>\n",
              "      <td>0.617647</td>\n",
              "      <td>0.415842</td>\n",
              "      <td>0.653354</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Credit</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>1.256315</td>\n",
              "      <td>0.9700</td>\n",
              "      <td>0.313433</td>\n",
              "      <td>0.600000</td>\n",
              "      <td>0.411765</td>\n",
              "      <td>0.653095</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Credit</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>2.495380</td>\n",
              "      <td>0.9680</td>\n",
              "      <td>0.313433</td>\n",
              "      <td>0.538462</td>\n",
              "      <td>0.396226</td>\n",
              "      <td>0.652060</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Credit</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.720486</td>\n",
              "      <td>0.9670</td>\n",
              "      <td>0.313433</td>\n",
              "      <td>0.512195</td>\n",
              "      <td>0.388889</td>\n",
              "      <td>0.651543</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Credit</td>\n",
              "      <td>RF</td>\n",
              "      <td>1.743766</td>\n",
              "      <td>0.9685</td>\n",
              "      <td>0.298507</td>\n",
              "      <td>0.555556</td>\n",
              "      <td>0.388350</td>\n",
              "      <td>0.645115</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Credit</td>\n",
              "      <td>Voting</td>\n",
              "      <td>3.141324</td>\n",
              "      <td>0.9680</td>\n",
              "      <td>0.283582</td>\n",
              "      <td>0.542857</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.637652</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Credit</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.005281</td>\n",
              "      <td>0.9665</td>\n",
              "      <td>0.268657</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.349515</td>\n",
              "      <td>0.629672</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Credit</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.029368</td>\n",
              "      <td>0.9560</td>\n",
              "      <td>0.328358</td>\n",
              "      <td>0.338462</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.653056</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Credit</td>\n",
              "      <td>GBC</td>\n",
              "      <td>1.605030</td>\n",
              "      <td>0.9630</td>\n",
              "      <td>0.268657</td>\n",
              "      <td>0.418605</td>\n",
              "      <td>0.327273</td>\n",
              "      <td>0.627862</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Credit</td>\n",
              "      <td>KNN</td>\n",
              "      <td>0.093189</td>\n",
              "      <td>0.9670</td>\n",
              "      <td>0.134328</td>\n",
              "      <td>0.529412</td>\n",
              "      <td>0.214286</td>\n",
              "      <td>0.565095</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Credit</td>\n",
              "      <td>LR</td>\n",
              "      <td>0.082358</td>\n",
              "      <td>0.9640</td>\n",
              "      <td>0.134328</td>\n",
              "      <td>0.391304</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.563543</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "11  Credit      automl  95.552348  ...  0.452830  0.675224     1\n",
              "10  Credit    Stacking  15.910119  ...  0.415842  0.653354     2\n",
              "8   Credit    Adaboost   1.256315  ...  0.411765  0.653095     3\n",
              "5   Credit     Bagging   2.495380  ...  0.396226  0.652060     4\n",
              "7   Credit  ExtraTrees   0.720486  ...  0.388889  0.651543     5\n",
              "6   Credit          RF   1.743766  ...  0.388350  0.645115     6\n",
              "4   Credit      Voting   3.141324  ...  0.372549  0.637652     7\n",
              "1   Credit          NB   0.005281  ...  0.349515  0.629672     8\n",
              "2   Credit          DT   0.029368  ...  0.333333  0.653056     9\n",
              "9   Credit         GBC   1.605030  ...  0.327273  0.627862    10\n",
              "3   Credit         KNN   0.093189  ...  0.214286  0.565095    11\n",
              "0   Credit          LR   0.082358  ...  0.200000  0.563543    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "itcmczwuFfMs",
        "colab_type": "text"
      },
      "source": [
        "# UCI Credit Approval\n",
        "From: https://archive.ics.uci.edu/ml/datasets/Credit+Approval"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "__xFfqJZFjIl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "d8ec0e04-6940-4bf0-9b61-fa41a90ae968"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/crx.csv')\n",
        "r = do_all_for_dataset('UCI Credit', df, target_col='class', drop_cols=[])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "1    306\n",
            "0    246\n",
            "Name: class, dtype: int64\n",
            "Y (test) counts:\n",
            "1    77\n",
            "0    61\n",
            "Name: class, dtype: int64\n",
            "[WARNING] [2020-07-13 20:47:02,251:AutoMLSMBO(1)::1c51b5b72cb21bcefcc7e214aef043ab] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>Voting</td>\n",
              "      <td>2.846164</td>\n",
              "      <td>0.847826</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>0.868421</td>\n",
              "      <td>0.862745</td>\n",
              "      <td>0.846604</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>12.236892</td>\n",
              "      <td>0.847826</td>\n",
              "      <td>0.844156</td>\n",
              "      <td>0.878378</td>\n",
              "      <td>0.860927</td>\n",
              "      <td>0.848307</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.014273</td>\n",
              "      <td>0.833333</td>\n",
              "      <td>0.922078</td>\n",
              "      <td>0.806818</td>\n",
              "      <td>0.860606</td>\n",
              "      <td>0.821695</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>1.970659</td>\n",
              "      <td>0.847826</td>\n",
              "      <td>0.831169</td>\n",
              "      <td>0.888889</td>\n",
              "      <td>0.859060</td>\n",
              "      <td>0.850011</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.427059</td>\n",
              "      <td>0.847826</td>\n",
              "      <td>0.831169</td>\n",
              "      <td>0.888889</td>\n",
              "      <td>0.859060</td>\n",
              "      <td>0.850011</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>0.978684</td>\n",
              "      <td>0.847826</td>\n",
              "      <td>0.818182</td>\n",
              "      <td>0.900000</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>0.851714</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>RF</td>\n",
              "      <td>0.496387</td>\n",
              "      <td>0.840580</td>\n",
              "      <td>0.831169</td>\n",
              "      <td>0.876712</td>\n",
              "      <td>0.853333</td>\n",
              "      <td>0.841814</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>automl</td>\n",
              "      <td>95.071934</td>\n",
              "      <td>0.826087</td>\n",
              "      <td>0.857143</td>\n",
              "      <td>0.835443</td>\n",
              "      <td>0.846154</td>\n",
              "      <td>0.822014</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>KNN</td>\n",
              "      <td>0.073794</td>\n",
              "      <td>0.797101</td>\n",
              "      <td>0.922078</td>\n",
              "      <td>0.763441</td>\n",
              "      <td>0.835294</td>\n",
              "      <td>0.780711</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>GBC</td>\n",
              "      <td>0.945455</td>\n",
              "      <td>0.826087</td>\n",
              "      <td>0.779221</td>\n",
              "      <td>0.895522</td>\n",
              "      <td>0.833333</td>\n",
              "      <td>0.832233</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>LR</td>\n",
              "      <td>1.018615</td>\n",
              "      <td>0.818841</td>\n",
              "      <td>0.779221</td>\n",
              "      <td>0.882353</td>\n",
              "      <td>0.827586</td>\n",
              "      <td>0.824037</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>UCI Credit</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.029712</td>\n",
              "      <td>0.789855</td>\n",
              "      <td>0.844156</td>\n",
              "      <td>0.792683</td>\n",
              "      <td>0.817610</td>\n",
              "      <td>0.782734</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "4   UCI Credit      Voting   2.846164  ...  0.862745  0.846604     1\n",
              "10  UCI Credit    Stacking  12.236892  ...  0.860927  0.848307     2\n",
              "1   UCI Credit          NB   0.014273  ...  0.860606  0.821695     3\n",
              "5   UCI Credit     Bagging   1.970659  ...  0.859060  0.850011     4\n",
              "7   UCI Credit  ExtraTrees   0.427059  ...  0.859060  0.850011     5\n",
              "8   UCI Credit    Adaboost   0.978684  ...  0.857143  0.851714     6\n",
              "6   UCI Credit          RF   0.496387  ...  0.853333  0.841814     7\n",
              "11  UCI Credit      automl  95.071934  ...  0.846154  0.822014     8\n",
              "3   UCI Credit         KNN   0.073794  ...  0.835294  0.780711     9\n",
              "9   UCI Credit         GBC   0.945455  ...  0.833333  0.832233    10\n",
              "0   UCI Credit          LR   1.018615  ...  0.827586  0.824037    11\n",
              "2   UCI Credit          DT   0.029712  ...  0.817610  0.782734    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7p3RoNebF2RM",
        "colab_type": "text"
      },
      "source": [
        "# King-Rook vs. King\n",
        "\n",
        "From: https://archive.ics.uci.edu/ml/datasets/Chess+%28King-Rook+vs.+King%29"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "goMnSeEJFzFM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 594
        },
        "outputId": "2469daa8-533a-4fff-9d06-97230c980d7b"
      },
      "source": [
        "df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/krkopt.csv')\n",
        "df['result'] = df['result'] != 'draw' # Change to binary classification problem\n",
        "r = do_all_for_dataset('King Rook', df, target_col='result', drop_cols=[])\n",
        "results.append(r)\n",
        "r"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Y (train) counts:\n",
            "True     20207\n",
            "False     2237\n",
            "Name: result, dtype: int64\n",
            "Y (test) counts:\n",
            "True     5053\n",
            "False     559\n",
            "Name: result, dtype: int64\n",
            "[WARNING] [2020-07-13 20:49:41,377:AutoMLSMBO(1)::d98b10bfbb65f403657eb830d18d5a8c] Could not find meta-data directory /usr/local/lib/python3.6/dist-packages/autosklearn/metalearning/files/f1_score_binary.classification_dense\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Dataset</th>\n",
              "      <th>Method</th>\n",
              "      <th>Time</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Recall</th>\n",
              "      <th>Precision</th>\n",
              "      <th>F1</th>\n",
              "      <th>AUC</th>\n",
              "      <th>Rank</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>Bagging</td>\n",
              "      <td>6.672816</td>\n",
              "      <td>0.995723</td>\n",
              "      <td>0.997427</td>\n",
              "      <td>0.997822</td>\n",
              "      <td>0.997625</td>\n",
              "      <td>0.988875</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>Stacking</td>\n",
              "      <td>34.802662</td>\n",
              "      <td>0.992338</td>\n",
              "      <td>0.997427</td>\n",
              "      <td>0.994083</td>\n",
              "      <td>0.995752</td>\n",
              "      <td>0.971880</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>DT</td>\n",
              "      <td>0.060836</td>\n",
              "      <td>0.988774</td>\n",
              "      <td>0.993865</td>\n",
              "      <td>0.993668</td>\n",
              "      <td>0.993767</td>\n",
              "      <td>0.968310</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>automl</td>\n",
              "      <td>95.704650</td>\n",
              "      <td>0.981468</td>\n",
              "      <td>0.996834</td>\n",
              "      <td>0.982829</td>\n",
              "      <td>0.989782</td>\n",
              "      <td>0.919705</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>GBC</td>\n",
              "      <td>3.809935</td>\n",
              "      <td>0.968460</td>\n",
              "      <td>0.998219</td>\n",
              "      <td>0.967767</td>\n",
              "      <td>0.982757</td>\n",
              "      <td>0.848841</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>RF</td>\n",
              "      <td>2.882421</td>\n",
              "      <td>0.956522</td>\n",
              "      <td>0.984762</td>\n",
              "      <td>0.967529</td>\n",
              "      <td>0.976069</td>\n",
              "      <td>0.843007</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>KNN</td>\n",
              "      <td>1.487516</td>\n",
              "      <td>0.955809</td>\n",
              "      <td>0.997823</td>\n",
              "      <td>0.955105</td>\n",
              "      <td>0.975997</td>\n",
              "      <td>0.786926</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>3.099249</td>\n",
              "      <td>0.945474</td>\n",
              "      <td>0.969721</td>\n",
              "      <td>0.969721</td>\n",
              "      <td>0.969721</td>\n",
              "      <td>0.848009</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>Voting</td>\n",
              "      <td>6.729951</td>\n",
              "      <td>0.941376</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.938870</td>\n",
              "      <td>0.968471</td>\n",
              "      <td>0.705725</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>LR</td>\n",
              "      <td>0.409874</td>\n",
              "      <td>0.900392</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.900392</td>\n",
              "      <td>0.947586</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>Adaboost</td>\n",
              "      <td>3.365810</td>\n",
              "      <td>0.900392</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.900392</td>\n",
              "      <td>0.947586</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>King Rook</td>\n",
              "      <td>NB</td>\n",
              "      <td>0.025042</td>\n",
              "      <td>0.855488</td>\n",
              "      <td>0.934692</td>\n",
              "      <td>0.907571</td>\n",
              "      <td>0.920932</td>\n",
              "      <td>0.537114</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      Dataset      Method       Time  ...        F1       AUC  Rank\n",
              "5   King Rook     Bagging   6.672816  ...  0.997625  0.988875     1\n",
              "10  King Rook    Stacking  34.802662  ...  0.995752  0.971880     2\n",
              "2   King Rook          DT   0.060836  ...  0.993767  0.968310     3\n",
              "11  King Rook      automl  95.704650  ...  0.989782  0.919705     4\n",
              "9   King Rook         GBC   3.809935  ...  0.982757  0.848841     5\n",
              "6   King Rook          RF   2.882421  ...  0.976069  0.843007     6\n",
              "3   King Rook         KNN   1.487516  ...  0.975997  0.786926     7\n",
              "7   King Rook  ExtraTrees   3.099249  ...  0.969721  0.848009     8\n",
              "4   King Rook      Voting   6.729951  ...  0.968471  0.705725     9\n",
              "0   King Rook          LR   0.409874  ...  0.947586  0.500000    10\n",
              "8   King Rook    Adaboost   3.365810  ...  0.947586  0.500000    11\n",
              "1   King Rook          NB   0.025042  ...  0.920932  0.537114    12\n",
              "\n",
              "[12 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8SOZN4eY0wxK",
        "colab_type": "text"
      },
      "source": [
        "# Credit Card Fraud"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9YjUuIqrntx-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#df = pd.read_csv('https://raw.githubusercontent.com/stepthom/sandbox/master/data/creditcard_sample.csv')\n",
        "# This dataset is huge, so let's take a sample to speed things up\n",
        "#df = df.sample(frac=0.3, replace=False, random_state=1, axis=0)\n",
        "#r = do_all_for_dataset('Credit Card', df, target_col='Class', drop_cols=[])\n",
        "#results.append(r)\n",
        "#r"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EQbDEh7x3skI",
        "colab_type": "text"
      },
      "source": [
        "# Overall Results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f9O4ujW03vuk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 887
        },
        "outputId": "b506fba9-4453-48dd-9751-ba1e4a0d1c23"
      },
      "source": [
        "r = pd.concat([m for m in results], axis = 0).reset_index()\n",
        "r = r.drop(columns = \"index\",axis =1)\n",
        "r.groupby('Method').describe().Rank.sort_values(by='mean')\n",
        "r.groupby('Method').describe().Time.sort_values(by='mean')"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>25%</th>\n",
              "      <th>50%</th>\n",
              "      <th>75%</th>\n",
              "      <th>max</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Method</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>automl</th>\n",
              "      <td>10.0</td>\n",
              "      <td>4.3</td>\n",
              "      <td>2.451757</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2.00</td>\n",
              "      <td>4.5</td>\n",
              "      <td>6.00</td>\n",
              "      <td>8.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Stacking</th>\n",
              "      <td>10.0</td>\n",
              "      <td>4.4</td>\n",
              "      <td>3.134042</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2.00</td>\n",
              "      <td>4.0</td>\n",
              "      <td>5.00</td>\n",
              "      <td>11.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Bagging</th>\n",
              "      <td>10.0</td>\n",
              "      <td>4.5</td>\n",
              "      <td>2.013841</td>\n",
              "      <td>1.0</td>\n",
              "      <td>4.00</td>\n",
              "      <td>4.0</td>\n",
              "      <td>4.75</td>\n",
              "      <td>9.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>GBC</th>\n",
              "      <td>10.0</td>\n",
              "      <td>4.7</td>\n",
              "      <td>3.335000</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2.00</td>\n",
              "      <td>4.0</td>\n",
              "      <td>6.50</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Adaboost</th>\n",
              "      <td>10.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>4.082483</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.00</td>\n",
              "      <td>4.5</td>\n",
              "      <td>8.50</td>\n",
              "      <td>11.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Voting</th>\n",
              "      <td>10.0</td>\n",
              "      <td>5.5</td>\n",
              "      <td>3.064129</td>\n",
              "      <td>1.0</td>\n",
              "      <td>3.25</td>\n",
              "      <td>6.5</td>\n",
              "      <td>7.75</td>\n",
              "      <td>9.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>RF</th>\n",
              "      <td>10.0</td>\n",
              "      <td>5.7</td>\n",
              "      <td>2.626785</td>\n",
              "      <td>2.0</td>\n",
              "      <td>4.00</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.75</td>\n",
              "      <td>11.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ExtraTrees</th>\n",
              "      <td>10.0</td>\n",
              "      <td>6.6</td>\n",
              "      <td>2.503331</td>\n",
              "      <td>3.0</td>\n",
              "      <td>5.00</td>\n",
              "      <td>7.0</td>\n",
              "      <td>8.75</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>DT</th>\n",
              "      <td>10.0</td>\n",
              "      <td>8.5</td>\n",
              "      <td>2.592725</td>\n",
              "      <td>3.0</td>\n",
              "      <td>8.00</td>\n",
              "      <td>8.5</td>\n",
              "      <td>10.00</td>\n",
              "      <td>12.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>NB</th>\n",
              "      <td>10.0</td>\n",
              "      <td>8.5</td>\n",
              "      <td>3.439961</td>\n",
              "      <td>3.0</td>\n",
              "      <td>6.50</td>\n",
              "      <td>9.5</td>\n",
              "      <td>11.00</td>\n",
              "      <td>12.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LR</th>\n",
              "      <td>10.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>2.211083</td>\n",
              "      <td>5.0</td>\n",
              "      <td>9.25</td>\n",
              "      <td>10.5</td>\n",
              "      <td>11.75</td>\n",
              "      <td>12.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>KNN</th>\n",
              "      <td>10.0</td>\n",
              "      <td>10.3</td>\n",
              "      <td>2.002776</td>\n",
              "      <td>7.0</td>\n",
              "      <td>9.25</td>\n",
              "      <td>11.0</td>\n",
              "      <td>12.00</td>\n",
              "      <td>12.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            count  mean       std  min   25%   50%    75%   max\n",
              "Method                                                         \n",
              "automl       10.0   4.3  2.451757  1.0  2.00   4.5   6.00   8.0\n",
              "Stacking     10.0   4.4  3.134042  1.0  2.00   4.0   5.00  11.0\n",
              "Bagging      10.0   4.5  2.013841  1.0  4.00   4.0   4.75   9.0\n",
              "GBC          10.0   4.7  3.335000  1.0  2.00   4.0   6.50  10.0\n",
              "Adaboost     10.0   5.0  4.082483  1.0  1.00   4.5   8.50  11.0\n",
              "Voting       10.0   5.5  3.064129  1.0  3.25   6.5   7.75   9.0\n",
              "RF           10.0   5.7  2.626785  2.0  4.00   6.0   6.75  11.0\n",
              "ExtraTrees   10.0   6.6  2.503331  3.0  5.00   7.0   8.75  10.0\n",
              "DT           10.0   8.5  2.592725  3.0  8.00   8.5  10.00  12.0\n",
              "NB           10.0   8.5  3.439961  3.0  6.50   9.5  11.00  12.0\n",
              "LR           10.0  10.0  2.211083  5.0  9.25  10.5  11.75  12.0\n",
              "KNN          10.0  10.3  2.002776  7.0  9.25  11.0  12.00  12.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>25%</th>\n",
              "      <th>50%</th>\n",
              "      <th>75%</th>\n",
              "      <th>max</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Method</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>NB</th>\n",
              "      <td>10.0</td>\n",
              "      <td>0.019692</td>\n",
              "      <td>0.031509</td>\n",
              "      <td>0.002595</td>\n",
              "      <td>0.005162</td>\n",
              "      <td>0.011045</td>\n",
              "      <td>0.013637</td>\n",
              "      <td>0.107367</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>DT</th>\n",
              "      <td>10.0</td>\n",
              "      <td>0.065190</td>\n",
              "      <td>0.113343</td>\n",
              "      <td>0.004762</td>\n",
              "      <td>0.009441</td>\n",
              "      <td>0.029540</td>\n",
              "      <td>0.058897</td>\n",
              "      <td>0.380956</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>KNN</th>\n",
              "      <td>10.0</td>\n",
              "      <td>0.381270</td>\n",
              "      <td>0.687203</td>\n",
              "      <td>0.004526</td>\n",
              "      <td>0.019030</td>\n",
              "      <td>0.073627</td>\n",
              "      <td>0.151124</td>\n",
              "      <td>1.855113</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LR</th>\n",
              "      <td>10.0</td>\n",
              "      <td>0.709857</td>\n",
              "      <td>1.041170</td>\n",
              "      <td>0.040350</td>\n",
              "      <td>0.150811</td>\n",
              "      <td>0.331699</td>\n",
              "      <td>0.825899</td>\n",
              "      <td>3.510823</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>RF</th>\n",
              "      <td>10.0</td>\n",
              "      <td>1.961118</td>\n",
              "      <td>2.914727</td>\n",
              "      <td>0.316920</td>\n",
              "      <td>0.425232</td>\n",
              "      <td>0.786971</td>\n",
              "      <td>1.897193</td>\n",
              "      <td>9.886277</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Adaboost</th>\n",
              "      <td>10.0</td>\n",
              "      <td>1.980205</td>\n",
              "      <td>2.715555</td>\n",
              "      <td>0.321829</td>\n",
              "      <td>0.432842</td>\n",
              "      <td>1.107545</td>\n",
              "      <td>1.891708</td>\n",
              "      <td>9.223710</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ExtraTrees</th>\n",
              "      <td>10.0</td>\n",
              "      <td>2.037973</td>\n",
              "      <td>3.680456</td>\n",
              "      <td>0.216772</td>\n",
              "      <td>0.347896</td>\n",
              "      <td>0.573773</td>\n",
              "      <td>1.535887</td>\n",
              "      <td>12.199482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>GBC</th>\n",
              "      <td>10.0</td>\n",
              "      <td>2.589474</td>\n",
              "      <td>4.127622</td>\n",
              "      <td>0.177967</td>\n",
              "      <td>0.369075</td>\n",
              "      <td>1.275242</td>\n",
              "      <td>2.471211</td>\n",
              "      <td>13.845171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Voting</th>\n",
              "      <td>10.0</td>\n",
              "      <td>5.121221</td>\n",
              "      <td>6.153792</td>\n",
              "      <td>0.904772</td>\n",
              "      <td>1.166013</td>\n",
              "      <td>2.993744</td>\n",
              "      <td>6.330395</td>\n",
              "      <td>21.091315</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Bagging</th>\n",
              "      <td>10.0</td>\n",
              "      <td>7.663587</td>\n",
              "      <td>15.245154</td>\n",
              "      <td>0.384849</td>\n",
              "      <td>0.843386</td>\n",
              "      <td>2.233020</td>\n",
              "      <td>6.233131</td>\n",
              "      <td>50.486441</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Stacking</th>\n",
              "      <td>10.0</td>\n",
              "      <td>24.046603</td>\n",
              "      <td>29.486327</td>\n",
              "      <td>4.681041</td>\n",
              "      <td>6.256513</td>\n",
              "      <td>14.073506</td>\n",
              "      <td>26.464990</td>\n",
              "      <td>102.303363</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>automl</th>\n",
              "      <td>10.0</td>\n",
              "      <td>95.134853</td>\n",
              "      <td>0.421146</td>\n",
              "      <td>94.526651</td>\n",
              "      <td>94.801465</td>\n",
              "      <td>95.166766</td>\n",
              "      <td>95.486084</td>\n",
              "      <td>95.704650</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            count       mean        std  ...        50%        75%         max\n",
              "Method                                   ...                                  \n",
              "NB           10.0   0.019692   0.031509  ...   0.011045   0.013637    0.107367\n",
              "DT           10.0   0.065190   0.113343  ...   0.029540   0.058897    0.380956\n",
              "KNN          10.0   0.381270   0.687203  ...   0.073627   0.151124    1.855113\n",
              "LR           10.0   0.709857   1.041170  ...   0.331699   0.825899    3.510823\n",
              "RF           10.0   1.961118   2.914727  ...   0.786971   1.897193    9.886277\n",
              "Adaboost     10.0   1.980205   2.715555  ...   1.107545   1.891708    9.223710\n",
              "ExtraTrees   10.0   2.037973   3.680456  ...   0.573773   1.535887   12.199482\n",
              "GBC          10.0   2.589474   4.127622  ...   1.275242   2.471211   13.845171\n",
              "Voting       10.0   5.121221   6.153792  ...   2.993744   6.330395   21.091315\n",
              "Bagging      10.0   7.663587  15.245154  ...   2.233020   6.233131   50.486441\n",
              "Stacking     10.0  24.046603  29.486327  ...  14.073506  26.464990  102.303363\n",
              "automl       10.0  95.134853   0.421146  ...  95.166766  95.486084   95.704650\n",
              "\n",
              "[12 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    }
  ]
}