test3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from utils import *\n",
    "import pandas as pd\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "data = pd.read_table(\"data/rte_crowd.txt\", header=None)\n",
    "truth = pd.read_table(\"data/rte_truth.txt\", header=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    800\n",
       "1      2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "truth.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    800\n",
       "1    164\n",
       "2      2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "truth = truth.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[~data[1].isin(data.merge(truth, on=0)[\"1_x\"])].nunique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [0, 1, 2]\n",
       "Index: []"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[~data[1].isin(data.merge(truth, on=0)[\"1_x\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0\n",
       "1     \n",
       "1  400\n",
       "2  400"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "truth.groupby(1).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800 164 2\n"
     ]
    }
   ],
   "source": [
    "n, m, k = np.max(np.array(data), axis=0)\n",
    "print(n, m, k)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1,  0,  1, ..., -1, -1, -1],\n",
       "       [ 1,  0,  0, ...,  1,  0,  1],\n",
       "       [ 1,  0,  1, ..., -1, -1, -1],\n",
       "       ...,\n",
       "       [-1, -1, -1, ..., -1, -1, -1],\n",
       "       [-1, -1, -1, ..., -1, -1, -1],\n",
       "       [-1, -1, -1, ..., -1, -1, -1]], dtype=int64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = transform_data(data)\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranking = truth.sort_values(1)[0]-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged = data.merge(truth,on=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9390243902439024\n"
     ]
    }
   ],
   "source": [
    "# proportion of missing values\n",
    "missing = np.mean(labels == -1)\n",
    "print(missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAYkklEQVR4nO3de7hcdX3v8fcHNiAQlMTs8MQkskEjAp7a0i2gHjGK1oCWpMfSBjVGjE2taLUPVS61cqzSg6c9Xnos0ig0wVJoDoKJKJYYjNFyc4MIhICJAklISHa4iIDFBr7nj/XbsBhmMmvPZV9++byeZz+zLr+11ve3ZvZn1qw1F0UEZmaWlz1GuwAzM+s8h7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7hmR9G5J14zCdl8vab2kxyTNrTP/XklvqbiukPTyFuvoyrKSrpa0oJX1doukWZI2j3YdjYyV+iQtkfTZ0a5jNDjcm0jB9OsUXNsk/bOkCWOgrr4USD1D0yLikoj4vVEo52+AL0fEhIj45ihsv6si4oSIWAog6X2SfjTaNZk143Cv5vcjYgJwFPAa4JO1Dcoh220jua2KDgbWjnYRZmWS9hyL6xopDvdhiIj7gauBV8EzL+VPk7QeWJ+m/YmkDZIekrRC0kuGlk/t/1zSLyTtkPR3kvZI8/aQ9ElJ90naLuliSS9K84aO0hdK2ghcC6xJq30kvap4be1RpaTXSfqxpF+m29eV5q2W9BlJ/yHpV5KukTS5Ud8b9UvSz4FDgW+lOvbZ1T6UdLSk6yU9ImmrpC9L2rum2Yn19lFa/v2S1kl6WNK/Szq4wXb2kfT3kjamV1wXSNq3NP/jaftbJL2/Sc2rJX1A0uHABcBrU18fabatodMTkj6R7tetkuZKOlHSz9L+PLtm/wxIejSt6/NNajs77ad7Jb07TXtNWran1O6dkm4d7r6SNFHSVZIG0z6/StL00rKTVLya3ZLmf7Nm3aeX+n1qg+2/SdLtpfHvSbqpNP4jpdN9kg5P98cjktZKOqnUbomkr0j6jqTHgTfVbOcASd+X9A8qvFLSynQf3C3pj6qua1yICP/t4g+4F3hLGp5BcYT6mTQewEpgErAv8GZgB8UR/j7A/wXWlNYVwPdT+5cCPwM+kOa9H9hAEZQTgCuAr6d5fWnZi4H907aGpvWU1v8+4EdpeBLwMDAf6AFOSeMvTvNXAz8HXpHWtxo4r8E+aNavZ/ZRhX34u8CxqaY+YB3wsYr7aG7aR4en5T8JXFez7MvT8BeBFWk9BwDfAv5Xmjcb2EbxJL0/8K/lZevUv7pUwzP7uDR/V9uaBewEPgXsBfwJMJi2eQBwJPCfwKGp/fXA/DQ8ATi2QU1D6/18uk/eCDwOHJbm3wmcUGp/JXB6g3Xtqv4XA+8E9kvz/h/wzdKy3wb+DZiY+vfGmvr+Jk0/EXgCmFhn+y8Afg1MTvfrA8CWtL1907wXp/VsAM4G9qZ4XP6q1OclwC+B11McuL4gTftsWv4m4LOp7f7AJuDUtM2jKB7jRzZa12hn0bCza7QLGOt/FMH0GPAIcB9wPrBvmhfAm0ttLwT+d2l8AvBfQF+p/ezS/A8Bq9LwKuBDpXmHpWWHQjBIAZDmD01rFO7zgZtq+nI98L40vBr4ZE0t322wD5r1614qhnudeR8DriyN72ofXQ0sLM3bgyIwDi4t+3JAFEH3slLb1wL3pOGLKD2RUTzBtRTuFbY1iyKc9kzjB6RtHVNqfzMwNw2vAT4NTG7yuJxFEZ77l6YtA/46DZ8BXJKGJ6X9NLXOenZZf532vw08nIanAk9TP7CH+l1+fG6n8ZPVD4H/QfHEf03qy2yKI+bbUps3UAT/HqXlLgX+ZxpeAlxcs94l6f6+A/h4afofAz+saftPwDmN1jXe/sbauduxam5EfK/BvE2l4ZcAtwyNRMRjkh4EplEEXG37+9IyQ8veVzOvBziowbaaqV3f0DqnlcYfKA0/QRHajdbVrF+VSHoFxdFmP8XRYA9FuJU12kcHA1+S9H/Kq0x1lPvam9Z9s6Ryu6Hzpi+p2WbtfhqOZtsCeDAinkrDv06320rzf82z+34hxdHuXZLuAT4dEVc12PbDEfF4aby8r/4FWKfi4v8fUQTZ1uHWL2k/4AsUQTsxzT9AxTnoGcBDEfFwg/oejIidpfFdPcZ+QPGEsDkNP0zxauTJNE7q26aIeLqmz+XHdL3/kbdTHKBdUJp2MHDM0Km1pAf4epN1jRs+596+8tdqbqF40AAgaX+Kl4P3l9rMKA2/NC3zvGXTvJ08NwSiwXA9tesbWuf9ddo2U6VfVX0FuAuYGREvpHiJrZo2jfbRJuBPI+LA0t++EXFdzfI7KALzyFK7F0VxURxga51tVFW735tta1giYn1EnAJMAT4HXJ72dz0Ta+Y9s6+iuD50PfAHFK/ivv78xSvVfzrFq8hj0v11XJouivtjkqQDW+hqraFwPy4N/4Ai3N/Is+G+BZhRvgbD8x/T9f4vvgp8F/hOaX9tAn5Q81iaEBF/1mRd44bDvbP+FThV0m+ruLD4t8CNEXFvqc3H00WqGcBHKc5XQvHy8i8kHZKOtv4W+LeaI5+yQYqXxIc2mP8d4BWS3iWpR9IfA0cAjY4C2+1XVQcAjwKPSXol8Gd12jTaRxcAZ0k6EkDSiySdXLtwOrL7KvAFSVNS22mS3paaLAPeJ+mIdGR6zjDq3wZMV7oIXGFbwyLpPZJ603ofSZOf2sUin5a0t6Q3AO+gOCc+5GLgE8B/ozjn/jwV6j+AIvwfkTSJ0r5KrwSuBs5P99deko6jNddRPIkcTXE6cS3p6Jpn3zxwI8UppE+kbc0Cfh+4rML6PwzcDVyVLhZfRfH/MT+tay8VF6IPb7H+Mcfh3kERsQr4a+AbFEeHLwPm1TRbTnFK4FaKi1EXpukXURxdrQHuobjI9pFdbOsJ4FzgP9I7B46tmf8gxT/76cCDFP/k74iIHV3qV1V/CbyL4kLYV3k2uMvq7qOIuJLiaPYySY9SnEc9ocF2zqC4+HZDavs9ivAgIq6muIh4bWpz7TDqv5biovoDkob2ZcNttWA2sFbSY8CXgHkR8Z8N2j5AcfpiC3AJ8MGIuKs0/0qKgLyy5vRNrV3V/0WKi5o7gBsojoDL5lNcf7mL4pz6x5p38flSfbcAayPiN2ny9cB9EbE9tfkNcBLFfb6D4vrXe2v63Gj9ASyiOGJfnmr+PYrH8RaKffk5iovTWVC6eGAjQFJQnI7YMNq12O5BxVtV/3QX14wsUz5yN8uUpHdSnDcezisTy4TfLWOWIUmrKa6xzK95d4ntJnxaxswsQz4tY2aWoTFxWmby5MnR19c32mWYmY0rN998846I6K03b0yEe19fHwMDA6NdhpnZuCKp4aerfVrGzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMtQ03CVdlH7g9o468/5SxQ83Ty5NO0vFDynf3ep3WpuZWXuqHLkvofiO6edIP6TwVmBjadoRFN+PfGRa5vz0c1xmZjaCmn5CNSLWSOqrM+sLFD8Asbw0bQ5wWUQ8CdwjaQPFL6tc34FaG+o789uV2t173tu7WYaZ2ZjR0jl3SScB90fET2tmTeO5Pyq7mef+eG15HYskDUgaGBwcbKUMMzNrYNjhnn5z8q+AT9WbXWda3e8UjojFEdEfEf29vXW/98bMzFrUyheHvQw4BPipJIDpwC2SjqY4Ui//qvx0nv3lejMzGyHDPnKPiNsjYkpE9EVEH0WgHxURDwArgHmS9pF0CDATuKmjFZuZWVNV3gp5KcUF0cMkbZa0sFHbiFgLLAPupPiV9NMi4qlOFWtmZtVUebfMKU3m99WMnwuc215ZZmbWDn9C1cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy1DTcJd0kaTtku4oTfs7SXdJuk3SlZIOLM07S9IGSXdLeluX6jYzs12ocuS+BJhdM20l8KqI+C3gZ8BZAJKOAOYBR6Zlzpe0Z8eqNTOzSpqGe0SsAR6qmXZNROxMozcA09PwHOCyiHgyIu4BNgBHd7BeMzOroBPn3N8PXJ2GpwGbSvM2p2lmZjaC2gp3SX8F7AQuGZpUp1k0WHaRpAFJA4ODg+2UYWZmNVoOd0kLgHcA746IoQDfDMwoNZsObKm3fEQsjoj+iOjv7e1ttQwzM6ujpXCXNBs4AzgpIp4ozVoBzJO0j6RDgJnATe2XaWZmw9HTrIGkS4FZwGRJm4FzKN4dsw+wUhLADRHxwYhYK2kZcCfF6ZrTIuKpbhVvZmb1NQ33iDilzuQLd9H+XODcdooyM7P2+BOqZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZahruki6StF3SHaVpkyStlLQ+3U4szTtL0gZJd0t6W7cKNzOzxqocuS8BZtdMOxNYFREzgVVpHElHAPOAI9My50vas2PVmplZJU3DPSLWAA/VTJ4DLE3DS4G5pemXRcSTEXEPsAE4ujOlmplZVa2ecz8oIrYCpNspafo0YFOp3eY07XkkLZI0IGlgcHCwxTLMzKyeTl9QVZ1pUa9hRCyOiP6I6O/t7e1wGWZmu7dWw32bpKkA6XZ7mr4ZmFFqNx3Y0np5ZmbWilbDfQWwIA0vAJaXps+TtI+kQ4CZwE3tlWhmZsPV06yBpEuBWcBkSZuBc4DzgGWSFgIbgZMBImKtpGXAncBO4LSIeKpLtZuZWQNNwz0iTmkw6/gG7c8Fzm2nKDMza48/oWpmliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWWorXCX9BeS1kq6Q9Klkl4gaZKklZLWp9uJnSrWzMyqaTncJU0D/hzoj4hXAXsC84AzgVURMRNYlcbNzGwEtXtapgfYV1IPsB+wBZgDLE3zlwJz29yGmZkNU8vhHhH3A38PbAS2Ar+MiGuAgyJia2qzFZhSb3lJiyQNSBoYHBxstQwzM6ujndMyEymO0g8BXgLsL+k9VZePiMUR0R8R/b29va2WYWZmdbRzWuYtwD0RMRgR/wVcAbwO2CZpKkC63d5+mWZmNhzthPtG4FhJ+0kScDywDlgBLEhtFgDL2yvRzMyGq6fVBSPiRkmXA7cAO4GfAIuBCcAySQspngBO7kShZmZWXcvhDhAR5wDn1Ex+kuIo3szMRok/oWpmliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mlqG2wl3SgZIul3SXpHWSXitpkqSVktan24mdKtbMzKpp98j9S8B3I+KVwKuBdcCZwKqImAmsSuNmZjaCWg53SS8EjgMuBIiI30TEI8AcYGlqthSY216JZmY2XO0cuR8KDAL/LOknkr4maX/goIjYCpBup9RbWNIiSQOSBgYHB9sow8zMarUT7j3AUcBXIuJ3gMcZximYiFgcEf0R0d/b29tGGWZmVqudcN8MbI6IG9P45RRhv03SVIB0u729Es3MbLhaDveIeADYJOmwNOl44E5gBbAgTVsALG+rQjMzG7aeNpf/CHCJpL2BXwCnUjxhLJO0ENgInNzmNszMbJjaCveIuBXorzPr+HbWa2Zm7fEnVM3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczswy1He6S9pT0E0lXpfFJklZKWp9uJ7ZfppmZDUcnjtw/CqwrjZ8JrIqImcCqNG5mZiOorXCXNB14O/C10uQ5wNI0vBSY2842zMxs+No9cv8i8Ang6dK0gyJiK0C6nVJvQUmLJA1IGhgcHGyzDDMzK2s53CW9A9geETe3snxELI6I/ojo7+3tbbUMMzOro6eNZV8PnCTpROAFwAsl/QuwTdLUiNgqaSqwvROFmplZdS0fuUfEWRExPSL6gHnAtRHxHmAFsCA1WwAsb7tKMzMblm68z/084K2S1gNvTeNmZjaC2jkt84yIWA2sTsMPAsd3Yr1mZtYaf0LVzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLkMPdzCxDDnczsww53M3MMuRwNzPLUMvhLmmGpO9LWidpraSPpumTJK2UtD7dTuxcuWZmVkU7R+47gdMj4nDgWOA0SUcAZwKrImImsCqNm5nZCGo53CNia0TckoZ/BawDpgFzgKWp2VJgbps1mpnZMHXknLukPuB3gBuBgyJiKxRPAMCUBssskjQgaWBwcLATZZiZWdJ2uEuaAHwD+FhEPFp1uYhYHBH9EdHf29vbbhlmZlbSVrhL2osi2C+JiCvS5G2Spqb5U4Ht7ZVoZmbD1c67ZQRcCKyLiM+XZq0AFqThBcDy1sszM7NW9LSx7OuB+cDtkm5N084GzgOWSVoIbARObqtCMzMbtpbDPSJ+BKjB7ONbXa+ZmbXPn1A1M8uQw93MLEMOdzOzDDnczcwy5HA3M8uQw93MLEMOdzOzDDnczcwy5HA3M8tQO18/MO70nfntSu3uPe/tXa7EzKy7fORuZpYhh7uZWYYc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5llaLf6EFNV/rCTmY13PnI3M8uQw93MLENdC3dJsyXdLWmDpDO7tR0zM3u+rpxzl7Qn8I/AW4HNwI8lrYiIO7uxvdFS9dx8VVXP4Q9nu52+LrA7Xo/Ipc+j+bgZLd2470br/364unXkfjSwISJ+ERG/AS4D5nRpW2ZmVkMR0fmVSn8IzI6ID6Tx+cAxEfHhUptFwKI0ehhw9y5WORnY0fFCx77dtd+w+/bd/d79tNP3gyOit96Mbr0VUnWmPedZJCIWA4srrUwaiIj+ThQ2nuyu/Ybdt+/u9+6nW33v1mmZzcCM0vh0YEuXtmVmZjW6Fe4/BmZKOkTS3sA8YEWXtmVmZjW6clomInZK+jDw78CewEURsbaNVVY6fZOh3bXfsPv23f3e/XSl7125oGpmZqPLn1A1M8uQw93MLENjJtybfV2BCv+Q5t8m6ajRqLMbKvT93anPt0m6TtKrR6POTqv6FRWSXiPpqfT5iSxU6bukWZJulbRW0g9GusZuqPBYf5Gkb0n6aer3qaNRZ6dJukjSdkl3NJjf+XyLiFH/o7jo+nPgUGBv4KfAETVtTgSupngP/bHAjaNd9wj2/XXAxDR8Qg59r9LvUrtrge8AfzjadY/gfX4gcCfw0jQ+ZbTrHqF+nw18Lg33Ag8Be4927R3o+3HAUcAdDeZ3PN/GypF7la8rmANcHIUbgAMlTR3pQrugad8j4rqIeDiN3kDxuYHxrupXVHwE+AawfSSL67IqfX8XcEVEbASIiBz6X6XfARwgScAEinDfObJldl5ErKHoSyMdz7exEu7TgE2l8c1p2nDbjEfD7ddCimf48a5pvyVNA/4AuGAE6xoJVe7zVwATJa2WdLOk945Ydd1Tpd9fBg6n+NDj7cBHI+LpkSlvVHU838bKLzE1/bqCim3Go8r9kvQminD/712taGRU6fcXgTMi4qniQC4bVfreA/wucDywL3C9pBsi4mfdLq6LqvT7bcCtwJuBlwErJf0wIh7tcm2jreP5NlbCvcrXFeT6lQaV+iXpt4CvASdExIMjVFs3Vel3P3BZCvbJwImSdkbEN0ekwu6p+njfERGPA49LWgO8GhjP4V6l36cC50VxInqDpHuAVwI3jUyJo6bj+TZWTstU+bqCFcB701XlY4FfRsTWkS60C5r2XdJLgSuA+eP8yK2sab8j4pCI6IuIPuBy4EMZBDtUe7wvB94gqUfSfsAxwLoRrrPTqvR7I8WrFSQdRPGNsb8Y0SpHR8fzbUwcuUeDryuQ9ME0/wKKd0ucCGwAnqB4hh/3Kvb9U8CLgfPTUezOGOffoFex31mq0veIWCfpu8BtwNPA1yKi7tvoxouK9/lngCWSbqc4VXFGRIz7rwKWdCkwC5gsaTNwDrAXdC/f/PUDZmYZGiunZczMrIMc7mZmGXK4m5llyOFuZpYhh7uZWYYc7mZmGXK4m5ll6P8Ds8gLgMacxNYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(np.mean(labels != -1, axis=1), bins=30)\n",
    "plt.title(\"Proportion of labeled items by each worker\")\n",
    "plt.savefig(\"pic/rte_proportion.png\", dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.270875\n"
     ]
    }
   ],
   "source": [
    "# total error rate\n",
    "error = np.mean(merged[\"1_y\"] != merged[2])\n",
    "print(error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged[\"error\"] = merged[\"1_y\"] != merged[2]\n",
    "error_worker = merged.groupby(['1_x']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUpUlEQVR4nO3de5BmdX3n8feHAQSBLCgNDsgwkcULWjqQcXBlV1Ew4RIFak2tJFE0xtFUMFKllbBWrYFY7FIplWziihmVBXcTXKoQZRWzIUQkhosO7oDg4HJx5OLIDBrCRWOc4bt/PGfiY9M9z+nL090/eL+qTvVzzvmdc76/Pt2fPn0uz5OqQpLUnl0WuwBJ0uwY4JLUKANckhplgEtSowxwSWqUAS5JjTLA9bST5JgkdyZ5LMmpC7TNc5L8z4XY1og6NiU5frHr0PwwwJ/iul/YH3dhtWP46GLXtTNJjk1y/xg38UfAR6tq76r63Bi3I43VrotdgBbE66vqb0Y1SrJrVW2bNG1ZVW3vu6FR7ZMESFU90XedY3AocPsibn/BTbVvl8K6NDcegT+NJXlrkr9PckGSHwLnJLk4yYVJrkryOPCaJC9Kcm2Sh5PcnuQNQ+t4UvsptnNtkvOS/D3wI+B5Sd6WZGOSR5Pck+SdXdu9gC8BBw39x3BQkl2SnJ3k7iQ/SHJZkmftpG/vSHJXkh8muTLJQd30u4HnAf+7W/czplj2oCSXJ9ma5DtJfm9o3pokN3Tfi81JPppk96H5L05ydbfdB5O8f2jVuyf5dNfn25Osnqb2c5P8Wfd6tySPJ/njbnzPJP+UZL9u/A3duh7uvs8vGlrPpiR/kORW4PEku07azgu7/r2pG//VJBu6dV2f5KV916VFUlUOT+EB2AQcP828twLbgHcz+G9sT+Bi4B+BYxj8gd8HuAt4P7A78FrgUeAF3Tomt99jiu1cC9wLvLjbzm7AycBhQIBXMwj2o7r2xwL3T1rHWcCNwHOBZwB/Dlw6Tb9eCzwEHNW1/TPgup7fk12Am4EPdP19HnAP8Cvd/F8CXtH1YyWwETirm7cPsBl4L7BHN350N+8c4J+Ak4BlwH8BbtxJ/d/sXr8SuBu4aWjeLd3r5wOPA6/rvqe/3+2r3Yf6uQE4BNhzuO/d9+Ze4Fe76UcBW4Cju/rO6No+Y7p1OSz+sOgFOIx5Bw9+8R4DHh4a3tHNeytw76T2FwOfHhr/d8D3gV2Gpl0KnDNV+2lquBb4oxFtPge8p3t9LE8O8I3AcUPjy4GfArtOsa5PAX88NL5313bl0PdkugA/eorvyX8E/vs07c8Cruhenw7832nanQP8zdD4EcCPp2m7J4OwfzZwNoM/nvd3/TgX+NOu3X8CLhtabhfgAeDYoX7+1hQ/D+d263vN0PQLgQ9Oavtt4NXTrcth8Qf/DXp6OLWmPwd+34hpBwH31c+fs/4ucPCIdex0O0lOBP6QwVHkLsAzgW/uZPlDgSuSDNexHTiQQWgNOwj4xo6RqnosyQ+6mjeNqPNQBqdvHh6atgz4u67u5wMfAVZ3Ne/K4IgdBkend+9k3d8fev0jYI+pzidX1Y+TrGfwn8mrgPOAVQz+y3k1g/8odvTzu0PLPZHkPkbvm3cBX6mqL0/q9xlJ3j00bfduGztblxaR58A11dtRDk/7HnBIkuGflRX8fGj2eUvLf2nTnXe+HPgQcGBV7QtcxeB0ynTruw84sar2HRr2qKrJ4b2j5kOHtrcXg6PZqdpOtZ3vTNrOPlV1Ujf/QuAO4PCq+gUGR8cZWvawHtvo4ysMTpccCXy9G/8VYA1wXddmcj/D4I/IqH3zLmBFkguGpt0HnDep38+sqktHrEuLyADXKDcxOM/6+90FtWOB1wOfmcM6d2dwbnorsK07Gv/lofkPAs9O8q+Gpn0cOC/JoQBJJpKcMs36/xJ4W5JV3R+L/8zgHPKmHrV9DXiku2C3Z5JlSV6S5OXd/H2AR4DHkrwQ+J2hZb8APCfJWUmekWSfJEf32OZUvgK8BfhWVf0zg9NQv83gj8vWrs1lwMlJjkuyG4Nz7z8Brh+x7keBE4BXJTm/m/YJ4F1Jjs7AXklOTrLPLOvXAjDAnx523HGxY7ii74JdeLwBOJHBhcGPAW+pqjtmW0xVPQr8HoMA+gfg14Erh+bfweA8+z3dHREHAf+1a/PXSR5lcEFzynCsqmsYnB++nMFFxcOAN/WsbTuDP1CrgO8w6PMngR1/TN7X1fsog9D7X5P69bpu+e8DdzLFXTk9Xc/gXPiOo+1vMTgvvmOcqvo28JsMTqk81G339d0+26mqerir9cQkH6yq9cA7gI8y2Cd3MbhGoiUsVf5XJEkt8ghckhplgEtSowxwSWqUAS5JjVrQB3n233//Wrly5UJuUpKad/PNNz9UVROTpy9ogK9cuZL169cv5CYlqXlJvjvVdE+hSFKjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSo/xMzEWw8uwv9mq36fyTx1yJpJZ5BC5JjRoZ4En2SPK1JLckuT3Jud30c5I8kGRDN5w0al2SpPnT5xTKT4DXVtVj3QenfjXJl7p5F1TVh8ZXniRpOiMDvAYfmvlYN7pbN/hBmpK0yHqdA0+yLMkGYAtwdVXd1M06M8mtSS5Kst80y65Nsj7J+q1bt85P1ZKkfgFeVdurahXwXGBNkpcAFwKHAauAzcCHp1l2XVWtrqrVExNPej9ySdIszegulKp6GLgWOKGqHuyC/QngE8Ca+S9PkjSdPnehTCTZt3u9J3A8cEeS5UPNTgNuG0uFkqQp9bkLZTlwSZJlDAL/sqr6QpL/kWQVgwuam4B3jq1KSdKT9LkL5VbgyCmmv3ksFUmSevFJTElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNWpkgCfZI8nXktyS5PYk53bTn5Xk6iR3dl/3G3+5kqQd+hyB/wR4bVW9DFgFnJDkFcDZwDVVdThwTTcuSVogIwO8Bh7rRnfrhgJOAS7ppl8CnDqOAiVJU+t1DjzJsiQbgC3A1VV1E3BgVW0G6L4eMM2ya5OsT7J+69at81S2JKlXgFfV9qpaBTwXWJPkJX03UFXrqmp1Va2emJiYZZmSpMlmdBdKVT0MXAucADyYZDlA93XLfBcnSZpen7tQJpLs273eEzgeuAO4Ejija3YG8Pkx1ShJmsKuPdosBy5JsoxB4F9WVV9IcgNwWZK3A/cCvzbGOiVJk4wM8Kq6FThyiuk/AI4bR1GSpNH6HIFriVt59hd7tdt0/sljrkTSQvJReklqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRIwM8ySFJvpxkY5Lbk7ynm35OkgeSbOiGk8ZfriRphz4farwNeG9VfSPJPsDNSa7u5l1QVR8aX3mSpOmMDPCq2gxs7l4/mmQjcPC4C5Mk7dyMzoEnWQkcCdzUTTozya1JLkqy3zTLrE2yPsn6rVu3zq1aSdK/6B3gSfYGLgfOqqpHgAuBw4BVDI7QPzzVclW1rqpWV9XqiYmJuVcsSQJ6BniS3RiE919U1WcBqurBqtpeVU8AnwDWjK9MSdJkfe5CCfApYGNVfWRo+vKhZqcBt81/eZKk6fS5C+UY4M3AN5Ns6Ka9Hzg9ySqggE3AO8dQnyRpGn3uQvkqkClmXTX/5UiS+vJJTElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNWpkgCc5JMmXk2xMcnuS93TTn5Xk6iR3dl/3G3+5kqQd+hyBbwPeW1UvAl4B/G6SI4CzgWuq6nDgmm5ckrRARgZ4VW2uqm90rx8FNgIHA6cAl3TNLgFOHVONkqQpzOgceJKVwJHATcCBVbUZBiEPHDDNMmuTrE+yfuvWrXMsV5K0Q+8AT7I3cDlwVlU90ne5qlpXVauravXExMRsapQkTaFXgCfZjUF4/0VVfbab/GCS5d385cCW8ZQoSZpKn7tQAnwK2FhVHxmadSVwRvf6DODz81+eJGk6u/ZocwzwZuCbSTZ0094PnA9cluTtwL3Ar42lQknSlEYGeFV9Fcg0s4+b33IkSX35JKYkNcoAl6RGGeCS1CgDXJIaZYBLUqP63Eb4lLTy7C/2brvp/JPHWIkkzY5H4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqOetu+FovnR9z1lfD8Zaf55BC5JjerzqfQXJdmS5LahaeckeSDJhm44abxlSpIm63MEfjFwwhTTL6iqVd1w1fyWJUkaZWSAV9V1wA8XoBZJ0gzM5Rz4mUlu7U6x7DdvFUmSepltgF8IHAasAjYDH56uYZK1SdYnWb9169ZZbk6SNNmsAryqHqyq7VX1BPAJYM1O2q6rqtVVtXpiYmK2dUqSJplVgCdZPjR6GnDbdG0lSeMx8kGeJJcCxwL7J7kf+EPg2CSrgAI2Ae8cX4mSpKmMDPCqOn2KyZ8aQy2SpBnwSUxJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUqGY+0MEPDpCkn+cRuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqNGBniSi5JsSXLb0LRnJbk6yZ3d1/3GW6YkabI+R+AXAydMmnY2cE1VHQ5c041LkhbQyACvquuAH06afApwSff6EuDU+S1LkjTKbD/Q4cCq2gxQVZuTHDBdwyRrgbUAK1asmOXmpPHyA0PUorFfxKyqdVW1uqpWT0xMjHtzkvS0MdsAfzDJcoDu65b5K0mS1MdsA/xK4Izu9RnA5+enHElSX31uI7wUuAF4QZL7k7wdOB94XZI7gdd145KkBTTyImZVnT7NrOPmuRZJ0gz4JKYkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGjXbT+SRxsJPxpH68whckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGzek+8CSbgEeB7cC2qlo9H0VJkkabjwd5XlNVD83DeiRJM+ApFElq1FyPwAv46yQF/HlVrZvcIMlaYC3AihUr5rg5SaP4dgRPH3M9Aj+mqo4CTgR+N8mrJjeoqnVVtbqqVk9MTMxxc5KkHeYU4FX1ve7rFuAKYM18FCVJGm3WAZ5kryT77HgN/DJw23wVJknaubmcAz8QuCLJjvX8ZVX91bxUJUkaadYBXlX3AC+bx1okSTPgBzr00PeqviQtJO8Dl6RGGeCS1CgDXJIaZYBLUqMMcElqlHehPI14N83C8j1J5s7v4c55BC5JjTLAJalRBrgkNcoAl6RGeRFzCfOio8ZpMS8Q+rM9PzwCl6RGGeCS1CgDXJIaZYBLUqMMcElqlHehaEHM910HPmK9ND3d7i6ZSX/H8bPoEbgkNcoAl6RGzSnAk5yQ5NtJ7kpy9nwVJUkabdYBnmQZ8N+AE4EjgNOTHDFfhUmSdm4uR+BrgLuq6p6q+mfgM8Ap81OWJGmUVNXsFkzeCJxQVb/djb8ZOLqqzpzUbi2wtht9AfDtWda6P/DQLJddauzL0vNU6QfYl6VqLn05tKomJk+cy22EmWLak/4aVNU6YN0ctjPYWLK+qlbPdT1LgX1Zep4q/QD7slSNoy9zOYVyP3DI0Phzge/NrRxJUl9zCfCvA4cn+cUkuwNvAq6cn7IkSaPM+hRKVW1Lcibwf4BlwEVVdfu8VfZkcz4Ns4TYl6XnqdIPsC9L1bz3ZdYXMSVJi8snMSWpUQa4JDVqyQX4qMfzM/Cn3fxbkxy1GHX20aMvL0xyQ5KfJHnfYtTYR49+/Ea3L25Ncn2Sly1GnX306MspXT82JFmf5N8uRp199H0riyQvT7K9e3ZjyemxT45N8o/dPtmQ5AOLUWcfffZJ158NSW5P8pU5bbCqlszA4GLo3cDzgN2BW4AjJrU5CfgSg/vQXwHctNh1z6EvBwAvB84D3rfYNc+hH68E9uten9j4Ptmbn10beilwx2LXPdu+DLX7W+Aq4I2LXfcs98mxwBcWu9Z56su+wLeAFd34AXPZ5lI7Au/zeP4pwKdr4EZg3yTLF7rQHkb2paq2VNXXgZ8uRoE99enH9VX1D93ojQyeCViK+vTlsep+s4C9mOLhtCWi71tZvBu4HNiykMXNwFPpLTn69OXXgc9W1b0wyIC5bHCpBfjBwH1D4/d302baZilopc5RZtqPtzP4D2kp6tWXJKcluQP4IvBbC1TbTI3sS5KDgdOAjy9gXTPV9+fr3yS5JcmXkrx4YUqbsT59eT6wX5Jrk9yc5C1z2eBS+0SePo/n93qEfwlopc5RevcjyWsYBPhSPW/c9+0frgCuSPIq4IPA8eMubBb69OVPgD+oqu3JVM2XhD79+AaD9wJ5LMlJwOeAw8dd2Cz06cuuwC8BxwF7AjckubGq/t9sNrjUArzP4/mtPMLfSp2j9OpHkpcCnwROrKofLFBtMzWjfVJV1yU5LMn+VbXU3lCpT19WA5/pwnt/4KQk26rqcwtSYT8j+1FVjwy9virJxxreJ/cDD1XV48DjSa4DXgbMKsAX/cT/pBP8uwL3AL/Izy4CvHhSm5P5+YuYX1vsumfbl6G257B0L2L22ScrgLuAVy52vfPQl3/Nzy5iHgU8sGN8KQ0z+fnq2l/M0ryI2WefPGdon6wB7m11nwAvAq7p2j4TuA14yWy3uaSOwGuax/OTvKub/3EGV9NPYhAYPwLetlj17kyfviR5DrAe+AXgiSRnMbhq/ch0611oPffJB4BnAx/rjva21RJ8B7meffn3wFuS/BT4MfAfqvvNW0p69mXJ69mPNwK/k2Qbg33yplb3SVVtTPJXwK3AE8Anq+q22W7TR+klqVFL7S4USVJPBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElq1P8HjyqeSTBoLosAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(error_worker[\"error\"], bins=30)\n",
    "plt.title(\"Error rate of each worker\")\n",
    "plt.savefig(\"pic/rte_error.png\", dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_item = merged.groupby([0]).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVkklEQVR4nO3de5RlZX3m8e9jI3gBA0rBcC8gYMSs2JoKuMYhg5KE20RwlmYgLkTD2Dqjiaw4awSyRlkmzDAZL5MZI6QVBlhRLgmiOGJGwkTJRTSNIheBcGuhpW0KvADCkOnmN3+cXcPp9lTXqTp1qrpfv5+19qqz3/3uvX/1dvVTu/bZZ+9UFZKktjxnuQuQJC0+w12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuzSEJK9JcneSJ5KctET7PCfJn86j/xNJDhpnTdp+GO76CUnWJnmqC4uZ6WPLXdfWJDkqybox7uKDwMeqaueq+uwY97NgXW33ASS5OMkfLHdNWj47LHcB2mb9elX95VydkuxQVRu3aFtRVZuG3dFc/ZMESFU9M+w2x+AA4PZl3L80Lx65a16SvDXJ3yb5aJLvA+d0R4nnJ7k2yY+B1yZ5WZIvJ/lhktuTvL5vGz/Rf8B+vpzk3CR/CzwJHJTkbUnuSPJ4kvuSvKPr+0Lgi8DefX9p7J3kOUnOTHJvkkeTXJnkxVv53t6e5J4k309yTZK9u/Z7gYOAz3fb3mnAunsnuSrJdJL7k/xO37LDk3y1G4v1ST6WZMe+5S9Pcl233w1Jzu7b9I5JLu2+59uTTG2l/krys0lWAW8G/n1X7+eHqPGcJH+W5E+7fd2a5NAkZyV5OMmDSX5ttn1rG1RVTk6bTcBa4FdmWfZWYCPw2/T+8ns+cDHwI+A19A4YdgHuAc4GdgReBzwOvLTbxpb9nzdgP18GHgBe3u3nucAJwMFAgH9OL/Rf1fU/Cli3xTbOAG4E9gV2Av4EuGyW7+t1wCPAq7q+/x24YcgxeQ5wE/D+7vs9CLgPOKZb/ovAq7vvYxK4AzijW7YLsB54L/C8bv6Ibtk5wP8BjgdWAP8JuHEr/24F/GzfGP/BPGqc2dcxXZ2XAvcDv9eN/duB+5f7Z9Np+GnZC3Da9qYuyJ4Aftg3vb1b9lbggS36Xwxc2jd/JPA94Dl9bZcB5wzqP0sNXwY+OEefzwLv6V4PCvc7gKP75vcC/i+ww4BtXQj8Yd/8zl3fyb4xmS3cjxgwJmcB/2OW/mcAV3evTwG+OUu/c4C/7Js/DHhqK+OxtXDfao3dvq7rW/br3c/Aim5+l277uy73z6fTcJPn3DWbk2r2c+4PztG2N/BgbX6O/DvAPnNsY6v7SXIc8AHgUHpHoi8Abt3K+gcAVyfpr2MTsCfw3S367g18Y2amqp5I8mhX89o56jyA3imhH/a1rQD+uqv7UOAjwFRX8w70jqIB9gPu3cq2v9f3+kngeYPe5xjCVmvsbOh7/RTwSD37XshT3ded6f2y1zbOc+5aiEG3Eu1vewjYL0n/z9f+bB6ow9yO9P/36c5zXwV8CNizqnYFrqV3ima27T0IHFdVu/ZNz6uqLYN9puYD+vb3QuAl/OQvgUEepHfKon8/u1TV8d3y84E7gUOq6kX0Tlelb92Dh9jHfG05HnPVqMYY7hqHrwE/pveG3nOTHEXvz/zLR9jmjvTOhU8DG7uj+P43+DYAL0nyM31tFwDnJjkAIMlEkhNn2f6ngbclWdn9IvmPwNeqau0QtX0deCzJ+5I8P8mKJD+f5Je65bsAjwFPJPk54N/0rfs/gX+S5IwkOyXZJckRQ+xzLhvonVcftkY1xnDXbGauDJmZrh52xar6R+D1wHH03qT8OPCWqrpzocVU1ePA7wBXAj8AfhO4pm/5nfTO69/XXZWyN/BHXZ8vJXmc3purA4Ozqq4H/gO9vw7W0zuaPnnI2jbR++W1kt6bkI8AnwRmftH8u67ex4FPAFds8X39arf+94C7GXD10AJcCBzWjcVnh6hRjUmVD+uQpNZ45C5JDTLcJalBhrskNchwl6QGbRMfYtp9991rcnJyucuQpO3KTTfd9EhVTQxatk2E++TkJGvWrFnuMiRpu5LkO7Mtm/O0TJL9kvxVdze+25O8p2t/cXcnu7u7r7v1rXNWd3e9u5IcszjfhiRpWMOcc98IvLeqXkbvznbvSnIYcCZwfVUdAlzfzdMtO5ne3fyOBT6eZMU4ipckDTZnuFfV+qr6Rvf6cXp32tsHOBG4pOt2CXBS9/pE4PKqerqq7qd369fDF7luSdJWzOtqmSSTwCvp3Ttkz6paD71fAMAeXbd92PxufuvY/G6AM9talWRNkjXT09MLKF2SNJuhwz3JzvTuu3FGVT22ta4D2n7iHgdVtbqqpqpqamJi4Ju9kqQFGirckzyXXrB/qqo+0zVvSLJXt3wv4OGufR29e1TP2Jfe7VQlSUtkmKtlQu8Oc3dU1Uf6Fl0DnNa9Pg34XF/7yd3tSw8EDqF3u1FJ0hIZ5jr31wCnArcmublrOxs4D7gyyen0nnX5JoCquj3JlcC36V1p867aypPtJUmLb85wr6q/YfB5dICjZ1nnXODcEeqSJI1gm/iEquZn8swvDN137XknjLESSdsqbxwmSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDRrmAdkXJXk4yW19bVckubmb1s48WzXJZJKn+pZdMMbaJUmzGOYxexcDHwMunWmoqn818zrJh4Ef9fW/t6pWLlJ9kqQFGOYB2TckmRy0LEmA3wBet8h1SZJGMOo59yOBDVV1d1/bgUm+meQrSY6cbcUkq5KsSbJmenp6xDIkSf1GDfdTgMv65tcD+1fVK4HfBT6d5EWDVqyq1VU1VVVTExMTI5YhSeq34HBPsgPwL4ErZtqq6umqerR7fRNwL3DoqEVKkuZnlCP3XwHurKp1Mw1JJpKs6F4fBBwC3DdaiZKk+RrmUsjLgK8CL02yLsnp3aKT2fyUDMAvA7ck+Rbw58A7q+r7i1mwJGluw1wtc8os7W8d0HYVcNXoZUmSRuEnVCWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBwzxm76IkDye5ra/tnCTfTXJzNx3ft+ysJPckuSvJMeMqXJI0u2GO3C8Gjh3Q/tGqWtlN1wIkOYzes1Vf3q3z8ZkHZkuSls6c4V5VNwDDPuT6RODyqnq6qu4H7gEOH6E+SdICjHLO/d1JbulO2+zWte0DPNjXZ13XJklaQjsscL3zgd8Hqvv6YeC3gAzoW4M2kGQVsApg//33X2AZWmqTZ35hqH5rzzthzJVI2poFHblX1Yaq2lRVzwCf4NlTL+uA/fq67gs8NMs2VlfVVFVNTUxMLKQMSdIsFhTuSfbqm30DMHMlzTXAyUl2SnIgcAjw9dFKlCTN15ynZZJcBhwF7J5kHfAB4KgkK+mdclkLvAOgqm5PciXwbWAj8K6q2jSWyiVJs5oz3KvqlAHNF26l/7nAuaMUJUkajZ9QlaQGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBi30rpDSNmnYu1aCd65U2zxyl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDVoznBPclGSh5Pc1tf2X5LcmeSWJFcn2bVrn0zyVJKbu+mCMdYuSZrFMEfuFwPHbtF2HfDzVfULwD8AZ/Utu7eqVnbTOxenTEnSfAzzgOwbkkxu0falvtkbgTcucl3bhWE/6u7H3CUttcU45/5bwBf75g9M8s0kX0ly5GwrJVmVZE2SNdPT04tQhiRpxkjhnuT3gI3Ap7qm9cD+VfVK4HeBTyd50aB1q2p1VU1V1dTExMQoZUiStrDgcE9yGvAvgDdXVQFU1dNV9Wj3+ibgXuDQxShUkjS8BYV7kmOB9wGvr6on+9onkqzoXh8EHALctxiFSpKGN+cbqkkuA44Cdk+yDvgAvatjdgKuSwJwY3dlzC8DH0yyEdgEvLOqvj+m2iVJsxjmaplTBjRfOEvfq4CrRi1KkjQaP6EqSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDZoz3JNclOThJLf1tb04yXVJ7u6+7ta37Kwk9yS5K8kx4ypckjS7YY7cLwaO3aLtTOD6qjoEuL6bJ8lhwMnAy7t1Pj7zwGxJ0tKZM9yr6gZgy4dcnwhc0r2+BDipr/3yqnq6qu4H7gEOX5xSJUnDWug59z2raj1A93WPrn0f4MG+fuu6NknSElrsN1QzoK0GdkxWJVmTZM309PQilyFJP90WGu4bkuwF0H19uGtfB+zX129f4KFBG6iq1VU1VVVTExMTCyxDkjTIQsP9GuC07vVpwOf62k9OslOSA4FDgK+PVqIkab52mKtDksuAo4Ddk6wDPgCcB1yZ5HTgAeBNAFV1e5IrgW8DG4F3VdWmMdUuSZrFnOFeVafMsujoWfqfC5w7SlGSpNH4CVVJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ2a8zF7s0nyUuCKvqaDgPcDuwJvB6a79rOr6tqF7mcYk2d+Yah+a887YZxlSNI2Y8HhXlV3ASsBkqwAvgtcDbwN+GhVfWgxCpQkzd9inZY5Gri3qr6zSNuTJI1gscL9ZOCyvvl3J7klyUVJdhu0QpJVSdYkWTM9PT2oiyRpgUYO9yQ7Aq8H/qxrOh84mN4pm/XAhwetV1Wrq2qqqqYmJiZGLUOS1GcxjtyPA75RVRsAqmpDVW2qqmeATwCHL8I+JEnzsBjhfgp9p2SS7NW37A3AbYuwD0nSPCz4ahmAJC8AfhV4R1/zHyZZCRSwdotlkqQlMFK4V9WTwEu2aDt1pIokSSPzE6qS1CDDXZIaZLhLUoMMd0lqkOEuSQ0a6WoZSc/y7qTalnjkLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGjfoM1bXA48AmYGNVTSV5MXAFMEnvGaq/UVU/GK1MSdJ8LMaR+2uramVVTXXzZwLXV9UhwPXdvCRpCY3jtMyJwCXd60uAk8awD0nSVowa7gV8KclNSVZ1bXtW1XqA7useg1ZMsirJmiRrpqenRyxDktRv1Id1vKaqHkqyB3BdkjuHXbGqVgOrAaampmrEOiRJfUY6cq+qh7qvDwNXA4cDG5LsBdB9fXjUIiVJ87PgcE/ywiS7zLwGfg24DbgGOK3rdhrwuVGLlCTNzyinZfYErk4ys51PV9VfJPl74MokpwMPAG8avUxJ0nwsONyr6j7gFQPaHwWOHqUoSdJo/ISqJDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBoz4gW9IymzzzC0P1W3veCWOuRNuSUZ6hul+Sv0pyR5Lbk7ynaz8nyXeT3NxNxy9euZKkYYxy5L4ReG9VfaN7UPZNSa7rln20qj40enmSpIUY5Rmq64H13evHk9wB7LNYhUmSFm5R3lBNMgm8Evha1/TuJLckuSjJbrOssyrJmiRrpqenF6MMSVJn5HBPsjNwFXBGVT0GnA8cDKykd2T/4UHrVdXqqpqqqqmJiYlRy5Ak9Rkp3JM8l16wf6qqPgNQVRuqalNVPQN8Ajh89DIlSfMxytUyAS4E7qiqj/S179XX7Q3AbQsvT5K0EKNcLfMa4FTg1iQ3d21nA6ckWQkUsBZ4xwj7kCQtwChXy/wNkAGLrl14OZKkxeDtBySpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDvJ+7pLHznvNLzyN3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ3y9gOSNAbLfcuFsYV7kmOBPwJWAJ+sqvPGtS9JGmS5A3Y5jeW0TJIVwB8DxwGH0Xto9mHj2Jck6SeN65z74cA9VXVfVf0jcDlw4pj2JUnaQqpq8TeavBE4tqr+dTd/KnBEVb27r88qYFU3+1LgrhF2uTvwyAjrt8Sx2Jzj8SzHYnMtjMcBVTUxaMG4zrlnQNtmv0WqajWwelF2lqypqqnF2Nb2zrHYnOPxLMdic62Px7hOy6wD9uub3xd4aEz7kiRtYVzh/vfAIUkOTLIjcDJwzZj2JUnawlhOy1TVxiTvBv4XvUshL6qq28exr86inN5phGOxOcfjWY7F5poej7G8oSpJWl7efkCSGmS4S1KDtptwT3JskruS3JPkzAHLk+S/dctvSfKq5ahzqQwxHm/uxuGWJH+X5BXLUedSmGss+vr9UpJN3ecwmjXMeCQ5KsnNSW5P8pWlrnEpDfF/5WeSfD7Jt7rxeNty1Lnoqmqbn+i9KXsvcBCwI/At4LAt+hwPfJHeNfavBr623HUv83j8U2C37vVxrY7HMGPR1+9/A9cCb1zuupf5Z2NX4NvA/t38Hstd9zKPx9nAf+5eTwDfB3Zc7tpHnbaXI/dhbmdwInBp9dwI7Jpkr6UudInMOR5V9XdV9YNu9kZ6nzVo0bC3uvht4Crg4aUsbhkMMx6/CXymqh4AqKqWx2SY8ShglyQBdqYX7huXtszFt72E+z7Ag33z67q2+fZpxXy/19Pp/VXTojnHIsk+wBuAC5awruUyzM/GocBuSb6c5KYkb1my6pbeMOPxMeBl9D5oeSvwnqp6ZmnKG5/t5X7uc97OYMg+rRj6e03yWnrh/s/GWtHyGWYs/ivwvqra1Ds4a9ow47ED8IvA0cDzga8mubGq/mHcxS2DYcbjGOBm4HXAwcB1Sf66qh4bc21jtb2E+zC3M/hpuuXBUN9rkl8APgkcV1WPLlFtS22YsZgCLu+CfXfg+CQbq+qzS1Lh0hr2/8ojVfVj4MdJbgBeAbQY7sOMx9uA86p30v2eJPcDPwd8fWlKHI/t5bTMMLczuAZ4S3fVzKuBH1XV+qUudInMOR5J9gc+A5za6BHZjDnHoqoOrKrJqpoE/hz4t40GOwz3f+VzwJFJdkjyAuAI4I4lrnOpDDMeD9D7K4Yke9K7S+19S1rlGGwXR+41y+0MkryzW34BvasgjgfuAZ6k99u4SUOOx/uBlwAf745YN1aDd8Abcix+agwzHlV1R5K/AG4BnqH3pLTblq/q8Rny5+P3gYuT3ErvNM77qmp7vxWwtx+QpBZtL6dlJEnzYLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBv0/OzYFWT9wRf0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(error_item[\"error\"], bins=30)\n",
    "plt.title(\"Error rate of each item\")\n",
    "plt.savefig(\"pic/rte_error_item.png\", dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2626, 1374],\n",
       "       [ 793, 3207]], dtype=int64)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(merged[\"1_y\"], merged[2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAADaCAYAAACRvxJJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABIsklEQVR4nO3debweZX3//9fHICCbJLLlhCCYBBRRjxoh1NaCigepira1QF2w6i+NX7QatQrSVrpQqVqjrWhEpeLGIoJQi0ZALbUaBTQKiEjY44kgglWwZUk+vz9mJlznymz3fc+9zH3ez/OYx7nvmWtmrrlmueeaaxlzd0RERERERKTYo4YdARERERERkVGnjJOIiIiIiEgFZZxEREREREQqKOMkIiIiIiJSQRknERERERGRCso4iYiIiIiIVFDGSUQGwsxuNbPn1wzrZra4y/V0PW+3zOw6Mzush/m/YmbHNxcjERERaZoyTiJSSycZn9nG3Z/s7t/sYf4XuvtZDUapZ2b2p2Z2m5ndb2ZfMrN5JWF/x8y+Z2a/MbMfmdnvBtMON7NrzOxXZvZLM7vQzBYE0xeY2UVmdo+ZbTCzFf3etlFgiX9K0+SXZvZeM7OCsMvM7NI0jX5hZl8ws/nB9K+Y2X3B8KCZXRNM39fMvmFmvzWzn4TncRqPk83sdjP7tZmdY2a79HfrRUTaSRknESllZtsMOw4yWGb2ZOBjwKuAPYHfAh8pCDsPuBh4H7Ar8F7g381sbhrkx8CUu+8KTAA3Ah8NFvFZ4JZ0PX8A/KOZHd7sFuUfx50e2w2fC8uBlwJPA54KvAj484Kwc4EzgH2BxwO/Af4tm5hmvHfKBuDbwBeC+c8GfgA8DjgZON/Mdk+nvZpkPz+bZP88BvjX3jdPRGT8KOMkIltJS5feaWY/Au43s7OBfUhuiO8zs3ek4ZaZ2bfT0oQf1q2uZmYHm9l30vk2mtmHzWzbKNhRZnazmd1tZu8zs0cF87/WzK43s3vNbI2ZPb5gPUeZ2Y/TkpCfmdnba8bvU2b2keBJ/n+b2V5m9sF0nT8xs6dH6fX8YNuuSp/e32lmH0jHb29mn01LF35lZlea2Z7ptG+a2evTz68xs2+Z2fvTdd1iZi8M1rWfmV2RbtNlZna6mX22znZ14BXAv7v7Fe5+H/DXwB+a2c45YX8HuNPdv+Dum9z9s8AvgD8EcPc73X06CL8JWJxuy07AYcCp7v6Qu/8QOB94bZ1ImtmjzOxEM7spTdfz0oxcVsriZvY6M7sd+Hqatv9tZqvM7B7gFDN7rJl9Oi3Juc3M/io71vLCd5aMpY4H/tndN7j7z4B/Bl6TF9Ddv5Km76/d/bfAh0kyOnlpsi/we8Bn0u/7A88A3u3u/+vuXwSuAf4oneXFwCfd/Y50X/8TcIyZ7dDQdoqIjA1lnESkyHEkJQC7uvtxwO3Ai9On2u+1pLrVfwD/AMwD3g58MXiSXWYTsBLYDTgUeB7w/6IwLwOWktz0HU16M21mLwXeRXJjvjvwXyRP1PN8Evhzd98ZOAj4eo24Zf4E+Ks0jg8A3wG+n34/H/hAwXwfAj7k7rsAi4Dz0vHHA48FFpI8+V8B/G/BMg4BbkjX9V7gk2ZbqnF9HvheuoxTSEoLcpnZPmkmrWj404JZnwz8MPvi7jcBDwL7560mHeJxB8XxSLf37ek2EcxnRfNW+AuSUpvfJyktuRc4PQrz+8CTgKn0+yHAzcAewKkkpSuPBZ6Qhn018GfB/HH4GSyp0liWxvsUxH1GGqefn1xrq+E5wHUF014N/Je73xKs52Z3/03BuuL9Z8B2wJKacRERmTWUcRKRIv+SPoUuurl/JXCJu1/i7pvd/VLgKuCoqgW7+9XuvtbdH3b3W0mqhf1+FOyf3P0ed78d+CBJRg6S6kzvcffr3f1h4B+ByYJSp4eAA81sF3e/192/XxW3wIVpPP8PuBD4P3f/tLtvAs4Fnl4w30PAYjPbzd3vc/e1wfjHAYvTkpmr3f3XBcu4zd0/nq7rLGA+sGd6E/4s4G/c/UF3/xZJNblc7n67u+9aMny+YNadgP+Jxv0PkFfi9G1gwsyOM7NHW9LJxSJgS4lFFg+SjOBfAT9Jx/8G+G/gr9MSuWeQlITULe34c+DktNTmAZKM5B/bzCp1p7j7/cFxPO3u/5oeOw8CxwAnuftv0mPxn5mZGd0SPu9ccPfPV6Tx7QVxj9P4f4CdggxyLjN7KvA3wF8WBHk18KmS9WTryvblV4DXpyV0jwXemY5XiZOISEQZJxEpckfF9McDLw+frgO/S3KTX8rM9jezL5vZz83s1ySZn91K1n8bSYlCtt4PBeu8h+Qp+QK29kckGbnbzOw/zezQqrgF7gw+/2/O950K5nsdScnMT9LqeC9Kx38GWAOcY2bTlnQG8OiCZfw8+5BWzSJd3wRwTzAOqvdTN+4D4g4CdiFpWzODu/+SpETwrSRpdCRwGbAhJ+w9JBnBi4LMzSuA/Ui246PA5/LmLfB44MLgWLiepDRzzyBMnD7h992AbUmOr8xtzDyW+pG+sHUa7wLc5+5eNIMlvUV+BXizu/9XzvTfBfYiKREtWk+2rmxfnklSYvtNklKsb6Tj6+4DEZFZQxknESkS38DF3+8APhM9Xd/R3U+rseyPkpQ6LEmrtL2Lrat7LQw+7wNk7WTuIKl+F673Me7+7a02wP1Kdz+apJrVl3ik2lzfuPuNadXGPUjai5xvZjumbXj+1t0PJGkX9CKS0oFObATmRe1PFhYFTqvI3VcyvKJg1utIOi3IlvMEkupbP80L7O7/6e7Pcvd5JKU1B5BUJ8yzDUna7JLOe5u7v8jdd3f3Q0hK5Yrmjd0BvDA6FrZP2wxtiV4c3eDz3SQlgWFp5T5A2fwzmNkrKtK4qKrejDROPxdVvyMtUb0M+Ht3/0xBsOOBC9K2SuF6nhC1T9uyrrS0+N3uvq+7752O/xkz00BERFDGSUTqu5OkHUjms8CLzWzKzOakVa0OM7O9ayxrZ+DXwH1m9kTgDTlh/tLM5prZQuDNJNXjAFYDJ1nS8xtp4/6XxzOb2bbpTe1j3f2hdH2bguluPbx7qYiZvdLMdnf3zcCv0tGbLOmW+ylmNieNy0NhfOpw99tIqkOekm7foSSN+4vC3+5Bb2s5w+cKZv0cyb79PTPbEfg7khvyrUqc0m1+elpNbxfg/cAGd1+TTvtDMzvAko4cdidpG/aDtPQJM3uSme2cbs8rgRcQtB+zpOON1xTEczVwalZN08x2N7Oji9IjJ302kWSmT03j8HiSkrPanW24++cq0rioqt6ngbda0h37BPA2Zlax28KS9oRfB05399UFYR4DvDxehrv/FFgHvDs9R19G0ovfF9P55pnZIkscSJL2f5cevyIiElDGSUTqeg/wV2m1qLe7+x0kVbTeRdKL2h0k7S7qXFfeDvwpSXWhj/NIpih0EXA1yU3ff5B09IC7X0hSknNOWs3vWuCFOfNDUvpxaxpuBUm7LNLM3X0kvYs17UjgOjO7j6SjiGPTdlJZFapfk1Qp+086uEEPvIKkQ41fknTMcS5J5xWNcffrSNLrc8BdJBndLZ13mNlqMwtv4N9BUnpzB0lVzZcF0xYAXyXZ19cAm6PpUySdL9ybrvNId/9Fup5tSUqg1pLvQyRtvL5mZr9Jwx3S4ea+Cbg/jcO3SDrfOLPDZXTjY8C/k6TJtSTH+MeyiZa8VDkrEXw9yUOLd4elWdHyXkrSdukbbO1Yko5W7gVOA/44S2OS6oqXkKTBV4Az3f2M3jdPRGT8WEl1ahGRsZSWbDzZ3U8adlx6ZWbnAj9x93cPOy5NS9vsnJBWfRQRERkqZZxERFrEzJ5F0iHGLSTV2r4EHOruPxhmvERERJpkZmeStAe+y923ek2FmRlJzYOjSF7U/hpPe881syPTaXOAT9Rsf11JVfVERNplL5Ie0O4D/gV4gzJNIiIyhj5FUv29yAtJ3jm3BFhO0vEUaVvi09PpBwLHpW04e7ZNdRARERkV7v7vJG1jRERExpa7X2Fm+5YEORr4dPoah7VmtquZzQf2Bda7+80AZnZOGvbHvcZJJU4iIiIiItI2C5j5rr0N6bii8T1TiZOIiIiIiDRm6vAd/Zf3lL9x4+ofPXAd8H/BqDM67NUzfv8jJO/eKxrfs6GXOJnZkWZ2g5mtN7MThx2fNjOzM83sLjO7Nhg3z8wuNbMb0/9zg2knpel+g5lNDSfW7WJmC83sG2Z2fdpd8JvT8UrnhqTvmvmemf0wTeO/TccrjRtmyfu3fmBmX06/K40blL6D6hozW2dmV6XjlMYNS6vnnG9mP0mvzYcqnZtjyXvY1gXDr83sLUpjKXP3PZtYu2bv0gH4P3dfGgydvgphAzNfAr83MF0yvmdDzTj1s/HWLPUptm5EdyJwubsvAS5Pv5Om87HAk9N5PpLuDyn3MPA2d38SsAw4IU1LpXNzHgCe6+5PAyaBI81sGUrjfngzyTulMkrj5h3u7pPuvjT9rjRu3oeAr7r7E4GnkRzTSueGuPsN6TE8CTyTpPeyC1EaSwnHecg3lQ4NuBh4tSWWAf/j7huBK4ElZrafJe8DPDYN27NhlzgdTNp4y90fBLLGW9IFd7+CpJvi0NHAWenns0hekpiNP8fdH3D3W4D1JPtDSrj7xqyrS3f/DckP9AKUzo3xRPZyz0eng6M0bpQlLwH+A+ATwWilcf8pjRtkZrsAz+GRF2Q/6O6/QuncL88DbnL321AaS4XNFX9VzOxs4DvAAWa2wcxeZ2YrzGxFGuQSkpeXrwc+Tvqidnd/GHgjsIbkPu289MXuPRt2G6e8xludvvVdyu2Z5r5x941mtkc6fgGwNgjXWMO52cKSnl6eDnwXpXOj0qeTVwOLgdPd/btmpjRu1geBdwA7B+OUxs1y4Gtm5sDH0mooSuNmPQH4BfBvZvY0kuvGm1E698uxwNnpZ6WxFHKcTT2+K7bq5edpb3onFEy7hCRj1ahhlzj1rfGWVFLa98DMdgK+CLzF3X9dFjRnnNK5grtvSquF7A0cbGZbvfguoDTukJllLxS8uu4sOeOUxtWe7e7PIKmOfoKZPackrNK4O9sAzwA+6u5PB+4nrTJWQOncpbTK00uAL1QFzRmnNJ5lHHiIzaVDGw0749S3xluyxZ2W9GlP+v+udLzSvktm9miSTNPn3P2CdLTSuQ/SKjffJKknrzRuzrOBl5jZrSRVpJ9rZp9Fadwod59O/99F0ibkYJTGTdsAbHD376bfzyfJSCmdm/dC4Pvufmf6XWkspTbjpUMbDTvj1LfGW7LFxcDx6efjgYuC8cea2XZmth/JW5e/N4T4tYqZGUld+uvd/QPBJKVzQ8xsdzPbNf38GOD5wE9QGjfG3U9y973dfV+S6+7X3f2VKI0bY2Y7mtnO2WfgBcC1KI0b5e4/B+4wswPSUc8jecml0rl5x/FINT1QGksJBx5yLx3aaKhtnNz9YTPLGm/NAc5sqvHWbJQ2ojsM2M3MNgDvBk4DzjOz1wG3Ay8HcPfrzOw8kh+Yh4ET3Jvp4mTMPRt4FXCNma1Lx70LpXOT5gNnpe2cHkXSqPPLZvYdlMb9puO4OXsCFybPWtgG+Ly7f9XMrkRp3LQ3AZ9LH8DeDPwZ6bVD6dwMM9sBOAL482C0rhdSyHE2tbRUqYx5S3N8IiIiIiIyep761Ef7xZfsVhpmv4U/vzp4VUMrDLtXPRERERERGSOO8VBuPyHtpoyTiIiIiIg0xoHNY1ipTRknERERERFpjAMPDr0PuuYp4yQiIiIiIo3a7KqqJyIiIiIiUsiBTWPYxqlvZWhmdqSZ3WBm682s7C3eWfjl/YqLJJTG/ac0Hgylc/8pjftPadx/SuPBUDpLzDEe8jmlQxv1JeOUvn/ldJK3TB8IHGdmB1bMppOu/5TG/ac0Hgylc/8pjftPadx/SuPBUDrLDFmJU9nQRv2qqncwsN7dbwYws3OAo0lehiYiIiIiImMqKXEavxZB/dqiBcAdwfcNwCFFgbe17XzO3Llst89C3+6O+wF4YOGOPGXuL7jm3t3Z7o77eWDhjgBbPmfTMtl8oQcW7pg7PpwWh8nWEy83XH+VOK7hsvZ/6m+3xLtq/fH8cXzC+fZ/6m/56Y922LL8ON4Ac+bOZRebt1XnkGGYMM3zpsdxCPfDU+b+gp/+aIcZaVuUDnE65a0nDt+Lqv0Xpmf8ueg4zJOlcd5+zNufZcsKZfs3L3554v1yzb27z/gPzDh/4rgU7fNOlO3zbFze8V9nXfss2Gar60WVeLlF14DsHIqvMZ0oO86y6XnnXbjOTq9dRfKuOeF6YetjYbs77md7dthyLBetJzwui8aF16Y4bNF88fmSxRPYall1l1s1T3auh+vIPsfzZNNCeXHI5smWHS93nwXbsIvN83h5RdsTxylvW8LtrbsteWHidQK515Rw3jBceGzFx08c5/A6G+7rOmlSJTuO89Yd7vM8eXEP5R0HVdted71l0+vEMS9udeave10JPbBwR7ZnB5Y+bfut7i/i+4OyOIbbnXdc5S0779pQND28xob3Y0Vpnfe7ma0n2+5ujtcsjmHalM2frefqHz1wt7t396M0JG0tVSpj7s13sm5mLwem3P316fdXAQe7+5uCMMtJi3a3Z4dn/q4d1Xg8RGQ8rV+1DIDFK9fO+H7TMatZdO4KFq9cOyPM+lXLtoQVkXxrptcxNTE57Gi0xvpVy7jpmNWlYZSe0pTL/Pyr3X3psONR1/5PeYyffvG+pWFe8ISfVG6TmR0JfAiYA3zC3U+Lpv8l8Ir06zbAk4Dd3f0eM7sV+A2wCXi4ifTrV4nTBmBh8H1vYDoM4O5nAGcAM54GxTdEsU5vgLLllS2zE+Hyelluth3x9sTxLYp/3nxV8Sj6UQzXEd54dircpjri7S+arx/7LV5mHI+8OFXdgJft06p93a1BZQaKtjs7pqrOs7zzuk56hbJjs0zV9KrrS17YvPBZHNdMr5ux3vgY6iXDlpfxy/sfC9Mp73jrJj51bqiztJiamJwRPv6cicPVWXY2X9m6w7S56ZjVpesJlxsvP1xmUZzKtrUb4fkU7qc10+tmXJfz0jH8XBaHvHhnFp27gsUM/gFD3n7oVd3jatG5KyozP1nabx3P+HvOfENIT+jsWKw61jtZJ1CYpp0sv2zfZOOrzuk69zJl14Ci8J0cW8BW2zBbMtMOPERvHUAEfSYcQZK3uNLMLnb3LU1/3P19wPvS8C8GVrr7PcFiDnf3u3uKSBinPpU4bQP8FHge8DPgSuBP3f26vPDb7bPQJ972li0HV3iwFR14ebKTKfwfzxveWOWtLy9c3WnhSZ73OR5XtIxO4hePq/M9Tt+q+JfNHwsvaOGPfXgRjG9MivZbnfTtZB+X3UTVyTjFcSqLdxifvJu5OGyd9C6TzRtuX3xzFd+Ix3GE/B/QsjQI45kX7zrxz/tRqXPzF68rjlO8jvBHNO9hQfy96vwr+x5vSxzXsvMsnDfb1rLll2WI6t4YxvELt7fs2MyLb3ys54XJ4hZeJ/LWkcUjL+3isNl1Ji8zkxfn+HpU50FGXKIZrzOWd97FaRPGIwwbxjs+buMMVLgN8XldJ1NSdYMafs/SoY664auO0ap4xNPLjqs4fNXvZJjW8TUpXEcTaVX2O1K2zXWuUfE68uKcl2mPxRmT8LenTglcnrz9UXbtLDte6tzTFN1j1l1uuIxs3nh8neO5arvidcyZv75VJU5LnrKDf+CixaVhXrLomtJtMrNDgVPcfSr9fhKAu7+nIPzngW+4+8fT77cCS0c+4wRgZkcBHyQpWjvT3U8tCruLzfND7Hl9iUeRsqfdeRdNkVFUdtMcVl/Lu4ELVf1QFj25i29+w2WF4/LE1ek6UVRVD4q3d5Dn8qhWDew2XqNehatuKUun8zUdj6Z1s76yEqdR38+jpk6mtG5JZ9k8TZbk9MOwjpv4ocWg41D1UKvJ9UxNTLauqt7ip+zg7/3SAaVh/mjxuqqM0x8DR0ZNfw5x9zfmhN2BpFRqcVbiZGa3APeSFIB9LK3t1pO+ZZw6sYvN88d98OTS0ohMXJKRKXrSUVbNIlt+rOzpap2LYFG1jbzqF/FTmrLw4fi6TwfLnuKWhYnjGYaNn26GT0vyqzOUq3pSlVeKUvS56Kl51dOrMGxZPMsu0J1W1YufVtaJQxb/+LjIq6oTZ2by0ivctng5ecsvik/e8RSXLuQtN5wWx7VOiVNdRaWdZfLiH04rKrkM06LqfMjbF3nnX7xfi54M511z6pbkxCWuda5rZcdL0bU7/Byfl1XHabgNRdsVn2t5VXfi/ZSlR9F64/ny9kO8zXHJVF7aVV3HwrQL0zWMbzw9b1zZcV+n1LbTEqO6T/HjuDYlb/9l8en2Rrsooxmvp85vc9V66xz3TaZrXCMkr0S0bJndVLUsu18JrwVFD9/Kztc66ytT9dtT9HsdxrVuqV3RuvNKM9tW4rToKTv6aV96YmmYP1n8/duAsDTojDBzU6fPhCDsMcAr3f3FwbgJd582sz2AS4E3ufsVvWzXSGSclj5te//emoUzxtW58NYphi1TVpRdR6fVG+I41Z2/k3X3Mq0sXN1t6CYj0Em8ug3fTfzqZjLrHpvxD0Anxfu97NO8agBNHOfdxLGTZUF1dcCyz/1StG87OWd7qR5SZ1t73edV8SiKW6abG+2y86jOuVg2bzfncJWyzEK3y+/0upm3jrpp2238yravl23t5Bjr5znezfHQ5HWnk2tDFq6XOHUTrqljqptjJz6vezkm6tw31rnG19mObu/3wultzDj944Xlr3A9dslVjVXVM7MLgS+4++cLlnUKcJ+7v7/2RuToywtwO5V1ybjo3BUzDt7sYCk6qBavXMvUxCRTE5MzwoTzhrLlh+uJhQd/FqZu2GzIOxHDOOX9YMTLDcflxTfvRjyOV7ztVfMVzVO2zLIbpzo3NuFyyvZLvPyq5ValTVHax+uKj6s4TLyc7GlRfGxm/xevXLtlKIpbXrzK0rJqW8Jw8XEVb0+d+ariEcs7jvLOw7LjJ1523r7IW1bV/i27HtQ5v/KuUVXHcRY+PC5iUxOTlTfA4TkTb0/eMvOubfE6s+O3LP3K0iU7xvPChk9xy25W8q6lnWS28+KV9zlv3KJzVxQ+bc4bX/SbFY+P5832b94y847tPPE6wvnDZWfrL9quOtfpouOq09+STs7xvHF5v6NV6+x0WlG4utfLussumy887uO0i4+3Osd9tv/zhsUr185YdnZsxnGr+v0oWncdddM2XkedzFYn8SzLpBddo8JpZdfNsvlDRedqt8fVMG3GeNDnlA41XAksMbP9zGxb4Fjg4jiQmT0W+H3gomDcjma2c/YZeAFwba/bNRIlTsNo49RPRW1JemnTkS13FNtNyGhqqo3ToNXpnbGJnur6La7CFa63k3O51575RmnfyugrqqJZFr6bqnZ51QqbVNRZR6fxLatKOag2NVVVpYvO8WG1/Rmkflzj4urjcSZ0WNrWxmm/p+zkp1zw1NIwr9n/O3W6I9+qzwQzWwHg7qvTMK8haQt1bDDfE4AL06/bAJ8v62+hrpHLOM2W7shlpqr9Gt9MF30uMlv2UdzTVzw+/pyJ2w02fd50ok535G3IOMn4Kmp3Fstr31QnbFOUce5deG2M2/WOc4akH+J2gOOcfnltHrNjp9tzsm0Zp30P2tn/6oLJ0jD/3wHfatU2wQhmnERkdhuXEicRab+w1Ea97sowtS3j9PiDdvZ3ffEZpWFWPPGKVm0TjEgbp2HJbqLWr1rGmul1ucXha6bXNfKiUhGpr+jFrnGd9E7Pz342Kp+t6vRo1U2vW72oc0w0fV3Plhdva6/rGcTvj37j6qlqkyUiM23iUaVDG418rMMfozCjE1/o4+l1bqiyp0VhQ/5Y1niyTJbpyotX2Tzh/3g74u3OW25eZi/e7jBus9mgb9wGLTwWis6D+HOnx+wgLV65dkaj3LAjhLi+eSdPfcP5i87BXtMjXG6WxnnnZZ3lZPNWrStvvWUPg8J9H6+j03MlfD1B0bxF3Tj349hbM71uS3XVsm1purQgW168rd2+oyycv2xbmrjGx93f1zn2srjGx3g8f9kDyfhYjI/dsuO4Sl7culXV2UzTitKiaPw46mZ/Fd0nxelWdmzW0evxVHSMj9u9m2M85HNKhzZSVT0RGSlZGxC1cRKRYcvrHALKO2UQ6Ye2VdVbeNAu/tYvlGf83nrgpa3aJkh6mRCRhuS9PLTsZZ+ytbBL9xnjV06ymN7SrJM0Dxv35pWYlL1IWERGX3z+5r0CYGrikenh+d3rtWiQOumVr6jzEpFOZSVO46brjJOZLQQ+DewFbCZ52++HzGwecC6wL3Ar8Cfufm/vURUZfUXVdbLvWzIFLfrRnc229IJUsb/qvE9EREZb3nt9plZODicyDSrLCMXXNmWapEmbsGFHoXG9tHF6GHibuz8JWAacYGYHAicCl7v7EuDy9LuISOvUfYFuWVgRGU1lL+EWkd64G5v9UaVDG3Uda3ff6O7fTz//BrgeWAAcDZyVBjsLeGmPcRRpXNxYNByXfW5zo8x+iDsTCDuhyAs3DqpKkbKOK1TaJFKsqU4amohHGJ9M/KLTfj0MGdT299LpQfa/0046xrWTCumegzqHKFyI2b7AFcBBwO3uvmsw7V53n5szz3JgOcCcuXOfefiv+t85xDBf6tmUQb3QsCytyhrYdxO/Tl5+m/cS10GmSd4LYjNtPaYkX9bGCbbet3lt1sbphZh57RzyGslnYZpqFxEup19tLdSGQzJVN/tqi7q1vHZhSqPBaFvnEBNPnuuvO+ew0jD/8NQvtWqboIGMk5ntBPwncKq7X2Bmv6qTcQqpVz2R0ZBlFgb9QziI9eZltvNetqubABGJdfuAruwBTFPG9QGOzNS2jNP8J8/1488uv7f/p6d9sVXbBD32qmdmjwa+CHzO3S9IR99pZvPdfaOZzQfu6jWSIm1RdCPelhv0YdXxH6e2BYMqAe1WUVWhQZfgSvvl9S4ZdpRSVWLbJr10FJGlR787mijqzCbOWGXyMlpt3Dcymhxjs49f5xC99KpnwCeB6939A8Gki4HjgdPS/xf1FEORFglfqlxnvIyfUd/HVfEb9fjL6Ih7C4Wkl7Ysg1DUG+VsuzEfZK+beRm8sDfXcFr8PQwr0oTNPfVBN5p62aJnA68Cnmtm69LhKJIM0xFmdiNwRPpdRGSkqYMHEekHXVtkNnKHhzY/qnRoo65LnNz9W1DYQbsaLIlIq4xTdUEREZFhSqrqtTNzVKaRXvV6pc4hRJqX184q/gyoMbGIyAgr6/1PvdrNHm3rHGL3Ax/nf/SZo0rDfGzpZyu3ycyOBD4EzAE+4e6nRdMPI2kWdEs66gJ3/7s683ajp84hRKT98hoTi4iIiHSv9xInM5sDnE7S9GcDcKWZXezuP46C/pe7v6jLeTuijFMLFfV6FXZ7micsach638nGS7vFpUmLV64t7KAi/izSD+qdq56idOom/bJzv+h9WPEytY9G0/pVy2a0iyrbR3roVS7u8TETnyfSH5sLW/TUdjCw3t1vBjCzc4CjgTqZn17mLaSMUwsV3fROTUxy0/TqGUX3cdfD2fiplZO64IpI3+iGpJ6idOom/bZc34N5iz53uw7pP3Um0R9xN/nSX0nnEHN6XcwC4I7g+wbgkJxwh5rZD4Fp4O3ufl0H83Zk5FptrV+1rPA9I9n0OGxV+Kow3cSv6eU2RRcEkfYoun4M47qyftWy0rYUdRUto4llD0I/4h/Pm33vZ5qEy14zva416S/6He+HME0HkTHNzrl4qLq2j9O5mr3HqWwAdjOzq4JhebSYvCKruHOG7wOPd/enAf8KfKmDeTumziFEWiysotfPdXSz/LyX/Ra9GFhEZLaIq+LFVBIoedrWOcS8J+3uU//2stIw5xz68dJtMrNDgVPcfSr9fhKAu7+nZJ5bgaXAkk7nrWPkSpxEREREZiOVNsk42eyPKh1quBJYYmb7mdm2wLHAxWEAM9vLzCz9fDBJ3uaXdebtxki0cXpg4Y5JzcM+y3sC3jaDekI/DmnVtDDt84rblU4zhR2QiIhIIrs2trEtU141MpWQSa5HquN1vwj3h83sjcAaki7Fz3T368xsRTp9NfDHwBvM7GHgf4FjPalOlztvTxGigYxT2t3fVcDP3P1FZjYPOBfYF7gV+BN3v7dsGdvdcX/xq3QbNA43toPahnFIq6apV7rOKNMkIrK1smvjTcesZmrl5OAi0yFlkqQuBx5u4AW47n4JcEk0bnXw+cPAh+vO26smquq9Gbg++H4icLm7LwEuT7+LyCxT9DT1pmNWb5nWxieuIiK9CK+BIuPKoU7nEK3TU4mTme0N/AFwKvDWdPTRwGHp57OAbwLv7GU9IpJvECVfVdUT4+qjWbipia3fMbJ45doZT1P19FJk/MXVu4Z13mfv7cniM+h4DGq9ej+RjALHeHjz+HWl0GtVvQ8C7wB2Dsbt6e4bAdx9o5nt0eM6ZBaoaruVTY9f9HrTMauZmpis7OJTVetERIanrW16pDlrptfNOA6UuRt/DbwAd+R0nXEysxcBd7n71WZ2WBfzLweWA2zPDt1GQ8ZEVcYmmx63M8pKLzrJGPWrg40wc9dpfKrEmcZO4xB3bBEvI162OgcRKRaWXIT/i8LlzRPrx3VpVEoelGl6xKJzV4zcy+ezDE2/r/VTE5MsZu1It+GSBjmtrY5Xpuv3OJnZe4BXAQ8D2wO7ABcAzwIOS0ub5gPfdPcDypal9ziJjL66VfUG9SMsIu2gqnqPrH8Q6x2VDLM0q23vcdrlgD39WatfURrm689d1aptgh5KnNz9JOAkgLTE6e3u/kozex9wPHBa+v+i3qNZLa7ClX0e5LqBLVXHBkE3qDIoRU/Dkx/omedAXtumsnNy3F6CO6ybsjrizG9WChBeR/L2x6jfiFXFr06J0CDi0bRu1ldV8tXtsqC4ZClez7DO+SwewzqWB7XeUT5X+6mTY7qX438Q917hw4a27k+nvR1AlOm6xGnGQh7JOL3IzB4HnAfsA9wOvNzd7ymbXyVOIjIuRj0jmP3ox7I4j3omSUZHnHGKZRmp+HjSMSbSubaVOO18wF7+9I+8qjTMfz3//a3aJmimO3Lc/Zvu/qL08y/d/XnuviT9X5ppkvZZv2pZYbucOu11Op0nmx6GK/os7TYq+zKOR3bMh8d+XlyLehiMlxNOWzO9jjXT63KXX7SOvOnh8sqWMzUxyeKVa7cawul5y12/atmWG+RsHeH3bKgjnC9v2/Km1RWna9X4UVAVn3Cf5oXt5zaF+zaMw/pVy2aU4OQNi1euzc0gKdMkVUbtHJXOuY9nd+SNlDj1SiVOIjIbxFWKw94hs6fzTVcBGXTVZRkv4Q1sVlKpXtFEBq9tJU477b+XP+3040vDfPsF723VNkFDJU4iIlKt7OW//XopsF62KU3RcSQi9RmbNj+qdGijXt/jJCIiNWVP7BezdqtuicN2R012VxyuU6QX6lZcROpyxrM78nZm90QkV1H7m7y2HvF8cbuWsrZsUi1uY5RVzYu7cM/ageS1NwqX04010+tylym9idtoFe2jojZWefpxrnWzzLBNU9aJQ9xmKRu6XUc2Xz/bZoXL17VMZAgcNrmVDm2kjJPIGMl7UTDMbIuQdxMd31wX3cRLfXoyP77ibq2LzpO8866oXVA/zrVBnL/dHuf9Pj/C5etcFBk8B9ytdGgjZZxERPogr8tvkXHTluNcmSeRQSvvUa+t1fjUq57MSnG1jbB3s7A7ZpW4iIjIbJD3Xq64XVtRianezdV/betVb4clE774A68vDXPNS/6+cpvM7EjgQ8Ac4BPuflo0/RXAO9Ov9wFvcPcfptNuBX4DbAIebiL9RqbEKa+NRVXd50G2w1Ad6UeUpUU39dWr0jWvHUET73GKexur+0RSx8Hoq2rTNSqafB9ar7K2INnnvOnxe5vCaXmf4/mr1p/3nqiiuGTqvgeuk3h0Mr6OeN6i91g1Kd4n3W63zG4qqRusOte8tnDvvaqemc0BTgdeCBwIHGdmB0bBbgF+392fCvw9cEY0/XB3n2wq06kSJ5mVikqc8j6LiIiMu7jEKXy3XNW7u1Ti1H9tK3F6zOIJ3+/9y0vDXP+yvy3dJjM7FDjF3afS7ycBuPt7CsLPBa519wXp91uBpe5+d1cbkaOnEicz29XMzjezn5jZ9WZ2qJnNM7NLzezG9P/cpiIr0iQ9SRMRESmXZZ5EOlWjxGk3M7sqGOKc1gLgjuD7hnRckdcBXwmjAHzNzK7OWXZXeq2q9yHgq+7+ROBpwPXAicDl7r4EuDz9LjJysh8CZaBERERmiquyK/MknXDKM01pxulud18aDHE1u7z6fLlV5czscJKM0zuD0c9292eQVPU7wcye0+t2dZ1xMrNdgOcAnwRw9wfd/VfA0cBZabCzgJf2FkWR/tKPgYiISDk9ZJSOOE30qrcBWBh83xuYjgOZ2VOBTwBHu/svt0TBfTr9fxdwIXBwD1sE9Fbi9ATgF8C/mdkPzOwTZrYjsKe7b0wjuhHYo9dISvuUdd5Qp+OPfgnbLsXvKir63NQ6R7mTAhkPo9KAuOwlr/2KY7fL7edLWLtVp8MLXU+KKW26E76frGiomldkBq8Yql0JLDGz/cxsW+BY4OIwgJntA1wAvMrdfxqM39HMds4+Ay8Aru1xi3rKOG0DPAP4qLs/HbifDqrlmdnyrE7jQzzQQzSkjVTKI9K8NpxXoxbHUYtPHXGPoDKT0kZkNGzebKVDFXd/GHgjsIakOdB57n6dma0ws+zi/TfA44CPmNk6M7sqHb8n8C0z+yHwPeA/3P2rvW7TNj3MuwHY4O7fTb+fT5JxutPM5rv7RjObD9yVN3Naj/EMSHrV6yEeIiLSAlt65lo5OdyIjLibjlmtNBKRVnOo1eV45XLcLwEuicatDj6/HtjqhVHufjNJ/wuN6rrEyd1/DtxhZgeko54H/JikCO34dNzxwEU9xVBERFqlqBRn0bkrRrKEZ9TiVBWfUU1HEZEtHHArH1qop/c4mdkkSWOsbYGbgT8jyYydB+wD3A683N3vKVuO3uMk0plxeM/U+lXLWh3/QcrSKv6fTYN2HwsiVcJ2aOF7hfKovY2Mo7a9x2m7JyzwBf9wQmmYW15xcqu2CXqrqoe7rwPyNli5IBERERmoReeuYDF6iCAyfNZIVb1R0+t7nIYq66FptvWgU7S9cU92eeGy8eEgIsNR51welKZ6vMuWEW/DqPVcV6Qonr3EP69n0WyZTa4vb7nZ57x1tWWf5ImrKqrqooyDUezls2sOvtlKhzbqqapeU1RVT6Qz41A9a830OlWpqamsql72I6u0lHEW30zG1fXC7zoXZBy1rqrefnv7/L99Y2mY244/qVXbBD1W1RMR6Zaq1DQje9KutJTZIDve89o4VbV9EpFBa2epUhllnERERKQVbjpmdW61PGWYREbQ5mFHoHmqqjckeVVusvGZbLqqH/ROPbiJiIymOm069Psns13rqurtu7fPf/dflIa57bXvbNU2Qcs7h5gt9M6O3ulppIjIaMt+5/R7JzIefHP50EbKOMmsoB9iEZHRlj3gijt9EJGW0gtw+2PcquqFPZ7FVe/WTK9LGsV3UW1M1c1ERES6N5t+R2dDj5v96J01ayIRdjaSfR5mWrayqt7Jby4Nc9vyd7Rqm0AlTn130zGrtwzQ29MzVTcTERHp3mz7HVWJXefySj6LOiWREm6wuWJooZ4yTma20syuM7NrzexsM9vezOaZ2aVmdmP6f25TkW2LuJpBU22UdNKKiIh0b7b9js62jGI/KS274BVDC3WdcTKzBcBfAEvd/SBgDnAscCJwubsvAS5Pv88q4YU5fnIRlj6JiIjI4Oj3V7ox2zLcjdlcMdRgZkea2Q1mtt7MtspTWOJf0uk/MrNn1J23G71W1dsGeIyZbQPsAEwDRwNnpdPPAl7a4zpap6pqnk5AERERkXZQhrsLTs+dQ5jZHOB04IXAgcBxZnZgFOyFwJJ0WA58tIN5O9b1C3Dd/Wdm9n7gduB/ga+5+9fMbE9335iG2Whme/QaybYJGw9mjVCnVk5umbaY7hqmhg1a4waRdRq8FjWizDqwCDN7s6XxrIiIzB7j3FFCbDZsa7yN8TvBqtIgfFdmnXlmU+ciTbDeq+MdDKx395sBzOwckgKaHwdhjgY+7Ulvd2vNbFczmw/sW2PejvVSVW9uGoH9gAlgRzN7ZQfzLzezq8zsqod4oNtozFpxqVWTT0P0ZEWGJeyFcpjieKxftWzGkBemaFy/4jaotCp7OWmdF5eOijC9Ri3edeOzftWykYu7SJt1er+j+6MO9d7GaQFwR/B9QzquTpg683asl6p6zwducfdfuPtDwAXA7wB3pjk90v935c3s7me4+1J3X/potushGgLqrU/Gw6g8yYvjsXjl2hlDXpiicf2K26DSquzpa5ueaIfpNWrxrhufxSvXjlzcRUZJL/dCdeZVU4vOmJcPwG5ZIUo6LI8XkbPYOMtVFKbOvB3ruqoeSRW9ZWa2A0lVvecBVwH3A8cDp6X/L+o1knXkvTtpUDcW8ZPfQd08hdtZZ51FP7hZ9cGsOqGIiIhIm2TV6Orcy2RNF2Y0rahxHzSoe8ysZLn1TSeq2zHdXfEepw3AwuD73iT9KdQJs22NeTvWdYmTu38XOB/4PnBNuqwzSDJMR5jZjcAR6feuhdVS8j5n8kpNhlHtR6U3Iu1QVOVNVaFkmDo9/nS8igxWr70jr5lex5rpdVuq3oZD3rpay2miV70rgSVmtp+ZbUvSe/fFUZiLgVenvestA/4n7Wuhzrwds6Qt1XDtYvP8EHteT8sYdonTMDpWmA1vBRcRERGpUtQBVq9hY4O4xwzvK7N4XubnX11ROjNStlu40PdeubI0zM1ve1vlNpnZUcAHSV57dKa7n2pmKwDcfbWZGfBh4Ejgt8CfuftVRfP2tFH0VlVvpGw5wFZOzvg8yHVnn1XlTURERGT8DOIes9UlTaGa72oq4+6XAJdE41YHnx04oe68vWp1ximsYjM1QfB5cqDxGFaJj0qaRNqtqGvbYXR521QJdtGT3F6e8A5SP+If789sWf1Mk3DZqp0gs0Enx3cv50I/z6NxOleDDiDGSq8vwBURkTGhHqOKNdlzqdJZRGaFHl+AO4panXEam6JMEREZaXpXnohIZ2xz+dBGI985RFbFoZvOH/SGZxEpEl5bwuuErhsyTJ1W32tLFUgRSZT1hFl2Lretc4jt917oC094a2mY9e96a6u2CVpe4iQi0jSVBsgwqRqfyOwzlue9q8Spb8ISp6rSpE6fBmddOjbVTXjeu6H0dFpEREREYOYLbOOHcd2WELeuxGnBQt/nDeUlTjf+dftKnFrdq15dY5mTFxEREZGRNdtrMKhXvZaa7QeuiIiIiAyOHtoDXjG0UGXGyczONLO7zOzaYNw8M7vUzG5M/88Npp1kZuvN7AYzm+pXxDuhg1dEREREBmXWP7T3R97lVDS0UZ2qep8CPgx8Ohh3InC5u59mZiem399pZgcCxwJPBiaAy8xsf3ffVDdCVe2FOm1PVNVWKq9HraxNVN78as8kIiIikhhWz47hi6Rh8C+N7bZ3vG7W09qeM1vaAUSZyhInd78CuCcafTRwVvr5LOClwfhz3P0Bd78FWA8c3ExUm5d19LB+1bItA8wsoVozvS63QwhpXrgf4n0is0vd/a5jpL5+pVPZzUO23l7XvWZ63Zahjjrrazo9srjFy60b56rl9tMg1iGDNch9Gtfqye6bwvN2HK/Tqs1Uzpi9JU559nT3jQDuvtHM9kjHLwDCIpkN6biRVFTiFJqamGQxs7uUKU6TLL2Kxney3HCeYZbmhXGJP2fyjpde15ktt9v4jau62zfu6dCkfqVV1ZPQJtbb6dPWOuuMw6yZXjej99Wqp9hxrYUsXLzcTuKed253uu11nk6HYdavWsbUREerkBYYZglFdt80tfKROOTdRw2qxGbQFp27YtbfN27R0sxRmVrdkZvZvsCX3f2g9Puv3H3XYPq97j7XzE4HvuPun03HfxK4xN2/mLPM5cBygDlz5z7z8F/lvwC3SfFNcBsN6qZ5HNKqaUUZqozSSURExlnWnAEobNIg/dG27sgfM3+h7/fa8u7Ir//H9nVH3m2venea2XyA9P9d6fgNwMIg3N7AdN4C3P0Md1/q7kvn7LRjl9EQGZxZ39BTRERmtfh3UL+LUmo29qpX4GLg+PTz8cBFwfhjzWw7M9sPWAJ8r7coiowG1WcWERFJKNMkVWxz+dDTskt6+A7CLDSzb5jZ9WZ2nZm9OZh2ipn9zMzWpcNRddZbpzvys4HvAAeY2QYzex1wGnCEmd0IHJF+x92vA84Dfgx8FTihkx71+i07ydt8sg867m1OKxEREWneonNX6GGilKsqbeq9xCnr4XsJcHn6PfYw8DZ3fxKwDDgh7QE8s8rdJ9PhkjorrdXGqd92sXl+iPW/jZMI5LcTa6JziKIu7svi0S3VKRdpVtxZS9xZRCzsCrmphuxNLKvVXRfLUGTtlnTcjLbWtXHaa6EvenV5G6fr3td9GyczuwE4LO2kbj7wTXc/oGKei4APu/ulZnYKcJ+7v7+T9Xbbq55Ia/Ur05Ett6l3jYWqOqYQkd6E7+8Lx4U9g+VRD1rSdotXrq08zkW60Wt1vApFPXznxyXp6O7pwHeD0W80s1cDV5GUTN1btdJu2ziJiIiMjbxqR3WqIqk6s4hIgeqqeruZ2VXBsDyc3cwuM7Nrc4ajO4mGme0EfBF4i7v/Oh39UWARMAlsBP651rJUVU9mm3516d5pVb1wvirZcrPPIiIiTVBVvXZoW1W9HfZc6ItfUV5V75pV/a+qZ2aPBr4MrHH3DxQsa1+C1y6VUYmTSEO6ffKsJ9YiIjIs+g2Svulv5xBFPXxvYWYGfBK4Ps40Za9VSr0MuLbOStXGSWadfpXYTE1Mzih1CtdXND6br04bCZU0iYhIp9ZMryuctujcFUxNPPK9XzUyZHay/lZqOw04L+3t+3bg5QBmNgF8wt2PAp4NvAq4xszWpfO9K+1B771mNkmShbsV+PM6K1XGSURERGQWijtAqdMhikhtfewcwt1/CWzVzsfdp4Gj0s/fAqxg/ld1s15lnEREREREpDne9xKnoVDGSUREREREGtXn7siHQhknERERERFp1hiWOFV2R25mZwIvAu7Kuukzs/cBLwYeBG4C/szdf5VOOwl4HbAJ+At3X1MVCXVHPh7i7rjD7rPzOkUI5xlk/ERERKRZWScUTXVrnnWTHlt07opZ2X1667oj32OhP/GPyrsj/8Hq7rsjH5Y63ZF/CjgyGncpcJC7PxX4KXASgJkdCBwLPDmd5yNmNqex2IrIWKvzTqtBiOOxftWyGUNemKJx/YrboNKqbD1lvXWNmnA7Ri3edeOzftWykYu7yKCp+/SWcJLOIcqGFqqsqufuV6QvhgrHfS34uhb44/Tz0cA57v4AcIuZrQcOBr7TTHRFZJyNSolgHI+8eNUd17RsHYNKq7L1tOmJb7gdoxbvuvFZvHKtejyTWScrYZJ2Mcazc4gmXoD7WuAr6ecFwB3BtA3puErxk7RRe7IWPmmWfPEN3eKVa7f6XOeGtN/x65WOg9EX7qNh7q9ermF1rjl5pSjZtXPQ293JNTsLF4bP+7xmel3uMovmy5t3lH5HulW1HU38Xo5DOslwTU1MNvZQYs30uuRBwcTklv/xMJu09fy0zV46tFFPGSczOxl4GPhcNionWG7KmNlyM7vKzK56iAe2nBiZ+Puw5d30y+yk42D0hftomPtr0bkrBr6um45ZPdD1NiW83mefi26Q8sJm4m0fpd+RXpTt0yaexo9LOomMo1aen15jaKGuM05mdjxJpxGv8Ed6mNgALAyC7Q1M583v7me4+1J3X/potus2Gj0J2woUPd3t5YllXjuJTtavkg2Zreoe+7OpJHgY21nn2pfFK+/mPa9kKW9aWQlS3fmycTcds7pwGZ2sp1NF29pUSVA/qyq19Wm2iIw28/KhjbrKOJnZkcA7gZe4+2+DSRcDx5rZdma2H7AE+F7v0eyPqmpk0FvRc1W1tFGqxibjIbuJjW+EqhrGx1W7yh4mDELdY382lQTX3c4mS5vqXPvqtIEqKzWKr7FV6ywrjZqamGTRuSu2/C+at856ysSZpPWrluVu65rpdVu+l2Uis2XkaSKeVVr5NFtERp5tLh/aqE535GcDhwG7AXcC7ybpRW874JdpsLXuviINfzJJu6eHgbe4+1fiZcbUHfloi3/QsxulovGdLHe23PSKiIyKOEOnjJMMQ5ixjzuA0DG5tbZ1R77jbgv9yX+wsjTMlZ9+W6u2Cer1qndczuhPloQ/FTi1l0iJiIhIfyw6dwWL0UMrEekjb2+pUpkmetXru06qC82W9g6hfm+zugEVmX3qtBMaV3FbqKpe/mJFvdx12saqX2kdXtPb2JGIjB/dZ4yfrDvycWvjVFlVbxBUVU+qqu1l07M3iU9NTG71uYyqBIqIiAzPmul1M6rkqTpeZ9pWVW+nxy30g458S2mY737+7a3aJmhJiZNISE9IRURE2kclS7OI97dzCDObZ2aXmtmN6f+5BeFuNbNrzGydmV3V6fwxZZyk1ZSJEhERaQf9Zs8ufe5V70TgcndfAlyefi9yuLtPRqVbncy/harqycgoqm6nanYivRtWL5ZN9NoWtvVR9R6RasPsLbGobZ7O3d60rqrevIX+tOe9pTTMt8/vvqqemd0AHObuG81sPvBNdz8gJ9ytwFJ3v7ub+WMqcRIRkZGmp9Qi7RKes4vOXaFzeDZywL186M2e7r4RIP2/R0lMvmZmV5vZ8i7mn0ElTtKxrGQo7+l1N0+1O+kYInyHVN5nEeld3tPq7Cny1MTkjM9F4Xtdb7+emOt6ISJt1LoSp7kLffK5by4N898X/OVtQFgSdIa7n5F9MbPLgL1yZj0ZOMvddw3C3uvuW7VTMrMJd582sz2AS4E3ufsVZvarOvPHKt/jJCIiMi5uOmY1Uysnhx0NEZGxZtRqx3R3WWbQ3Z9fuHyzO81sflDV7q6CZUyn/+8yswuBg4ErgFrzx1RVT0REOjKIajeq2iMi0mJV1fR6r/F2MXB8+vl44KI4gJntaGY7Z5+BFwDX1p0/T2XGyczONLO7zOzanGlvNzM3s92CcSeZ2Xozu8HMpupEQkREJKRui0VE2q3PveqdBhxhZjcCR6TfMbMJM7skDbMn8C0z+yHwPeA/3P2rZfNXblNVGyczew5wH/Bpdz8oGL8Q+ATwROCZ7n63mR0InE1SDDYBXAbs7+6bytYxrDZOWV337KVssHW7nWya6sSLDE7ddihl7e1kMOq0RRp0D1911td0W6dsnfG6e932QaTdMHtgE5F62tbGaedd9/Zn/F55G6crvvyOVm0T1ChxcvcrgHtyJq0C3kHSW0XmaOAcd3/A3W8B1pNkokaaqoSIiMwuTZdotfl3pM1xF5ER5cBmLx9aqKs2Tmb2EuBn7v7DaNIC4I7g+4Z0XKX1q5bNeI/P+lXLCt8F0JTsaePilWu3DLGpicktPboVvWdIZhcdB/1XtySg6LwNrx3aX/1Vp6SiTpi8fbZmel3u/qvav3GJT7dx6kR2HMbL7XU9gygJympeiMhoauv52eeqekPRccbJzHYg6Qbwb/Im54zLzVKa2XIzu8rMrnqIB3LXpadgItINXTvaLSsNqrMf1RZKRGRE9bdziKHopsRpEbAf8MP0bbx7A983s71ISpgWBmH3BqbzFuLuZ7j7Undf+mi2A7b+ARy1H8RRi4+IyDhSxldEpOVcJU4AuPs17r6Hu+/r7vuSZJae4e4/J+na71gz287M9gOWkPRiUWnxyrUzqiSE3zupatOvajlx/IYp3sbse1zVMS8tsvHhIJ1RRwSjL9xH2l/dKaoaUrfKSKfXlvj6n/0vqkKd97lquXWFcV8zvW7LNoefq+bPCxeOC5eZp6iaYtV6i9aTt67w+6j8vonI1tp4fhpg7qVDG1W+ANfMzgYOA3Yzsw3Au939k3lh3f06MzsP+DHwMHBCVY960rmiG8E6N4u6iRSRQWjztWaQL8ktuiGamphMumDqQF47q7JMZhtvxkSkRVpaqlSmMuPk7sdVTN83+n4qcGpv0RIREWmnpqp1DzIDJyLSKAdrac95ZbrqVW+UzcZ2SLNxm0VkOGZD+6N4G8Pvdbe/iXSaDWktIuOqomOIllbVG7uMk35oRET6ZzY8qAm3cdG5K2Z8nw3bLyLSBNvspUMbmY9Ajm8Xm+eH2POGHY3WKHrjfTg+ayQch8trbNzmtggiMpqKrlMyU53reV1rptcxNTG51W9B9jmb3ss6RGQ4LvPzr3b3pcOOR1277LTAD5n8f6VhLvvvv2rVNsEYljjJI8qejOqpqYj0k64x9RSlUy/pV7eETPtIRPpKVfWkE3HpTtwVbNiNeF7Xs0Vd0dbtVS+vx6Sse9+picnCrn5FZPQM49UBvbytfmpisnT+oi6+q9ZZ1jV43XlHSWmvel0uS73qicgoGMeqeso4jQi1zRKRUaPrkoiIdE0lTiIiIqNBGTsRkdFk7tim8qGNlHHqo6r649n0+L+ISGwY14dRvyaNevxERGa1PpY4mdk8M7vUzG5M/8/NCXOAma0Lhl+b2VvSaaeY2c+CaUfVWW9lxsnMzjSzu8zs2mj8m8zsBjO7zszeG4w/yczWp9Om6kRiXFU9Dc2mx/9FRGLDuD7omiQiIl3rb1W9E4HL3X0JcHn6PVq93+Duk+4+CTwT+C1wYRBkVTbd3S+ps9LK7sjN7DnAfcCn3f2gdNzhwMnAH7j7A2a2h7vfZWYHAmcDBwMTwGXA/u6+qWwd6o5cRGB0ukeO4xF3zLB45drcuA4i/tk6BpVWZeuJu7ceZWXdcg9b3fhknVuMUtxFZDDa1h35Y3eY8EP3f31pmDU//Puut8nMbgAOc/eNZjYf+Ka7H1AS/gXAu9392en3U4D73P39nay3ssTJ3a8A7olGvwE4zd0fSMPclY4/GjjH3R9w91uA9SSZKBGR1hqlKmGjFBcZLJUAikir9LfEaU9335isxjcCe1SEP5akcCf0RjP7UVq7bquqfnm6beO0P/B7ZvZdM/tPM3tWOn4BcEcQbkM6TkRarKgr7LBL/WwoCle1DBidlzHH8Qi778+m5cU1fgF1P2SlDd2kVTfxWrxybWE33qNS8lGnG/IwvbJ4N909ebfLq5uO8Wsmullf2J27iEjfuMPmzeUD7GZmVwXD8nARZnaZmV2bMxzdSVTMbFvgJcAXgtEfBRYBk8BG4J/rLKvbjNM2wFxgGfCXwHlmZoDlhM3NUprZ8iyhHuKBLqMhIqHsxrjonWHx53BcOD57r1jVjbZKP/IpXQZr0CUxceajKBOS936qorD9yGwrcyQiQ7W5YoC73X1pMJwRzu7uz3f3g3KGi4A70yp6pP/votgLge+7+53Bsu90903uvhn4ODVryG1Tb8u3sgG4wJMGUt8zs83Abun4hUG4vYHpvAWkiXMGJG2cuoyHiNRw0zGrmVo5WTt8fCNaVLpRt/SjbPqolDI1aVRKYmLdpvWobk+maruafMlsN+vpl6Zekisi0g/W33c1XQwcD5yW/r+oJOxxRNX0zGx+VtUPeBlw7VZz5ei2xOlLwHPTFe8PbAvcTbIRx5rZdma2H7AE+F6X6xCRLsUZH7WNEBltKqUUkbHiwKbN5UNvTgOOMLMbgSPS75jZhJlt6SHPzHZIp18Qzf9eM7vGzH4EHA6srLPSOr3qnQ0cRlKidCfwbuAzwJkk9QIfBN7u7l9Pw58MvBZ4GHiLu3+lMhJmvwDuJ8l8Sf/shtK435TGg6F07j+lcf8pjftPaTwYSuf+e7y77z7sSNT12O338t/Z5/jSMF+98b2t6ikQalTVc/fjCia9siD8qcCpnUTC3Xc3s6valnhtozTuP6XxYCid+09p3H9K4/5TGg+G0llybe65VGnkdNvGSUREREREZGsObB6/LgyUcRIRERERkQY5uEqc+umM6iDSI6Vx/ymNB0Pp3H9K4/5TGvef0ngwlM4yU9Y5xJip7BxCRERERESkrsduu6f/zp7Hlob56oZ/Gb/OIUREREREROpzdQ4hIiIiIiJSylHGSUREREREpNIYNgdSxklERERERBrkY9k5hDJOIiIiIiLSHAdXd+QiIiIiIiIVVOIkIiIiIiJSwtWrnoiIiIiISDV1DiEiIiIiIlLG8U2bhh2Jxj1q2BEQEREREZEx4sBmLx96YGYvN7PrzGyzmS0tCXekmd1gZuvN7MRg/Dwzu9TMbkz/z62zXmWcRERERESkMQ74pk2lQ4+uBf4QuKIogJnNAU4HXggcCBxnZgemk08ELnf3JcDl6fdKyjiJiIiIiEhz3ME3lw89Ld6vd/cbKoIdDKx395vd/UHgHODodNrRwFnp57OAl9ZZrzJOIiIiIiLSKN/spcMALADuCL5vSMcB7OnuGwHS/3vUWaA6hxARERERkcb8hnvXXLb5vN0qgm1vZlcF389w9zOyL2Z2GbBXznwnu/tFNaJhOeN6yrEp4yQiIiIiIo1x9yMbWMbze1zEBmBh8H1vYDr9fKeZzXf3jWY2H7irzgJVVU9ERERERMbNlcASM9vPzLYFjgUuTqddDByffj4eqFOCpYyTiIiIiIi0h5m9zMw2AIcC/2Fma9LxE2Z2CYC7Pwy8EVgDXA+c5+7XpYs4DTjCzG4Ejki/V6/Xx/CtviIiIiIiIk1SiZOIiIiIiEgFZZxEREREREQqKOMkIiIiIiJSQRknERERERGRCso4iYiIiIiIVFDGSUREREREpIIyTiIiIiIiIhWUcRIREREREanw/wPj9EC+xoPWPAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1152x236.16 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(labels[:, ranking])\n",
    "plt.title(\"rte labels, missing = \"+str(np.round(missing, 4)) +\n",
    "          \", error = \"+str(np.round(error, 4)))\n",
    "plt.colorbar()\n",
    "plt.savefig(\"pic/rte_labels.png\", dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_confusion_matrix2(k, labels, groups=None, sym=True, cutoff=1e-7, L=50, N=10, seed=None):\n",
    "    m, n = labels.shape\n",
    "    if seed is not None:\n",
    "        np.random.seed(seed)\n",
    "    if groups is None:\n",
    "        groups = np.random.randint(3, size=m)\n",
    "    Zg = get_Zg(k, labels, groups)\n",
    "    M2s, M3s = get_M(Zg)\n",
    "    Cc = np.zeros((3, k, k))\n",
    "    W = np.zeros((3, k))\n",
    "    for g, (M2, M3) in enumerate(zip(M2s, M3s)):\n",
    "        Q = get_whiten(M2, sym)\n",
    "        M3_whiten = whiten_tensor(M3, Q)\n",
    "        values, vectors = robust_tensor_power(M3_whiten, L, N, sym)\n",
    "        w = values**-2\n",
    "        mu = np.linalg.inv(Q.T)@vectors@np.diag(values)\n",
    "        best = np.argmax(mu, axis=0)\n",
    "\n",
    "        # prevent multiple mu in same column\n",
    "        # not_in_best = []\n",
    "        # not_used_loc = np.array([], dtype=np.int64)\n",
    "        # for l in range(k):\n",
    "        #     loc = np.where(best == l)[0]\n",
    "        #     if len(loc) == 1:\n",
    "        #         Cc[g, :, l] = mu[:, loc].ravel()\n",
    "        #         W[g, l] = w[loc]\n",
    "        #     elif len(loc) == 0:\n",
    "        #         not_in_best.append(l)\n",
    "        #     else:\n",
    "        #         chosen = np.random.randint(len(loc))\n",
    "        #         Cc[g, :, l] = mu[:, loc[chosen]].ravel()\n",
    "        #         W[g, l] = w[loc[chosen]]\n",
    "        #         not_used_loc = np.append(not_used_loc, np.delete(loc, chosen))\n",
    "\n",
    "        # not_used_loc = np.random.permutation(not_used_loc)\n",
    "        # for i, l in enumerate(not_in_best):\n",
    "        #     Cc[g, :, l] = mu[:, not_used_loc[i]].ravel()\n",
    "        #     W[g, l] = w[not_used_loc[i]]\n",
    "\n",
    "        # the method in original code\n",
    "        for h in range(k):\n",
    "            l = best[h]\n",
    "            if W[g, l] != 0:\n",
    "                l = np.where(W[g, :] == 0)[0][0]\n",
    "            Cc[g, :, l] = mu[:, h].ravel()\n",
    "            W[g, l] = w[h]\n",
    "\n",
    "    W = np.mean(W, axis=0)\n",
    "    C = np.zeros((m, k, k))\n",
    "    for i in range(m):\n",
    "        Ca = (np.sum(Cc, axis=0)-Cc[groups[i], :, :])/2\n",
    "        Za = (np.sum(Zg, axis=0)-Zg[groups[i], :, :])/2\n",
    "        E = np.zeros((k, k))\n",
    "        for j in range(n):\n",
    "            if labels[i, j] != -1:\n",
    "                E[labels[i, j], :] += Za[j, :]\n",
    "        E /= n\n",
    "        Ci = E@np.linalg.inv(W[np.newaxis, :]*Ca.T)\n",
    "        if cutoff:\n",
    "            Ci[Ci < cutoff] = cutoff\n",
    "        colsums = np.sum(Ci, axis=0)\n",
    "        Ci /= colsums[np.newaxis, :]\n",
    "        C[i, :, :] = Ci\n",
    "    return C\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "C_true = get_true_confusion_matrix(data, truth)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.9, 0.1],\n",
       "       [0.1, 0.9]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "C_true[4,:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14312329517709166"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# baseline\n",
    "confusion_matrix_loss(np.ones(C_true.shape)/k, C_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = np.array([i%3 for i in range(m)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.010425562610071989"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_loss(get_confusion_matrix(\n",
    "    k, labels, groups=groups, sym=True), C_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups_balance = np.random.permutation(groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.010226444725547602"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_loss(get_confusion_matrix(\n",
    "    k, labels, groups=groups_balance,sym=True), C_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.012222671377328229"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_loss(get_confusion_matrix(\n",
    "    k, labels, sym=True, seed=0), C_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.012222671377328229"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_loss(get_confusion_matrix2(\n",
    "    k, labels, sym=True, seed=0), C_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "# iterations\n",
    "maxiter = 20\n",
    "step = 1\n",
    "Ls = np.array([i for i in range(step,maxiter+1,step)],dtype=int)\n",
    "losses = np.zeros(len(Ls))\n",
    "for i,L in enumerate(Ls):\n",
    "    print(L)\n",
    "    losses[i] = confusion_matrix_loss(get_confusion_matrix(\n",
    "        k, labels, sym=True, L=L, N=100, seed=0), C_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaIAAAEGCAYAAAAnhpGXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAkIklEQVR4nO3df5DUdX7n8ecLhgamQZgZMTGOiSYQPfyxq8dZ5swlexoT8EQ2Mbcl5R3k1CCl3MbbJKUUMbV3x1U0mo1F9OTw4rlaBtfbCwkXMMoZE+8s2RIBf7CKjtzmnGAWFAR1NPzwfX/0p/FL0z3MAN/+NtOvR1XXdL+/n8+3P/OlnZffb3/604oIzMzMijKq6AGYmVl7cxCZmVmhHERmZlYoB5GZmRXKQWRmZoXqKHoAJ5tTTz01zjrrrKKHYWZ2Unn55Zffj4gp9bY5iIbprLPOYsOGDUUPw8zspCLpbxtt86U5MzMrlIPIzMwK5SAyM7NCOYjMzKxQDiIzMyuUg8jMzArlIDIzs0I5iJrkpR/s4u6/fBN/7YaZ2eEcRE3yyrsf8uBfv8PeTw8UPRQzs5biIGqS7nIJgA8++YeCR2Jm1locRE1SDaLdA/sKHomZWWtxEDVJT3ksAB987CAyM8vKNYgkzZS0VVKfpDvqbJekZWn7q5Iuzmx7WNIOSa/X9LlH0pup/SpJk1P9SkkvS3ot/bw80+cvJb0iaYuk5ZJGp/pYSd9Jz/89SWfldSy6J1TOiHZ94iAyM8vKLYjSH/sHgFnAdGCupOk1zWYB09JtAfBgZtsjwMw6u14HnB8RFwJvAYtT/X1gdkRcAMwHHsv0+VpEfAk4H5gC/MtUvxHYHRFTgT8E7h7+bzo03Z3V94gcRGZmWXmeEV0C9EXEtojYBzwBzKlpMwd4NCrWA5MlnQ4QEc8Du2p3GhHPRER16tl6oDfVN0XE9lTfAoyTNDZt25vqHUAJqM6hngN8O93/LnCFJB3PL93I+NJoxo8ZzW4HkZnZYfIMojOAdzOP+1NtuG0GcwPwVJ36tcCmiDg0RU3S08AO4CMqoXPY86dw2wP01O5M0gJJGyRt2Llz5zCGd7jucsmX5szMauQZRPXOLGo/zTmUNvV3Li0BDgCP19TPo3KJ7ebDdhrxS8DpwFig+v7RkJ4/IlZExIyImDFlSt0vGBySngklX5ozM6uRZxD1A2dmHvcC24+hzREkzQeuBq6PzFIFknqBVcC8iHintl9EfAas5otLhIeeX1IHMIk6lwNPlK5OnxGZmdXKM4heAqZJOltSCbiOSghkrQbmpdlzlwJ7IuK9wXYqaSZwO3BNRAxk6pOBNcDiiHghU59Qfd8phc1VwJuZ55+f7v8q8FfZYDvRenxpzszsCLkFUXrPZRHwNPAG8GREbJG0UNLC1GwtsA3oAx4Cbqn2l7QSeBE4R1K/pBvTpvuBicA6SZslLU/1RcBU4M5U3yzpNKAMrJb0KvAKlfeJqn3+GOiR1Ad8AzhiivmJ5PeIzMyO1JHnziNiLZWwydaWZ+4HcGuDvnMb1Kc2qC8FljYYyj9p0OczvpjKnbvuCSU+3X+QT/cdZHxpdLOe1syspXllhSb64rNEXm/OzKzKQdREh9ab+2R/wSMxM2sdDqIm6pngMyIzs1oOoibqTgufesKCmdkXHERNVH2PyEFkZvYFB1ETnTK+g45RchCZmWU4iJpIEl3+LJGZ2WEcRE3WU/Z6c2ZmWQ6iJvN6c2Zmh3MQNVn3hJK/k8jMLMNB1GS+NGdmdjgHUZN1l0vs+XQ/+w9+XvRQzMxagoOoyQ4t8zPgsyIzM3AQNZ3XmzMzO5yDqMmqQeT15szMKhxETdbj9ebMzA7jIGqyrvIYwEFkZlaVaxBJmilpq6Q+SUd8DbcqlqXtr0q6OLPtYUk7JL1e0+ceSW+m9qskTU71KyW9LOm19PPyVO+UtCb12SLprsy+fkLSs2lffy2pN7eDkXR54VMzs8PkFkSSRgMPALOA6cBcSdNrms0CpqXbAuDBzLZHgJl1dr0OOD8iLgTeAhan+vvA7Ii4AJgPPJbpc29EnAtcBFwmaVa1Djya9vUfgN87hl91WMaMHsWk8WMcRGZmSZ5nRJcAfRGxLSL2AU8Ac2razKESBBER64HJkk4HiIjngV21O42IZyLiQHq4HuhN9U0RsT3VtwDjJI2NiIGIeC612QdsrPahEpDPpvvP1RlfLvyhVjOzL+QZRGcA72Ye96facNsM5gbgqTr1a4FNEXHY1LR0GW82X4TPK6ktwC8DEyX11O5M0gJJGyRt2Llz5zCGV19XucSujx1EZmaQbxCpTi2OoU39nUtLgAPA4zX184C7gZtr6h3ASmBZRGxL5d8Cfl7SJuDngb9L+zx8QBErImJGRMyYMmXKUIY3qO5yyR9oNTNLOnLcdz9wZuZxL7D9GNocQdJ84GrgioiITL0XWAXMi4h3arqtAN6OiPuqhXQp71dS3wnAtRGx52jPf7x6yiU2v/th3k9jZnZSyPOM6CVgmqSzJZWA64DVNW1WA/PS7LlLgT0R8d5gO5U0E7gduCYiBjL1ycAaYHFEvFDTZykwCbitpn6qpOoxWAw8PLxf8dh0lysrcGcy1MysbeUWRGlCwSLgaeAN4MmI2CJpoaSFqdlaYBvQBzwE3FLtL2kl8CJwjqR+STemTfcDE4F1kjZLWp7qi4CpwJ2pvlnSaeksaQmViQkbU/2m1OcrwFZJbwE/AvynHA7FEbrLJQ58Huz99IirgGZmbSfPS3NExFoqYZOtLc/cD+DWBn3nNqhPbVBfCixtMJR670UREd8FvtugT26qy/zsGtjHpM4xzX56M7OW4pUVCnAoiLzenJmZg6gI1fXmPvAUbjMzB1ERvN6cmdkXHEQFOLQCtz9LZGbmICrC+NJoxo8Z7dUVzMxwEBWmu1zypTkzMxxEhen2wqdmZoCDqDBeb87MrMJBVJCecsnTt83McBAVxu8RmZlVOIgK0lUu8en+g3y672DRQzEzK5SDqCA9mfXmzMzamYOoIIfWm/P7RGbW5hxEBemZUAmiD7zwqZm1OQdRQbo6K0HkKdxm1u4cRAXxCtxmZhUOooKcMr6DjlHyFG4za3u5BpGkmZK2SuqTdEed7ZK0LG1/VdLFmW0PS9oh6fWaPvdIejO1XyVpcqpfKellSa+ln5eneqekNanPFkl3Zfb145Kek7Qp7e+q3A5GDUl0+bNEZmb5BZGk0cADwCxgOjBX0vSaZrOAaem2AHgws+0RYGadXa8Dzo+IC4G3gMWp/j4wOyIuAOYDj2X63BsR5wIXAZdJmpXqvwM8GREXAdcB//kYftVj1t3pIDIzy/OM6BKgLyK2RcQ+4AlgTk2bOcCjUbEemCzpdICIeB7YVbvTiHgmIg6kh+uB3lTfFBHbU30LME7S2IgYiIjnUpt9wMZqHyCAU9L9SUC1f1N4dQUzs3yD6Azg3czj/lQbbpvB3AA8Vad+LbApIg6bG50u480Gnk2lbwL/SlI/sBb4t/WeRNICSRskbdi5c+cwhje47gkOIjOzPINIdWpxDG3q71xaAhwAHq+pnwfcDdxcU+8AVgLLImJbKs8FHomIXuAq4DFJRxyTiFgRETMiYsaUKVOGMrwh6fFXQZiZ0ZHjvvuBMzOPezny0tdQ2hxB0nzgauCKiIhMvRdYBcyLiHdquq0A3o6I+zK1G0nvQ0XEi5LGAacCO442hhOhq7PEnk/3c+Dg53SM9gRGM2tPef71ewmYJulsSSUqkwFW17RZDcxLs+cuBfZExHuD7VTSTOB24JqIGMjUJwNrgMUR8UJNn6VU3gO6rWZ3/w+4IrX5R8A44MRdezuK6uoKuwf2N+spzcxaTm5BlCYULAKeBt6gMjtti6SFkhamZmuBbUAf8BBwS7W/pJXAi8A5kvol3Zg23Q9MBNZJ2ixpeaovAqYCd6b6ZkmnpbOkJVRm7m1M9ZtSn98Efl3SK1Qu2/1a9gwrb4fWm/PlOTNrY3lemiMi1lIJm2xteeZ+ALc26Du3QX1qg/pSYGmDodR7L4qI+D5wWYM+uasGUWW9uYlFDcPMrFB+Y6JA1SDa/YkvzZlZ+3IQFeiLS3NegdvM2peDqEDVFbg9hdvM2pmDqEBjRo9i0vgxnqxgZm3NQVQwL/NjZu3OQVQwB5GZtTsHUcEcRGbW7hxEBfN6c2bW7hxEBesql9j9yT6auKCDmVlLcRAVrKdc4sDnwd7PDhy9sZnZCOQgKpjXmzOzducgKphXVzCzducgKtgXQeT15sysPTmICuYzIjNrdw6igvWUxwJeb87M2peDqGDjS6MZP2Y0uz52EJlZe3IQtYDucoldAw4iM2tPuQaRpJmStkrqk3RHne2StCxtf1XSxZltD0vaIen1mj73SHoztV8laXKqXynpZUmvpZ+Xp3qnpDWpzxZJd2X29YeZrxV/S9KHeR2LwXiZHzNrZ7kFkaTRwAPALGA6MFfS9Jpms4Bp6bYAeDCz7RFgZp1drwPOj4gLgbeAxan+PjA7Ii4A5gOPZfrcGxHnAhcBl0maBRAR/y4ivhwRXwb+CPjTY/ttj4+DyMzaWZ5nRJcAfRGxLSL2AU8Ac2razAEejYr1wGRJpwNExPPArtqdRsQzEVFdhmA90JvqmyJie6pvAcZJGhsRAxHxXGqzD9hY7VNjLrDyOH7fY9ZTLvGB3yMyszaVZxCdAbybedyfasNtM5gbgKfq1K8FNkXEYXOi02W82cCzNfWfAM4G/qrek0haIGmDpA07d+4cxvCGpqtcYrffIzKzNpVnEKlOrXZlz6G0qb9zaQlwAHi8pn4ecDdwc029g8oZz7KI2Fazu+uA70bEwXrPFRErImJGRMyYMmXKUIY3LN3lEgP7DvLZ/rpPb2Y2ouUZRP3AmZnHvcD2Y2hzBEnzgauB6yOzbLWkXmAVMC8i3qnptgJ4OyLuq7PL6yjoshxULs2BP0tkZu0pzyB6CZgm6WxJJSp/7FfXtFkNzEuz5y4F9kTEe4PtVNJM4HbgmogYyNQnA2uAxRHxQk2fpcAk4LY6+zsH6AJeHN6vd+IcWl3B7xOZWRvKLYjShIJFwNPAG8CTEbFF0kJJC1OztcA2oA94CLil2l/SSirhcI6kfkk3pk33AxOBdWna9fJUXwRMBe7MTMk+LZ0lLaEyc29jqt+UGepc4InsmVWzHQoiv09kZm2oI8+dR8RaKmGTrS3P3A/g1gZ95zaoT21QXwosbTCUeu9FVft9s9G2ZvF6c2bWzoZ0RiSpLGlUuv/Tkq6RNCbfobWPQ+vN+dKcmbWhoV6ae57K53LOoDL1+d9Q+cCpnQCnjO9g9Cj5Q61m1paGGkRKEwN+BfijiPhlKu+52Akgia5Of5bIzNrTkINI0s8A11OZmQY5v7/Ubry6gpm1q6EG0W1U1nRblWa+/STwXG6jakNeb87M2tWQzmoi4m+AvwFIkxbej4iv5zmwdtM9ocQb2/cWPQwzs6Yb6qy5P5F0iqQy8H1gq6Tfzndo7aW7099JZGbtaaiX5qZHxF7gq1Q+F/TjwL/Oa1DtqLtc4sOB/Rw4+HnRQzEza6qhBtGY9LmhrwJ/HhH7GeLipDY0PRMqH2rdPbC/4JGYmTXXUIPovwA/AMrA8+lrE/yGxglUXV3BU7jNrN0MKYgiYllEnBERV6Uvsftb4J/nPLa20t2ZVuD2FG4zazNDnawwSdK3ql8OJ+kPqJwd2QnSPaG63pyDyMzay1AvzT0MfAR8Ld32Av8tr0G1Iy98ambtaqirI/xURFybefzvJW3OYTxtq6uzGkSerGBm7WWoZ0SfSvrZ6gNJlwGf5jOk9jRm9ChOGdfhMyIzaztDPSNaCDwqaVJ6vBuYn8+Q2lfPhLH+unAzaztDXeLnFeBLkk5Jj/dKug14NcextR2vN2dm7WhYXxUeEXvTCgsA3zhae0kzJW2V1CfpjjrbJWlZ2v6qpIsz2x6WtEPS6zV97pH0Zmq/StLkVL9S0suSXks/L0/1TklrUp8tku6q2d/XJH0/bfuT4RyPE81BZGbtaFhBVKPh128DSBoNPADMovLdRXMl1X6H0SxgWrotAB7MbHsEmFln1+uA8yPiQuAtKquCA7wPzI6IC6hcNnws0+feiDgXuAi4TNKsNMZpqf9lEXEelVXGC9Pd6SAys/ZzPEF0tCV+LgH6ImJbROwDngDm1LSZAzyaPiS7Hpgs6XSAiHge2HXEk0Y8ExEH0sP1QG+qb4qI7am+hco3yo6NiIGIeC612QdsrPYBfh14ICJ2p+07hvrL56F7QuXL8SK8epKZtY9Bg0jSR5L21rl9BPzYUfZ9BvBu5nF/qg23zWBuAJ6qU78W2BQRh01BS5fxZlP5unOAnwZ+WtILktZLqncGhqQF1Q/z7ty5cxjDG56econ9B4O9nx04emMzsxFi0MkKETHxOPZd79Jd7f/qD6VN/Z1LS4ADwOM19fOAu4FfrKl3ACuBZRGxLZU7qFwW/AqVs6T/Len8iPjwsAFFrABWAMyYMSO305VD6819so9J48fk9TRmZi3leC7NHU0/cGbmcS+w/RjaHEHSfOBq4PrIXMeS1AusAuZFxDs13VYAb0fEfTXP/+cRsT8i/i+wlUowFaIrBZGncJtZO8kziF4Cpkk6W1IJuA5YXdNmNTAvzZ67FNgTEe8NttN0+ex24JqIGMjUJwNrgMUR8UJNn6XAJI6cjPBnpMVbJZ1K5VLdNgrSU/Z6c2bWfnILojShYBHwNPAG8GREbJG0UNLC1GwtlT/8fcBDwC3V/pJWAi8C50jql3Rj2nQ/MBFYJ2mzpOWpvgiYCtyZ6pslnZbOkpZQmbm3MdVvSn2eBj6Q9H3gOeC3I+KDPI7HUHi9OTNrR0NdWeGYRMRaKmGTrS3P3A/g1gZ95zaoT21QXwosbTCUulPN0/N/gyF8JqoZespjAa83Z2btJc9LczZM40ujGTdmlM+IzKytOIhaTE/Z682ZWXtxELUYL/NjZu3GQdRiussldjuIzKyNOIhaTHe55EtzZtZWHEQtxpfmzKzdOIhaTHe5xMC+g3y2/2DRQzEzawoHUYvx6gpm1m4cRC2my0FkZm3GQdRierzwqZm1GQdRi/F6c2bWbhxELcbrzZlZu3EQtZiJ4zoYPUo+IzKztuEgajGjRomuTn+WyMzah4OoBfWUS3zwsYPIzNqDg6gFdZdL7B5wEJlZe3AQtSCvN2dm7STXIJI0U9JWSX2S7qizXZKWpe2vSro4s+1hSTskvV7T5x5Jb6b2qyRNTvUrJb0s6bX08/JU75S0JvXZIumuzL5+TdLOzFeL30QL8HpzZtZOcgsiSaOBB4BZwHRgrqTpNc1mAdPSbQHwYGbbI8DMOrteB5wfERcCbwGLU/19YHZEXADMBx7L9Lk3Is4FLgIukzQrs+07EfHldPuvw/9NT7zucokPB/Zz4ODnRQ/FzCx3eZ4RXQL0RcS2iNgHPAHMqWkzB3g0KtYDkyWdDhARzwO7ancaEc9ExIH0cD3Qm+qbImJ7qm8BxkkaGxEDEfFcarMP2Fjt06p6JlQ+1Prhp/4skZmNfHkG0RnAu5nH/ak23DaDuQF4qk79WmBTRBz2YZx0GW828Gy2bbrM911JZ9Z7EkkLJG2QtGHnzp3DGN6x6er0enNm1j7yDCLVqcUxtKm/c2kJcAB4vKZ+HnA3cHNNvQNYCSyLiG2p/D+Bs9Jlvv8FfLvec0XEioiYEREzpkyZMpThHZdD6815CreZtYE8g6gfyJ5h9ALbj6HNESTNB64Gro+IyNR7gVXAvIh4p6bbCuDtiLivWoiIDzJnTQ8B//hoz90M3RN8RmRm7SPPIHoJmCbpbEkl4DpgdU2b1cC8NHvuUmBPRLw32E4lzQRuB66JiIFMfTKwBlgcES/U9FkKTAJuq6mfnnl4DfDG0H+9/Bxa+NSfJTKzNpBbEKUJBYuAp6n8gX8yIrZIWihpYWq2FtgG9FE5I7ml2l/SSuBF4BxJ/ZJuTJvuByYC69KU6+WpvgiYCtyZmY59WjpLWkJl5t7GmmnaX09Tul8Bvg78Wh7HYrgOvUfkS3Nm1gY68tx5RKylEjbZ2vLM/QBubdB3boP61Ab1pcDSBkOp914UEbGYL6Z/t4wxo0dxyrgOL3xqZm3BKyu0qJ4JY726gpm1BQdRi/J6c2bWLhxELaqr0ytwm1l7cBC1qB6vN2dmbcJB1KK6J1QuzWU+JmVmNiI5iFpUT7nE/oPBR/9w4OiNzcxOYg6iFuXPEplZu3AQtajqMj+ewm1mI52DqEVVFz7d7SAysxHOQdSiDq035yAysxHOQdSiqkHkS3NmNtI5iFpUZ6mDcWNGeb05MxvxHEQtrKc8ll2f+OvCzWxkcxC1sO5yyWdEZjbiOYhaWJeX+TGzNuAgamE95ZInK5jZiOcgamHd5ZI/R2RmI16uQSRppqStkvok3VFnuyQtS9tflXRxZtvDknZIer2mzz2S3kztV0manOpXSnpZ0mvp5+Wp3ilpTeqzRdJddcbxq5JC0owTfhCOQ3e5xCf7DvLZ/oNFD8XMLDe5BZGk0cADwCxgOjBX0vSaZrOAaem2AHgws+0RYGadXa8Dzo+IC4G3+OKrvt8HZkfEBcB84LFMn3sj4lzgIuAySbMy45wIfB343jH8mrnyh1rNrB3keUZ0CdAXEdsiYh/wBDCnps0c4NGoWA9MlnQ6QEQ8D+yq3WlEPBMR1SWp1wO9qb4pIran+hZgnKSxETEQEc+lNvuAjdU+yX8Efh/47Ph/5RPLQWRm7SDPIDoDeDfzuD/VhttmMDcAT9WpXwtsiojD5j6ny3izgWfT44uAMyPiLwZ7EkkLJG2QtGHnzp3DGN7x6XEQmVkbyDOIVKdW+y1vQ2lTf+fSEuAA8HhN/TzgbuDmmnoHsBJYFhHbJI0C/hD4zaM9V0SsiIgZETFjypQpQxneCeEzIjNrBx057rsfODPzuBfYfgxtjiBpPnA1cEVkvsJUUi+wCpgXEe/UdFsBvB0R96XHE4Hzgb+WBPCjwGpJ10TEhqONoRm83pyZtYM8z4heAqZJOltSCbgOWF3TZjUwL82euxTYExHvDbZTSTOB24FrImIgU58MrAEWR8QLNX2WApOA26q1iNgTEadGxFkRcRaV95taJoQAThk3htGj5NUVzGxEyy2I0oSCRcDTwBvAkxGxRdJCSQtTs7XANqAPeAi4pdpf0krgReAcSf2Sbkyb7qdyNrNO0mZJy1N9ETAVuDPVN0s6LZ0lLaEyc29jqt+U1+99Io0aJbo6S15vzsxGtDwvzRERa6mETba2PHM/gFsb9J3boD61QX0psLTBUOq9F1Xb/ytHa1OEHq83Z2YjnFdWaHFd5TGerGBmI5qDqMX1lMd6soKZjWgOohbn9ebMbKRzELW47nKJDz/dz8HPh/TxKjOzk46DqMV1l0tEwO4BnxWZ2cjkIGpxXl3BzEY6B1GL83pzZjbSOYhaXPcEB5GZjWwOohbX3en15sxsZHMQtbiu6qW5jx1EZjYyOYha3JjRozhlXIdnzZnZiOUgOgn0TPDqCmY2cjmITgJdnWO88KmZjVgOopNAd3ksH/g9IjMboRxEJ4GecsnvEZnZiOUgOgl0lUvs+mQfmW9FNzMbMRxEJ4Gecon9B4OP/uFA0UMxMzvhcg0iSTMlbZXUJ+mOOtslaVna/qqkizPbHpa0Q9LrNX3ukfRmar9K0uRUv1LSy5JeSz8vT/VOSWtSny2S7srsa2Fqv1nS/5E0PbeDcRy6/VkiMxvBcgsiSaOBB4BZwHRgbp0/9LOAaem2AHgws+0RYGadXa8Dzo+IC4G3gMWp/j4wOyIuAOYDj2X63BsR5wIXAZdJmpXqfxIRF0TEl4HfB751DL9q7g4t8+P3icxsBMrzjOgSoC8itkXEPuAJYE5NmznAo1GxHpgs6XSAiHge2FW704h4JiKq16jWA72pvikitqf6FmCcpLERMRARz6U2+4CNmT57M7suAy35Jkx1mR+fEZnZSJRnEJ0BvJt53J9qw20zmBuAp+rUrwU2RcRhH75Jl/FmA89mardKeofKGdHX6z2JpAWSNkjasHPnzmEM78TwV0GY2UiWZxCpTq32jGMobervXFoCHAAer6mfB9wN3FxT7wBWAssiYtuhJ4t4ICJ+Crgd+J16zxURKyJiRkTMmDJlylCGd0L1TPDCp2Y2cuUZRP3AmZnHvcD2Y2hzBEnzgauB6yMzp1lSL7AKmBcR79R0WwG8HRH3NdjtE8BXj/bcRegsdTBuzCh/lsjMRqQ8g+glYJqksyWVgOuA1TVtVgPz0uy5S4E9EfHeYDuVNJPK2cs1ETGQqU8G1gCLI+KFmj5LgUnAbTX1aZmH/wJ4e+i/XnN1d5a8uoKZjUi5BVGaULAIeBp4A3gyIrakKdMLU7O1wDagD3gIuKXaX9JK4EXgHEn9km5Mm+4HJgLr0rTr5am+CJgK3JnqmyWdls6SllCZubcx1W+q9klTujcD36Ay264ldU8oeb05MxuROvLceUSspRI22dryzP0Abm3Qd26D+tQG9aXA0gZDqfdeFBHxGw3at5zu8lhPVjCzEckrK5wkesolf47IzEYkB9FJoquz5M8RmdmIlOulOTtxeiaU+GTfQX7hW39T/zqjmVnOvn7FNGZ/6cdO+H4dRCeJXzrvR9n69x9x4PPPix6KmbWpSePH5LJfB9FJYuppE1g296Kih2FmdsL5PSIzMyuUg8jMzArlIDIzs0I5iMzMrFAOIjMzK5SDyMzMCuUgMjOzQjmIzMysUMp8r5wNgaSdwN8WPY4GTgXeL3oQg/D4jk+rjw9af4we3/E5nvH9RETU/YprB9EIImlDRMwoehyNeHzHp9XHB60/Ro/v+OQ1Pl+aMzOzQjmIzMysUA6ikWVF0QM4Co/v+LT6+KD1x+jxHZ9cxuf3iMzMrFA+IzIzs0I5iMzMrFAOopOMpDMlPSfpDUlbJP1GnTZfkbRH0uZ0+90mj/EHkl5Lz72hznZJWiapT9Krki5u4tjOyRyXzZL2Srqtpk1Tj5+khyXtkPR6ptYtaZ2kt9PPrgZ9Z0ramo7lHU0c3z2S3kz/fqskTW7Qd9DXQo7j+6akv8v8G17VoG/ux2+QMX4nM74fSNrcoG+ux7DR35SmvgYjwreT6AacDlyc7k8E3gKm17T5CvAXBY7xB8Cpg2y/CngKEHAp8L2Cxjka+HsqH7Qr7PgBPwdcDLyeqf0+cEe6fwdwd4PxvwP8JFACXql9LeQ4vl8EOtL9u+uNbyivhRzH903gt4bw75/78Ws0xprtfwD8bhHHsNHflGa+Bn1GdJKJiPciYmO6/xHwBnBGsaMatjnAo1GxHpgs6fQCxnEF8E5EFLpSRkQ8D+yqKc8Bvp3ufxv4ap2ulwB9EbEtIvYBT6R+uY8vIp6JiAPp4Xqg90Q/71A1OH5D0ZTjB4OPUZKArwEr83juoxnkb0rTXoMOopOYpLOAi4Dv1dn8M5JekfSUpPOaOzICeEbSy5IW1Nl+BvBu5nE/xYTpdTT+j7/I4wfwIxHxHlT+UACn1WnTKsfxBipnuPUc7bWQp0Xp0uHDDS4rtcrx+2fADyPi7Qbbm3YMa/6mNO016CA6SUmaAPwP4LaI2FuzeSOVy01fAv4I+LMmD++yiLgYmAXcKunnararTp+mfo5AUgm4BvjvdTYXffyGqhWO4xLgAPB4gyZHey3k5UHgp4AvA+9RufRVq/Djl8xl8LOhphzDo/xNaditTm3Yx9BBdBKSNIbKC+bxiPjT2u0RsTciPk731wJjJJ3arPFFxPb0cwewisrpe1Y/cGbmcS+wvTmjO2QWsDEifli7oejjl/ywerky/dxRp02hx1HSfOBq4PpIbxjUGsJrIRcR8cOIOBgRnwMPNXjewl+HkjqAXwG+06hNM45hg78pTXsNOohOMul68h8Db0TEtxq0+dHUDkmXUPl3/qBJ4ytLmli9T+VN7ddrmq0G5qniUmBP9RJAEzX8v9Aij1/GamB+uj8f+PM6bV4Cpkk6O53hXZf65U7STOB24JqIGGjQZiivhbzGl33P8ZcbPG9hxy/jF4A3I6K/3sZmHMNB/qY07zWY10wM33Kb4fKzVE59XwU2p9tVwEJgYWqzCNhCZQbLeuCfNnF8P5me95U0hiWpnh2fgAeozLZ5DZjR5GPYSSVYJmVqhR0/KoH4HrCfyv9h3gj0AM8Cb6ef3antjwFrM32vojLL6Z3qsW7S+PqovDdQfQ0urx1fo9dCk8b3WHptvUrlD+PpRR2/RmNM9Ueqr7tM26Yew0H+pjTtNeglfszMrFC+NGdmZoVyEJmZWaEcRGZmVigHkZmZFcpBZGZmhXIQmY0Akj4uegxmx8pBZGZmhXIQmZlZoRxEZmZWKAeRmZkVykFkZmaFchCZmVmhHERmI0OnpP7M7RtFD8hsqLz6tpmZFcpnRGZmVigHkZmZFcpBZGZmhXIQmZlZoRxEZmZWKAeRmZkVykFkZmaF+v/T9qRad1mcKQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Ls,losses)\n",
    "plt.xlabel(\"L\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.savefig(\"pic/rte_L.png\",dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "# iterations\n",
    "maxiter = 10\n",
    "step = 1\n",
    "Ns = np.array([i for i in range(step, maxiter+1,step)], dtype=int)\n",
    "losses_N = np.zeros(len(Ns))\n",
    "for i, N in enumerate(Ns):\n",
    "    print(N)\n",
    "    losses_N[i] = confusion_matrix_loss(get_confusion_matrix(\n",
    "        k, labels, sym=True, L=100, N=N, seed=0), C_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAERCAYAAABhKjCtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAgUklEQVR4nO3dfZBddZ3n8fenn5J09w0PSXMbwkOCk3sVGUCmh0WxUEQd4gqo60yFQsd1ZGOmcBzYGXd0tkp2nd2q3dKZmgcfslkGHWsRSxBGdg2QKccVRxekedIgJoQEQghJOgSTznM/fPePezrc3NzudKf73HPv7c+r6lbf+/v9zulvd0E+fc75nfNTRGBmZlapJesCzMysPjkgzMysKgeEmZlV5YAwM7OqHBBmZlaVA8LMzKpquoCQdIeknZLWzcC+rpL0VNnrkKQPTHLbN0r6f5IOS/rTCcbdKWm9pHVJ7e1J+42Sfp68firp4qT9HEk/lPSspGck/XHZvr4o6VfJNvdJOnV6v4Hx6zOz5td0AQF8A7hmJnYUET+MiEsi4hLgXcABYG3lOEkvVNl8N/Bp4Esn+DZ3Am8EfhOYB9yUtG8G3hERFwF/AaxO2oeBP4mINwGXAzdLuiDp+yfgwmSbDcDnJvFjnsh49ZlZk2u6gIiIhyn943yUpDdIelDS45J+LOmNJ7HrDwMPRMSBSdaxMyIeA4ZOMG5NJICfAWcn7T+NiNeSYY+Utb8SEU8k7weBZ4FFyee1ETFcuY2k1uTo4rHk6OKTk/2hx6vPzJpf0wXEOFYDfxQRvwX8KfDVk9jHcuCuGa2qTHLq5qPAg1W6PwE8UGWbxcBbgEerbPMHZdt8AtgTEb8N/Dbw7yQtmcH6zKwJtWVdQNokdQNvA+6WNNY8J+n7EPCFKpu9HBG/U7aPMymdYnmorO0rwBXJx7MkPZW8vzsi/utJlPpV4OGI+HFF/VdR+gf+7VV+ru8Ct0TE3oq+/0jpVNSdSdN7gYskfTj5fAqwVNIW4Olx6vl4cgQ0YX1m1ryaPiAoHSX9OrmOcIyIuBe4dxL7+D3gvog4erooIm4eey/phWr7nyxJtwE9wCcr2i8CbgeWRcSrZe3tlMLhzuRnKN/mY8D7gavj9QdtidIR1EMc78KTrc/MmlvTn2JK/rreLOl3AVRy8RR3cwMpnV6SdBPwO8ANETFa1n4upfD6aERsKGsX8PfAsxHxVxX7ugb4M+C6imslDwF/WDZDqiCpazr1mVnzU7M9zVXSXcA7gYXADuA24J+BrwFnAu3AtyOi2qmlavtbDPwEOGe8fyCTI4jFFW29QD8wHxgF9gEXRMReSWuAmyJim6Rh4EVgMNn03oj4gqTbgX+T9AEMR0SfpLcDPwZ+kewX4M8jYo2kjZROn40dbTwSESsltQD/BbiW0tHEAPCBiNgziZ+/an0n2s7MGl/TBYSZmc2Mpj/FZGZmJ6epLlIvXLgwFi9enHUZZmYN4/HHH98VET3V+poqIBYvXkx/f3/WZZiZNQxJL47X51NMZmZWlQPCzMyqckCYmVlVDggzM6vKAWFmZlWlOotJ0q2U1g8ISnf+fjwiDpX1fwa4sayWNwE9EbE7WWNhEBghuYs4zVrNzOxYqR1BSFpEacGcvoi4EGil9MjsoyLii2UL8nwO+FFElK/lcFXS73AwM6uxtE8xtQHzJLUBncC2Ccam9kC8iQyNjPLV/7uRhzcM1Ppbm5nVtdQCIiJeprTc5hbgFUoL1hy3XCeApE5Ky4R+t3wXwNpkFbgVadXZ1iJWP7yJB9ZtT+tbmJk1pDRPMZ0GXA8sAc4CuiR9ZJzh1wI/qTi9dEVEXAoso7Tu8pXjfJ8Vkvol9Q8MTP0oQBKFfI4NOwZPPNjMbBZJ8xTTu4HNETGQLLRzL6WV3ao5bjnPiNiWfN0J3AdcVm3DiFgdEX0R0dfTU/VxIidUzOfYsH0QP9nWzOx1aQbEFuBySZ3JIjdXA89WDpJ0CvAO4HtlbV2ScmPvKS2ZuS6tQgu9OQYPD/PKnkMnHmxmNkukNs01Ih6VdA/wBKX1kZ8EVktamfSvSoZ+EFgbEfvLNs8D9yVrSLcB34qIB9OqtZjPAbB+xyBnnTovrW9jZtZQUr0PIiJuo7SiW7lVFWO+AXyjom0TMNVlQU9aId8NwIbtg1xVPKNW39bMrK75Tmrg1M4O8vPnsN4Xqs3MjnJAJDyTyczsWA6IRDGf47kd+xgZ9UwmMzNwQBxV6M1xeHiULbsPZF2KmVldcEAkjs5k2u7TTGZm4IA4aunYTCZfhzAzAxwQR3V2tHHu6Z2eyWRmlnBAlCkkj9wwMzMHxDGKvd1s3rWfw8MjWZdiZpY5B0SZQj7H8Giwedf+Ew82M2tyDogyxV7PZDIzG+OAKHP+wm7aWuSZTGZmOCCO0dHWwpKFXazfvi/rUszMMueAqFDo9TOZzMzAAXGcYj7Hlt0HOHBkOOtSzMwy5YCoUEgeufHcDp9mMrPZLdWAkHSrpGckrZN0l6S5Ff3vlLRH0lPJ6/NlfddIWi9po6TPpllnuaMzmXyaycxmudQCQtIi4NNAX0RcCLQCy6sM/XFEXJK8vpBs2wp8BVgGXADcIOmCtGotd+7pncxpa/Ed1WY266V9iqkNmCepDegEtk1yu8uAjRGxKSKOAN8Grk+pxmO0toil+W4fQZjZrJdaQETEy8CXgC3AK8CeiFhbZehbJT0t6QFJb07aFgEvlY3ZmrQdR9IKSf2S+gcGBmak9kI+55vlzGzWS/MU02mU/upfApwFdEn6SMWwJ4DzIuJi4O+AfxzbvMouqy71FhGrI6IvIvp6enpmpPZiPsfOwcO8tv/IjOzPzKwRpXmK6d3A5ogYiIgh4F7gbeUDImJvROxL3q8B2iUtpHTEcE7Z0LOZ/OmpaSskF6p9P4SZzWZpBsQW4HJJnZIEXA08Wz5AUm/Sh6TLknpeBR4DlkpaIqmD0sXt+1Os9Rhjq8s5IMxsNmtLa8cR8aikeyidRhoGngRWS1qZ9K8CPgz8oaRh4CCwPCICGJb0KeAhSrOf7oiIZ9KqtdKZp8wlN6fNF6rNbFZLLSAAIuI24LaK5lVl/V8GvjzOtmuANelVNz5JpUdu+JlMZjaL+U7qcRR7c6zfMUjpgMbMbPZxQIyjmM+x5+AQOwcPZ12KmVkmHBDjGHsmk++HMLPZygExjkK+G/BMJjObvRwQ41jQPYeF3XN8BGFms5YDYgLF3m4fQZjZrOWAmEAhn2PDjn2Mjnomk5nNPg6ICRTzOQ4OjbD1tYNZl2JmVnMOiAkUvHiQmc1iDogJLD3DM5nMbPZyQEwgN7edRafO80wmM5uVHBAnUOzN+QjCzGYlB8QJFPI5nh/Yx9DIaNalmJnVlAPiBIq93QyNBC/s2p91KWZmNeWAOIGjz2TyaSYzm2VSDQhJt0p6RtI6SXdJmlvRf6Oknyevn0q6uKzvBUm/kPSUpP4065zIG3q6aRFs8IVqM5tlUgsISYuATwN9EXEhpZXhllcM2wy8IyIuAv4CWF3Rf1VEXBIRfWnVeSJz21tZvLDLRxBmNuukuqJcsv95koaATmBbeWdE/LTs4yPA2SnXc1KK+Ry/8hGEmc0yqR1BRMTLwJeALcArwJ6IWDvBJp8AHijfBbBW0uOSVoy3kaQVkvol9Q8MDMxE6ccp5HO88Op+Dg2NpLJ/M7N6lOYpptOA64ElwFlAl6SPjDP2KkoB8WdlzVdExKXAMuBmSVdW2zYiVkdEX0T09fT0zOjPMKbYmyMCNu70GtVmNnukeZH63cDmiBiIiCHgXuBtlYMkXQTcDlwfEa+OtUfEtuTrTuA+4LIUa52QV5czs9kozYDYAlwuqVOSgKuBZ8sHSDqXUnB8NCI2lLV3ScqNvQfeC6xLsdYJLV7QSUdri++oNrNZJbWL1BHxqKR7gCeAYeBJYLWklUn/KuDzwALgq6UMYTiZsZQH7kva2oBvRcSDadV6Im2tLbzhjG7PZDKzWSXVWUwRcRtwW0XzqrL+m4Cbqmy3Cbi4sj1LxXw3P9u8O+syzMxqxndST1KhN8e2PYfYe2go61LMzGrCATFJxeRC9XM+zWRms4QDYpJen8nkqa5mNjs4ICZp0anz6Opo9UwmM5s1HBCT1NIiluZzvhfCzGYNB8QUFPNeXc7MZg8HxBQUenO8uv8Iu/YdzroUM7PUOSCmYGwmk9eGMLPZwAExBYXebsCry5nZ7OCAmIKe7jmc1tnu6xBmNis4IKZAEgUvHmRms4QDYoqKvTk2bB8kIrIuxcwsVQ6IKSrkc+w/MsLLvz6YdSlmZqlyQEzRG3uTmUy+DmFmTc4BMUVL/UwmM5slUg0ISbdKekbSOkl3SZpb0S9Jfytpo6SfS7q0rO8aSeuTvs+mWedUnDKvnTNPmesjCDNreqkFhKRFwKeBvoi4EGgFllcMWwYsTV4rgK8l27YCX0n6LwBukHRBWrVOVcHPZDKzWSDtU0xtwDxJbUAnsK2i/3rgm1HyCHCqpDOBy4CNEbEpIo4A307G1oVib46NA/sYHhnNuhQzs9SkFhAR8TLwJWAL8AqwJyLWVgxbBLxU9nlr0jZe+3EkrZDUL6l/YGBgpsqfUCGf48jwKC/uPlCT72dmloU0TzGdRumv/iXAWUCXpI9UDquyaUzQfnxjxOqI6IuIvp6enumUPGl+JpOZzQZpnmJ6N7A5IgYiYgi4F3hbxZitwDlln8+mdBpqvPa68BtndCP5mUxm1tzSDIgtwOWSOiUJuBp4tmLM/cDvJ7OZLqd0GuoV4DFgqaQlkjooXdy+P8Vap2ReRyvnnd7pmUxm1tTa0tpxRDwq6R7gCWAYeBJYLWll0r8KWAO8D9gIHAA+nvQNS/oU8BCl2U93RMQzadV6MjyTycyaXWoBARARtwG3VTSvKusP4OZxtl1DKUDqUrE3xw9+tZNDQyPMbW/NuhwzsxnnO6lPUiGfY2Q02DSwP+tSzMxS4YA4SUU/k8nMmpwD4iQtXtBFe6s8k8nMmpYD4iR1tLVw/sJu3wthZk3LATENhd6cjyDMrGk5IKahmO9m62sH2Xd4OOtSzMxmnANiGgrJIzee81GEmTUhB8Q0eCaTmTUzB8Q0nHNaJ3PbW7y6nJk1JQfENLS0iEI+5yMIM2tKDohpKuQ9k8nMmpMDYpqK+RwDg4fZvf9I1qWYmc2oSQWEpC5JLcn7gqTrJLWnW1pjKPhCtZk1qckeQTwMzJW0CPgBpcdyfyOtohrJ0dXlHBBm1mQmGxCKiAPAh4C/i4gPAhekV1bjyM+fw/y5bV4bwsyazqQDQtJbgRuB7ydtE64lIako6amy115Jt1SM+UxZ/zpJI5JOT/pekPSLpK9/ij9XzUii2OuZTGbWfCa7YNAtwOeA+yLiGUnnAz+caIOIWA9cAiCpFXgZuK9izBeBLyZjrgVujYjdZUOuiohdk6wxM4V8jv/99DYigtLqqmZmjW9SRxAR8aOIuC4i/ntysXpXRHx6Ct/nauD5iHhxgjE3AHdNYZ91o9ibY++hYXbsPZx1KWZmM2ays5i+JWm+pC7gl8B6SZ+ZwvdZzgT/+EvqBK4BvlvWHMBaSY9LWjGF71VzY89k8v0QZtZMJnsN4oKI2At8gNI60ecCH53MhpI6gOuAuycYdi3wk4rTS1dExKXAMuBmSVeOs/8Vkvol9Q8MDEympBk3FhBeG8LMmslkA6I9ue/hA8D3ImKI0l/4k7EMeCIidkww5rgjjIjYlnzdSenaxWXVNoyI1RHRFxF9PT09kyxpZp3e1UFPbo6PIMysqUw2IP4H8ALQBTws6Txg7yS3nfDagqRTgHcA3ytr65KUG3sPvBdYN8nvl4min8lkZk1mshep/zYiFkXE+6LkReCqE22XXFt4D3BvWdtKSSvLhn0QWBsR+8va8sC/SHoa+Bnw/Yh4cDK1ZmXsoX2jo5M9sDIzq2+Tmuaa/JV/GzB2HeBHwBeAPRNtl9xct6CibVXF529QcVd2RGwCLp5MbfWi2NvNoaFRXnrtAOct6Mq6HDOzaZvsKaY7gEHg95LXXuDraRXViI7OZPKFajNrEpMNiDdExG0RsSl5/Wfg/DQLazRL/UwmM2sykw2Ig5LePvZB0hXAwXRKakzdc9o45/R5/MpHEGbWJCb7qI2VwDeTaxEArwEfS6ekxuWZTGbWTCY7i+npiLgYuAi4KCLeArwr1coaUCGfY9PAfo4Mj2ZdipnZtE1pRbmI2JvcUQ3w71Oop6EVe3MMjwabd+0/8WAzszo3nSVH/djSCn4mk5k1k+kEhO8Iq3B+TxetLfIzmcysKZxo0Z9BqgeBgHmpVNTA5rS1smRhl48gzKwpTBgQEZGrVSHNopjPsW7bhDeYm5k1hOmcYrIqCvkcW3Yf4MCR4axLMTObFgfEDCv2dhMBG3fuy7oUM7NpcUDMMD+TycyahQNihp23oIuOthbfUW1mDc8BMcNaW8TSM7pZv8OnmMyssTkgUlDM53wvhJk1vNQCQlJR0lNlr72SbqkY805Je8rGfL6s7xpJ6yVtlPTZtOpMQ6E3x/a9h9hzYCjrUszMTtpkn+Y6ZRGxHrgEQFIr8DJwX5WhP46I95c3JOO/Qmm50q3AY5Luj4hfplXvTCqOrQ2xc5DfXnx6xtWYmZ2cWp1iuhp4PlnLejIuAzYmixMdAb4NXJ9adTOs0OuZTGbW+GoVEMuBu8bpe6ukpyU9IOnNSdsi4KWyMVuTtuNIWiGpX1L/wMDAzFU8DWedMpfuOW2eyWRmDS31gJDUAVwH3F2l+wngvGStib8D/nFssypjqz4cMCJWR0RfRPT19PTMQMXTJ4lCvttHEGbW0GpxBLEMeCIidlR2JOtL7EverwHaJS2kdMRwTtnQs4FtNah1xhR7S6vLRfiht2bWmGoREDcwzuklSb2SlLy/LKnnVeAxYKmkJckRyHLg/hrUOmMK+RyvHRhiYN/hrEsxMzspqc1iApDUSWkm0ifL2lYCRMQq4MPAH0oaBg4Cy6P0J/ewpE8BDwGtwB0R8Uyatc60ozOZtu/jjNzcjKsxM5u6VAMiIg4ACyraVpW9/zLw5XG2XQOsSbO+NB2dybRjkLcvXZhxNWZmU+c7qVOysHsOC7o6fEe1mTUsB0SKCvmcV5czs4blgEhRsTfHczsGGR31TCYzazwOiBQV8jn2Hxnh5V8fzLoUM7Mpc0CkqNjbDeA7qs2sITkgUrQ0//pMJjOzRuOASNH8ue2cdcpcz2Qys4bkgEhZoTfn1eXMrCE5IFJWzOd4fuc+hkdGsy7FzGxKHBApK+RzHBkZ5YVXD2RdipnZlDggUlZMHrnhmUxm1mgcECn7jTO6kby6nJk1HgdEyua2t7J4QZePIMys4TggaqCQ7/a9EGbWcBwQNVDM53hh134ODY1kXYqZ2aSlFhCSipKeKnvtlXRLxZgbJf08ef1U0sVlfS9I+kWybX9addZCsXc+owHPD/h+CDNrHKktGBQR64FLACS1Ai8D91UM2wy8IyJek7QMWA38q7L+qyJiV1o11srYM5nWbx/kzWedknE1ZmaTk+qKcmWuBp6PiBfLGyPip2UfHwHOrlE9NXXegi46Wlt8HcLMGkqtrkEsB+46wZhPAA+UfQ5graTHJa0YbyNJKyT1S+ofGBiYgVJnXntrC+f3dPmZTGbWUFIPCEkdwHXA3ROMuYpSQPxZWfMVEXEpsAy4WdKV1baNiNUR0RcRfT09PTNY+cwq9ubY4GcymVkDqcURxDLgiYjYUa1T0kXA7cD1EfHqWHtEbEu+7qR07eKyGtSamkI+x8u/PsjgoaGsSzEzm5RaBMQNjHN6SdK5wL3ARyNiQ1l7l6Tc2HvgvcC6GtSammJ+7JEbPoows8aQakBI6gTeQykExtpWSlqZfPw8sAD4asV01jzwL5KeBn4GfD8iHkyz1rT5mUxm1mhSncUUEQcoBUB526qy9zcBN1XZbhNwcWV7I1t06jw6O1r9TCYzaxi+k7pGWlrE0nzORxBm1jAcEDVUzHc7IMysYTggaqiQz7Fr3xF27TucdSlmZifkgKghX6g2s0bigKiho1NdfaHazBqAA6KGenJzOLWznfW+F8LMGoADooYkUfBMJjNrEA6IGivmc2zYPkhEZF2KmdmEHBA1VujNMXh4mFf2HMq6FDOzCTkgamzsQrXXhjCzeueAqLFCvrS6nGcymVm9c0DU2KmdHeTnz/ERhJnVPQdEBjyTycwagQMiA8V8jud27GNk1DOZzKx+OSAyUOjNcXh4lC27D2RdipnZuFILCEnFZBGgsddeSbdUjJGkv5W0UdLPJV1a1neNpPVJ32fTqjMLR2cy+UK1mdWx1AIiItZHxCURcQnwW8ABSmtLl1sGLE1eK4CvAUhqBb6S9F8A3CDpgrRqrbWlYzOZfB3CzOpYrU4xXQ08HxEvVrRfD3wzSh4BTpV0JnAZsDEiNkXEEeDbydim0NnRxrmnd3omk5nVtVoFxHLgrirti4CXyj5vTdrGaz+OpBWS+iX1DwwMzFC56Sskj9wwM6tXqQeEpA7gOuDuat1V2mKC9uMbI1ZHRF9E9PX09Jx8oTVW7O1m8679HB4eyboUM7OqanEEsQx4IiJ2VOnbCpxT9vlsYNsE7U2jkM8xPBps3rU/61LMzKqqRUDcQPXTSwD3A7+fzGa6HNgTEa8AjwFLJS1JjkCWJ2Obxtjqcp7JZGb1qi3NnUvqBN4DfLKsbSVARKwC1gDvAzZSmuX08aRvWNKngIeAVuCOiHgmzVpr7fyF3bS1yDOZzKxupRoQEXEAWFDRtqrsfQA3j7PtGkoB0pQ62lpYsrCL9du9upyZ1SffSZ2hQq+fyWRm9csBkaFiPseW3Qc4cGQ461LMzI7jgMhQIXnkxnM7fJrJzOqPAyJDR2cy+TSTmdUhB0SGzj29k7ntLb6j2szqkgMiQ60tYukZOR9BmFldckBkzKvLmVm9ckBkrNjbzY69h/n1gSNZl2JmdgwHRMYKXjzIzOqUAyJjYzOZfJrJzOqNAyJjvfPnkpvb5gvVZlZ3HBAZk0Qxn2ODn8lkZnXGAVEHCr2lqa6lZxeamdUHB0QdKOZz7Dk4xM7Bw1mXYmZ2lAOiDngmk5nVo1QDQtKpku6R9CtJz0p6a0X/ZyQ9lbzWSRqRdHrS94KkXyR9/WnWmbVCvhvwTCYzqy+pLhgE/A3wYER8OFk6tLO8MyK+CHwRQNK1wK0RsbtsyFURsSvlGjO3oHsOC7vn+AjCzOpKagEhaT5wJfBvASLiCDDR7cITrV3d9Iq93T6CMLO6kuYppvOBAeDrkp6UdLukrmoDk7WrrwG+W9YcwFpJj0taMd43kbRCUr+k/oGBgZmsv6ZKz2Tax+ioZzKZWX1IMyDagEuBr0XEW4D9wGfHGXst8JOK00tXRMSlwDLgZklXVtswIlZHRF9E9PX09Mxg+bVVzOc4ODTC1tcOZl2KmRmQ7jWIrcDWiHg0+XwP4wfEcipOL0XEtuTrTkn3AZcBD6dUa+YKySM3bvifjzB/Xjtz2lqY09ZCR1sLc9pamdPekrS1Hu2b09bCnPZWOlpbjunvaCsb216xn4r9trcKSRn/9GZWj1ILiIjYLuklScWIWA9cDfyycpykU4B3AB8pa+sCWiJiMHn/XuALadVaD35z0Sn8wRVL2DF4iCPDoxweHuXw0AiDh4bZNXyEI8Mjpbak/chI6f10762TOC5YOtpamInImIngcXSZndhpnR18Z+VbTzxwitKexfRHwJ3JDKZNwMclrQSIiFXJmA8CayNif9l2eeC+5B+YNuBbEfFgyrVmqr21hc9fe8GUtokIhkaCw0l4HA2W4REOD42WQmRo9Gj/4eGRsvCp+DxcPm50+j/QDFxKiZnYidksMH9ueyr7VTM93qGvry/6+5v6lgkzsxkl6fGI6KvW5zupzcysKgeEmZlV5YAwM7OqHBBmZlaVA8LMzKpyQJiZWVUOCDMzq8oBYWZmVTXVjXKSBoAXs65jmhYCTb8GxiT5d3Es/z6O5d/H66bzuzgvIqo+6bSpAqIZSOof767G2ca/i2P593Es/z5el9bvwqeYzMysKgeEmZlV5YCoP6uzLqCO+HdxLP8+juXfx+tS+V34GoSZmVXlIwgzM6vKAWFmZlU5IOqApHMk/VDSs5KekfTHWdeUNUmtkp6U9H+yriVrkk6VdI+kXyX/jcz82pINRNKtyf8n6yTdJWlu1jXVkqQ7JO2UtK6s7XRJ/yTpueTraTPxvRwQ9WEY+JOIeBNwOXCzpKmtP9p8/hh4Nusi6sTfAA9GxBuBi5nFvxdJi4BPA30RcSHQCizPtqqa+wZwTUXbZ4EfRMRS4AfJ52lzQNSBiHglIp5I3g9S+gdgUbZVZUfS2cC/Bm7PupasSZoPXAn8PUBEHImIX2daVPbagHmS2oBOYFvG9dRURDwM7K5ovh74h+T9PwAfmInv5YCoM5IWA28BHs24lCz9NfAfgNGM66gH5wMDwNeTU263S+rKuqisRMTLwJeALcArwJ6IWJttVXUhHxGvQOkPTuCMmdipA6KOSOoGvgvcEhF7s64nC5LeD+yMiMezrqVOtAGXAl+LiLcA+5mh0weNKDm3fj2wBDgL6JL0kWyral4OiDohqZ1SONwZEfdmXU+GrgCuk/QC8G3gXZL+V7YlZWorsDUixo4o76EUGLPVu4HNETEQEUPAvcDbMq6pHuyQdCZA8nXnTOzUAVEHJInSOeZnI+Kvsq4nSxHxuYg4OyIWU7r4+M8RMWv/QoyI7cBLkopJ09XALzMsKWtbgMsldSb/31zNLL5oX+Z+4GPJ+48B35uJnbbNxE5s2q4APgr8QtJTSdufR8Sa7EqyOvJHwJ2SOoBNwMczriczEfGopHuAJyjN/nuSWfbIDUl3Ae8EFkraCtwG/DfgO5I+QSlEf3dGvpcftWFmZtX4FJOZmVXlgDAzs6ocEGZmVpUDwszMqnJAmJlZVQ4Is5RICkl/Wfb5TyX9pwxLMpsSB4RZeg4DH5K0MOtCzE6GA8IsPcOUbuK6NetCzE6GA8IsXV8BbpR0StaFmE2VA8IsRclTeb9JaZEbs4bigDBL318DnwBm7ToO1pgcEGYpi4jdwHcohYRZw3BAmNXGXwKezWQNxU9zNTOzqnwEYWZmVTkgzMysKgeEmZlV5YAwM7OqHBBmZlaVA8LMzKpyQJiZWVX/H/VZ2wQM0mzVAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Ns, losses_N)\n",
    "plt.xlabel(\"N\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.savefig(\"pic/rte_N.png\", dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00974306, 0.00974306, 0.00974306, 0.00974306, 0.00974306,\n",
       "       0.00974306, 0.00974306, 0.00974306, 0.00974306, 0.00974306])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses_N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "19\n",
      "29\n",
      "39\n",
      "49\n",
      "59\n",
      "69\n",
      "79\n",
      "89\n",
      "99\n",
      "109\n",
      "119\n",
      "129\n",
      "139\n",
      "149\n",
      "159\n",
      "169\n",
      "179\n",
      "189\n",
      "199\n",
      "209\n",
      "219\n",
      "229\n",
      "239\n",
      "249\n",
      "259\n",
      "269\n",
      "279\n",
      "289\n",
      "299\n",
      "309\n",
      "319\n",
      "329\n",
      "339\n",
      "349\n",
      "359\n",
      "369\n",
      "379\n",
      "389\n",
      "399\n",
      "409\n",
      "419\n",
      "429\n",
      "439\n",
      "449\n",
      "459\n",
      "469\n",
      "479\n",
      "489\n",
      "499\n"
     ]
    }
   ],
   "source": [
    "# symmetric or not\n",
    "iternum = 500\n",
    "sym_losses = np.zeros(iternum)\n",
    "asym_losses = np.zeros(iternum)\n",
    "for i in range(iternum):\n",
    "    if i%10==9:\n",
    "        print(i)\n",
    "    sym_losses[i] = confusion_matrix_loss(get_confusion_matrix(\n",
    "        k, labels, sym=True, L=20, N=10), C_true)\n",
    "    asym_losses[i] = confusion_matrix_loss(get_confusion_matrix(\n",
    "        k, labels, sym=False, L=20, N=10), C_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0066569608477419105 0.36679272744525226\n",
      "0.006627327623297769 0.26380978133865685\n"
     ]
    }
   ],
   "source": [
    "print(np.min(sym_losses),np.max(sym_losses))\n",
    "print(np.min(asym_losses), np.max(asym_losses))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.24893358557393622"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.percentile(np.append(sym_losses, asym_losses),99)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjzUlEQVR4nO3deZxU1Zn/8c9XbEEFRREdEAxgJKOIgCA/zSbGjURxiSEhY6I4EmJGE2Oiccmmv+iYmbivjAmKW+IWUcYsM+CSRVACplEENaBEtmiLImAiCjzzxz3dFk11d3VXVzdcvu/Xq15dde+59zzn1q2nzj331m1FBGZmli/btHcAZmbW+pzczcxyyMndzCyHnNzNzHLIyd3MLIec3M3McsjJfTOjzG2S3pI0sw3r/Zikv0haI+mEMtbzG0mntmJo7UbSBEnfb8P6TpS0OL0HQ9qq3lT3CElLGpk/SdKlbVBPs7Z5U+srUv4JSeNKLb8l2+qTu6RFkt6TtFu96dWSQlKf9LqXpF9KekPS25KekzQ2zeuTyq6p9/hCC0L6OHAk0CsihpfZvOb4/8ANEdE5Ih5q6Uoi4tMRcXvrhdX6JI2V9MemykXEGRHxo7aIKbkCOCu9B39uw3o3G+2wzRuUcsMR7R1HS23b3gFsJl4BvghcDyBpILB9vTJ3AnOADwFrgYHAP9Ur0zUi1pUZy4eARRHxTpnraUm9z7dxnZstSR0iYn0bV9vgeyBp21bYt7Zo3gbNs9X33JM7gVMKXp8K3FGvzEHApIh4JyLWRcSfI+I3LalMUk9JUyS9KWmBpK+k6acDPwMOST3/SxpY/iuS5ktaLWmepAPT9H3TYedKSc9LOq5gmUmSbpT0q7Tc05L2TvMWAv2A/071dqzfa5F0saS70vNOku6StCLV9SdJe6R5dYe9kraR9D1Jf5X0uqQ7JO2c5tUe7Zwq6dV0RPTdRrbZJEk3pWGfNZKelPRPkq5JQ1gvFA5lSLpA0sKCbXRi7TYCJhRs45UF679Z0q8lvQMcVn8oQtLx6YhuVVr3yDR9Z0kTJS2XtFTSpZI6pHkflvS7dLT3hqR7i7Sto6Q1QAdgTno/anuO50t6FnhH0raSjkvv7cq0rfctWM8iSedJelbSOymmPdI2Wy1pmqRdGtrGaR0XpTgXSTq5gTKbHPmk9/LDBe25Ir2vrykbatm+Xvmi9RRuc6Uhl7QN/gbcJmn7VOYtSfPIPpeNtefItG+8LekGQAXz9pb0WNqP35B0t6Suad6dwF588Jn4Tpp+v6S/pfX9XtKAxupvVxGxVT+ARcARwIvAvmQfsMVkvagA+qRy04AngTHAXvXW0SeV3bbEOn8H3AR0AgYDNcDhad5Y4I+NLDsaWEq2Uwv4cIq1ClgAXARsB3wKWA18JC03CXgTGE52xHY3cE/97dDI64uBu9LzrwL/DeyQttdQYKc07wlgXHr+rymmfkBn4EHgznrb7KdkR0mDyI6I9m2g3ZOAN1JdnYDHyI64TkkxXAo8Xm879STrwHwBeAfo0dA2Tut/G/hYWqZTmnZpmj88zT8yzd8T+Oc07yHgv4Adgd2BmcBX07xfAN8tWOfHG3lvA/hwvfegGuidtlH/1I4j0/v9nbR9tyso/xSwR4rvdeAZYAjQMW2zHzZQ9whgHXBVKntoqqtw/7m0ke1XFztwDTAF2BXoQravXN6CemrL/kcquz3wY+APad29gbnAkgbatBuwCvhc2l7npPXV7p8fTtuyI9Ad+D1wTUOfgYJ9ukta5hqgur1zWEMP99w/UNt7PxJ4gSyBFhpNtlN9H3gl9eDq9xreSD2q2se+9eYjqTfZuPr5EfFuRFST9da/XGKc44D/jIg/RWZBRPwVOJgsgf44It6LiMeAR8iGm2o9GBEzIzu0vZvsi6Ul3ge6kX2Y10fE7IhYVaTcycBVEfFyRKwBLgTGSCocDrwkIv4REXPIhr0GNVLv5FTXu8Bk4N2IuCOy4ZN7yZIYABFxf0Qsi4gNEXEv8BeyBN2YhyPiybTMu/XmnQ7cGhFT0/ylEfGCsiOWTwPfjOyo7nXgarJOQO22+hDQM73fTY7113NdRCyOiH+QfUn9KsXwPtkY/fbARwvKXx8Rr0XEUrL99enIjjLXpm3W1Ina70fE2oj4HfAr4PPNCVaSgK8A50TEmxGxGvh3Ptgeza1nA9kX0tq0DT4PXJbWvRi4rpFwPgPMi4gH0va6Bvhb7cz02Zma1l1D9oVzaGPti4hbI2J12p4XA4OUjkY3N07uH7gT+BeyXkn9IRki4q2IuCAiBpD1jKqBh9LOXGu3iOha8JhfpJ6eQO1OX+uvZD2tUvQGFjaw3sURsaGR9f6t4Pnfyb4MWuJO4H+AeyQtk/SfkqoaiOmv9eLZlmz7tSSm1wqe/6PI67plJZ2SvoBXpqGX/cl6co1Z3Mi8hrZ77VHT8oK6/ousBw9Z71rAzDSc8q9NxNBYTBttz/ReL2bj97jkbVTEW7HxuZ6/pjqbozvZEd3sgu3x2zS9JfXU1Pui7cnG2+SvNGyjspF1veteS9pd0j1pKG0VcBeN7COSOkj6cRqSW0XWs6exZdqTk3uSer+vkH3bP9hE2TfIek09yQ4Pm2MZsKukLgXT9mLTI4WGLAb2bmC9vSUVvqfNWW9975B9SGvVnTyOiPcj4pKI2I+s13gsG5+zKIzpQ/XiWcfGCafVSfoQ2XDPWUC3iOhKdvhe+0Xc0K1QG7tFakPbfTHZcFLhF/tOqRNARPwtIr4SET3JhrNuqh2bLlFhTBttz9Sx6E3L3+P6dpG0Y8HrvVKd9W20b0gqvLDgDbIvkQEF22PniCj8Uim1Htj0PVlO1ubCZRuyUdmC7VXr8rT+AyJiJ+BLFIzJF6n7X4DjyYZxdyYbWqTeMpsNJ/eNnQ58KopcqSLpPyTtn05qdQG+BiyIiBXNqSAdSk4HLld2YvKAVO/dJa7iZ8C5koYq8+GUzJ4m+9B9R1KVpBHAKOCe5sRXoJpsCKVK0jCycUsAJB0maaCyk4aryIYeil1Z8gvgHEl9JXUmOzy/Nyp/xcOOZB/MmhTvaWQ991qvAb0kbdeMdU4ETpN0uLITxXtK+ueIWA78L3ClpJ3SvL0lHZrqHi2pV1rHWymull6Fcx9wTIqhCvg22RfL9Baur5hLJG0n6RNkX9r3FykzBxggabCkTmTDE0Dd0cRPgasl7Q6QttXRLainmPuACyXtkrbr1xsp+6sU52fTUOA32PgKty7AGmClpD2B8+ot/xrZ+aLC8muBFWRfbv9eYsztwsm9QEQsjIhZDczegWzMciXwMlkP6rh6ZVZq4+vcv9XAur5I9q2/LK3zhxExtcQY7wcuA35OdsL0IWDXiHgvxfNpst7TTcApEfFCKest4vtkPdW3gEtSfbX+CXiALLHPJztBfFeRddxKNoTze7Kjondp/MPYKiJiHnAlMIPsAzqQ7GR4rcfILjn8m6Q3SlznTOA0svH0t8naXNuLPoXsJPY8su31ANAjzTsIeFrZ1TBTgLMj4pUWtutFst7l9WTv8ShgVHrvW8PfyOJfRtbZOKPY/hMRL5H9LmIa2bmM+ucRzic70ftUGr6YBnykufU04BKyoZhXyL5U72yoYDrCHk12EnYFsA8b7weXAAeSvZ+/YtMj9suB76XhpXPJhmv/SnakNI/s5PVmS9kwlJmZ5Yl77mZmOeTkbmaWQ00m93TSb6akOelSrkvS9IvTJUTV6fGZgmUuVPbLyxeLnEgxM7MKa3LMPV0+tGNErEln6P8InA2MBNZExBX1yu9HdpXEcLJLBacB/aPt79NhZrbVavLGYenC/zXpZVV6NPaNcDzZz9rXkv2ScwFZop/R0AK77bZb9OnTp9SYzcwMmD179hsR0b3YvJLuCpmuZ55Ndi+GGyPiaUmfBs6SdAowC/h2RLxF9mu5wkuEllDk15eSxgPjAfbaay9mzWroCkQzMytGUoO/0C3phGq6f8hgoBcwXNL+wM1k10EPJvsl2JW19RVbRZF13hIRwyJiWPfuRb94zMyshZp1tUxErCS769/IdHOi9QW/SKu9KdMSNv6Jby8a/mmxmZlVQClXy3TXB/c43p7svgovSOpRUOxEsnt3QPYrvDHK7uncl+xXYW327+LMzKy0MfcewO1p3H0b4L6IeETSnZIGkw25LCK7KRIR8byk+8h+nrsOONNXypjlw/vvv8+SJUt49936d0S2SurUqRO9evWiqqrYzVeL2yxuPzBs2LDwCVWzzd8rr7xCly5d6NatG9JmeTPE3IkIVqxYwerVq+nbt+9G8yTNjohhxZbzL1TNrGTvvvuuE3sbk0S3bt2afbTk5G5mzeLE3vZass2d3M3McqikHzGZmRVz9dSXWnV95xzZv1XX1xYeeugh+vfvz3777Vd0/oQJE9hhhx045ZRi/6yscrbO5P745c1f5rALWz8OM9viPfTQQxx77LFFk/u6des444wz2iEqD8uY2RbmnXfe4ZhjjmHQoEHsv//+3HvvvZx44ol186dOncpnP/tZADp37sz555/P0KFDOeKII5g5cyYjRoygX79+TJkyBYBJkyZxwgknMGrUKPr27csNN9zAVVddxZAhQzj44IN58803AVi4cCEjR45k6NChfOITn+CFF15g+vTpTJkyhfPOO4/BgwezcOFCRowYwUUXXcShhx7Ktddey8UXX8wVV2T3V1ywYAFHHHEEgwYN4sADD2ThwmL/c711OLmb2Rblt7/9LT179mTOnDnMnTuXkSNHMn/+fGpqagC47bbbOO2004Dsi2DEiBHMnj2bLl268L3vfY+pU6cyefJkfvCDH9Stc+7cufz85z9n5syZfPe732WHHXbgz3/+M4cccgh33HEHAOPHj+f6669n9uzZXHHFFfzbv/0bH/3oRznuuOP4yU9+QnV1NXvvnf0P9ZUrV/K73/2Ob3/72xvFfvLJJ3PmmWcyZ84cpk+fTo8ePaiUrXNYxsy2WAMHDuTcc8/l/PPP59hjj+UTn/gEX/7yl7nrrrs47bTTmDFjRl1C3m677Rg5cmTdch07dqSqqoqBAweyaNGiunUedthhdOnShS5durDzzjszatSoumWeffZZ1qxZw/Tp0xk9enTdMmvXrm0wxi984QubTFu9ejVLly6tO8ro1KlT2duiMU7uZrZF6d+/P7Nnz+bXv/41F154IUcddRTjxo1j1KhRdOrUidGjR7Pttllqq6qqqruMcJtttqFjx451z9etW1e3ztrpDZXbsGEDXbt2pbq6uqQYd9xxx02mtfUPRj0sY2ZblGXLlrHDDjvwpS99iXPPPZdnnnmGnj170rNnTy699FLGjh3b6nXutNNO9O3bl/vvvx/IEvWcOXMA6NKlC6tXry5pHb169eKhhx4Csp7/3//+91aPtZZ77mbWYu1x6eJzzz3HeeedxzbbbENVVRU333wzkI1n19TUNHhJYrnuvvtuvva1r3HppZfy/vvvM2bMGAYNGsSYMWP4yle+wnXXXccDDzzQ6DruvPNOvvrVr/KDH/yAqqoq7r//fvr161eReLfOe8v4UkizFpk/fz777rtve4dR1FlnncWQIUM4/fTT2zuUiii27Ru7t4x77ma2xRs6dCg77rgjV155ZdOFtxJO7ma2xZs9e3Z7h7DZ8QlVM7MccnI3M8shJ3czsxxycjczyyGfUDWzlmvJZcWNydElx0888QTbbbcdH/3oR4vOnzJlCvPmzeOCCy6oSP1O7mZmFfDEE0/QuXPnosl93bp1HHfccRx33HEVq9/DMma2xTnhhBMYOnQoAwYM4JZbbmH9+vWMHTuW/fffn4EDB3L11VezcOFCDjzwwLpl/vKXvzB06FAA+vTpw0UXXcQhhxzCsGHDeOaZZzj66KPZe++9mTBhApAl50MPPZTPf/7z9O/fnwsuuIC7776b4cOHM3DgwLrb9dbU1HDSSSdx0EEHcdBBB/Hkk0+yaNEiJkyYwNVXX83gwYP5wx/+wNixY/nWt77FYYcdxvnnn8+kSZM466yzAHjttdc48cQTGTRoEIMGDWL69OllbyP33M1si3Prrbey66678o9//IODDjqIoUOHsnTpUubOnQtkt9zt2rUrO++8M9XV1QwePJjbbrtto/vO9O7dmxkzZnDOOecwduxYnnzySd59910GDBhQ9w825syZw/z589l1113p168f48aNY+bMmVx77bVcf/31XHPNNZx99tmcc845fPzjH+fVV1/l6KOPZv78+Zxxxhl07tyZc889F4CJEyfy0ksvMW3aNDp06MCkSZPqYvnGN77BoYceyuTJk1m/fj1r1qwpexs1mdwldQJ+D3RM5R+IiB9K2hW4F+gDLAI+HxFvpWUuBE4H1gPfiIj/KTtSM7PkuuuuY/LkyQAsXryY9957j5dffpmvf/3rHHPMMRx11FEAjBs3jttuu42rrrqKe++9l5kzZ9ato3ZIZODAgaxZs6bulr+dOnVi5cqVABx00EF191zfe++969Y7cOBAHn/8cQCmTZvGvHnz6ta7atWqBm8kNnr0aDp06LDJ9Mcee6zuNsUdOnRg5513bvG2qVXKsMxa4FMRMQgYDIyUdDBwAfBoROwDPJpeI2k/YAwwABgJ3CRp09aYmbXAE088wbRp05gxYwZz5sxhyJAhrF27ljlz5jBixAhuvPFGxo0bB8BJJ53Eb37zGx555BGGDh1Kt27d6tZTeFvf+rf8rb0dcFO3AgbYsGEDM2bMoLq6murqapYuXUqXLl2Kxl7sVsCV0mRyj0ztMUJVegRwPHB7mn47cEJ6fjxwT0SsjYhXgAXA8NYM2sy2Xm+//Ta77LILO+ywAy+88AJPPfUUb7zxBhs2bOCkk07iRz/6Ec888wyQ/UOMo48+mq997Wt1/52ptR111FHccMMNda9r7/le6q2AAQ4//PC6u1uuX7+eVatWlR1XSWPuqec9G/gwcGNEPC1pj4hYDhARyyXtnorvCTxVsPiSNK3+OscD4wH22muvlrfAzNpPO1y6OHLkSCZMmMABBxzARz7yEQ4++GCWLl3KiBEj2LBhAwCXX/7BJZonn3wyDz74YN2QSmu77rrrOPPMMznggANYt24dn/zkJ5kwYQKjRo3ic5/7HA8//DDXX399o+u49tprGT9+PBMnTqRDhw7cfPPNHHLIIWXF1axb/krqCkwGvg78MSK6Fsx7KyJ2kXQjMCMi7krTJwK/johfNrRe3/LXbMuwOd/ytyFXXHEFb7/9Nj/60Y/aO5SyVPSWvxGxUtITZGPpr0nqkXrtPYDXU7ElQO+CxXoBy5pTj5lZazjxxBNZuHAhjz32WHuH0uaaHHOX1D312JG0PXAE8AIwBTg1FTsVeDg9nwKMkdRRUl9gH2AmZmZtbPLkyTz77LPstttu7R1Kmyul594DuD2Nu28D3BcRj0iaAdwn6XTgVWA0QEQ8L+k+YB6wDjgzItZXJnwza2sRUfdPp61ttOQ/5jWZ3CPiWWBIkekrgMMbWOYy4LJmR2Nmm7VOnTqxYsUKunXr5gTfRiKCFStW0KlTp2Yt51+omlnJevXqxZIlS6ipqWnvULYqnTp1olevXs1axsndzEpWVVVF37592zsMK4FvHGZmlkNO7mZmObRVDsvMeHlFs5c55LAKBGJmViHuuZuZ5ZCTu5lZDjm5m5nlkJO7mVkOObmbmeWQk7uZWQ45uZuZ5ZCTu5lZDjm5m5nlkJO7mVkOObmbmeWQk7uZWQ45uZuZ5ZCTu5lZDjm5m5nlkJO7mVkOObmbmeVQk8ldUm9Jj0uaL+l5SWen6RdLWiqpOj0+U7DMhZIWSHpR0tGVbICZmW2qlH+ztw74dkQ8I6kLMFvS1DTv6oi4orCwpP2AMcAAoCcwTVL/iFjfmoGbmVnDmuy5R8TyiHgmPV8NzAf2bGSR44F7ImJtRLwCLACGt0awZmZWmmaNuUvqAwwBnk6TzpL0rKRbJe2Spu0JLC5YbAlFvgwkjZc0S9Ksmpqa5kduZmYNKjm5S+oM/BL4ZkSsAm4G9gYGA8uBK2uLFlk8NpkQcUtEDIuIYd27d29u3GZm1oiSkrukKrLEfndEPAgQEa9FxPqI2AD8lA+GXpYAvQsW7wUsa72QzcysKaVcLSNgIjA/Iq4qmN6joNiJwNz0fAowRlJHSX2BfYCZrReymZk1pZSrZT4GfBl4TlJ1mnYR8EVJg8mGXBYBXwWIiOcl3QfMI7vS5kxfKWNm1raaTO4R8UeKj6P/upFlLgMuKyMuMzMrg3+hamaWQ07uZmY55ORuZpZDTu5mZjnk5G5mlkNO7mZmOeTkbmaWQ07uZmY55ORuZpZDTu5mZjnk5G5mlkNO7mZmOeTkbmaWQ07uZmY55ORuZpZDTu5mZjnk5G5mlkNO7mZmOeTkbmaWQ07uZmY55ORuZpZDTu5mZjnUZHKX1FvS45LmS3pe0tlp+q6Spkr6S/q7S8EyF0paIOlFSUdXsgFmZrapUnru64BvR8S+wMHAmZL2Ay4AHo2IfYBH02vSvDHAAGAkcJOkDpUI3szMimsyuUfE8oh4Jj1fDcwH9gSOB25PxW4HTkjPjwfuiYi1EfEKsAAY3spxm5lZI5o15i6pDzAEeBrYIyKWQ/YFAOyeiu0JLC5YbEmaVn9d4yXNkjSrpqamBaGbmVlDSk7ukjoDvwS+GRGrGitaZFpsMiHilogYFhHDunfvXmoYZmZWgpKSu6QqssR+d0Q8mCa/JqlHmt8DeD1NXwL0Lli8F7CsdcI1M7NSlHK1jICJwPyIuKpg1hTg1PT8VODhguljJHWU1BfYB5jZeiGbmVlTti2hzMeALwPPSapO0y4CfgzcJ+l04FVgNEBEPC/pPmAe2ZU2Z0bE+tYO3MzMGtZkco+IP1J8HB3g8AaWuQy4rIy4zMysDP6FqplZDjm5m5nlkJO7mVkOObmbmeWQk7uZWQ45uZuZ5ZCTu5lZDjm5m5nlkJO7mVkOObmbmeWQk7uZWQ45uZuZ5ZCTu5lZDjm5m5nlkJO7mVkOObmbmeWQk7uZWQ45uZuZ5ZCTu5lZDjm5m5nlkJO7mVkOObmbmeVQk8ld0q2SXpc0t2DaxZKWSqpOj88UzLtQ0gJJL0o6ulKBm5lZw0rpuU8CRhaZfnVEDE6PXwNI2g8YAwxIy9wkqUNrBWtmZqVpMrlHxO+BN0tc3/HAPRGxNiJeARYAw8uIz8zMWqCcMfezJD2bhm12SdP2BBYXlFmSpm1C0nhJsyTNqqmpKSMMMzOrr6XJ/WZgb2AwsBy4Mk1XkbJRbAURcUtEDIuIYd27d29hGGZmVkyLkntEvBYR6yNiA/BTPhh6WQL0LijaC1hWXohmZtZcLUruknoUvDwRqL2SZgowRlJHSX2BfYCZ5YVoZmbNtW1TBST9AhgB7CZpCfBDYISkwWRDLouArwJExPOS7gPmAeuAMyNifUUiNzOzBjWZ3CPii0UmT2yk/GXAZeUEZWZm5fEvVM3McsjJ3cwsh5zczcxyyMndzCyHnNzNzHLIyd3MLIec3M3McsjJ3cwsh5zczcxyyMndzCyHnNzNzHLIyd3MLIec3M3McsjJ3cwsh5zczcxyyMndzCyHnNzNzHLIyd3MLIec3M3McsjJ3cwsh5zczcxyyMndzCyHmkzukm6V9LqkuQXTdpU0VdJf0t9dCuZdKGmBpBclHV2pwM3MrGGl9NwnASPrTbsAeDQi9gEeTa+RtB8wBhiQlrlJUodWi9bMzErSZHKPiN8Db9abfDxwe3p+O3BCwfR7ImJtRLwCLACGt06oZmZWqpaOue8REcsB0t/d0/Q9gcUF5ZakaZuQNF7SLEmzampqWhiGmZkV09onVFVkWhQrGBG3RMSwiBjWvXv3Vg7DzGzr1tLk/pqkHgDp7+tp+hKgd0G5XsCylodnZmYt0dLkPgU4NT0/FXi4YPoYSR0l9QX2AWaWF6KZmTXXtk0VkPQLYASwm6QlwA+BHwP3STodeBUYDRARz0u6D5gHrAPOjIj1FYrdzMwa0GRyj4gvNjDr8AbKXwZcVk5QZmZWHv9C1cwsh5zczcxyyMndzCyHnNzNzHLIyd3MLIec3M3McsjJ3cwsh5zczcxyyMndzCyHnNzNzHLIyd3MLIec3M3McsjJ3cwsh5zczcxyyMndzCyHnNzNzHKoyX/WsSW4eupLzSp/cBvUAXDOkf1bUJOZWfncczczyyEndzOzHHJyNzPLISd3M7MccnI3M8uhsq6WkbQIWA2sB9ZFxDBJuwL3An2ARcDnI+Kt8sJs3MGv3lLJ1ZuZbXFao+d+WEQMjohh6fUFwKMRsQ/waHptZmZtqBLDMscDt6fntwMnVKAOMzNrRLnJPYD/lTRb0vg0bY+IWA6Q/u5ebEFJ4yXNkjSrpqamzDDMzKxQub9Q/VhELJO0OzBV0gulLhgRtwC3AAwbNizKjMPMzAqUldwjYln6+7qkycBw4DVJPSJiuaQewOutEGe7a9lJ2ytaPQ4zs1K0eFhG0o6SutQ+B44C5gJTgFNTsVOBh8sN0szMmqecnvsewGRJtev5eUT8VtKfgPsknQ68CowuP0wzM2uOFif3iHgZGFRk+grg8HKCMjOz8vgXqmZmOeTkbmaWQ07uZmY55ORuZpZDTu5mZjnk5G5mlkNO7mZmOeTkbmaWQ07uZmY55ORuZpZDTu5mZjnk5G5mlkNO7mZmOeTkbmaWQ07uZmY55ORuZpZDTu5mZjnk5G5mlkNO7mZmOeTkbmaWQy3+B9nWtBkTz21W+UNOv6JCkZjZ1sY9dzOzHKpYz13SSOBaoAPws4j4caXqyourp77U7GXOObJ/BSIxy6HHL2/+Modd2PpxtJGKJHdJHYAbgSOBJcCfJE2JiHmVqM/MbHOwOXXQKtVzHw4siIiXASTdAxwPOLk34uBXb2n2MjMmNr+eQ/p1a14dL69ofh2b6fmDlnz4mqtFH9YW9Cqb+748tdf4ZtfREuds+8uK19GifbKZ+z00/7wZbbSNS6GIaP2VSp8DRkbEuPT6y8D/i4izCsqMB2q3xEeAF1s9kJbZDXijvYOoMLcxH9zGfCinjR+KiO7FZlSq564i0zb6FomIW4Dmd1UrTNKsiBjW3nFUktuYD25jPlSqjZW6WmYJ0LvgdS9gWYXqMjOzeiqV3P8E7COpr6TtgDHAlArVZWZm9VRkWCYi1kk6C/gfskshb42I5ytRVwVsdkNFFeA25oPbmA8VaWNFTqiamVn78i9UzcxyyMndzCyHtprkLmmkpBclLZB0QZH5knRdmv+spAML5t0q6XVJc9s26uZpaRsl9Zb0uKT5kp6XdHbbR1+aMtrYSdJMSXNSGy9p++hLU86+muZ3kPRnSY+0XdTNU+bncZGk5yRVS5rVtpGXrsw2dpX0gKQX0ufykGYHEBG5f5Cd1F0I9AO2A+YA+9Ur8xngN2TX6B8MPF0w75PAgcDc9m5LJdoI9AAOTM+7AC/VX3ZzeJTZRgGd0/Mq4Gng4PZuU2vvq2n+t4CfA4+0d3sq0UZgEbBbe7ejwm28HRiXnm8HdG1uDFtLz73udggR8R5QezuEQscDd0TmKaCrpB4AEfF74M02jbj5WtzGiFgeEc8ARMRqYD6wZ1sGX6Jy2hgRsSaVqUqPzfFqgrL2VUm9gGOAn7Vl0M1UVhu3EC1uo6SdyDqUEwEi4r2IWNncALaW5L4nsLjg9RI2TV6llNmctUobJfUBhpD1bDc3ZbUxDVdUA68DUyMid20ErgG+A2yoUHytodw2BvC/kman25hsjsppYz+gBrgtDa/9TNKOzQ1ga0nuTd4OocQym7Oy2yipM/BL4JsRsaoVY2stZbUxItZHxGCyX0wPl7R/64bXKlrcRknHAq9HxOzWD6tVlbuvfiwiDgQ+DZwp6ZOtGVwrKaeN25INA98cEUOAd4BNxuybsrUk91Juh7Cl3zKhrDZKqiJL7HdHxIMVjLMcrfI+pkPcJ4CRrR5h+cpp48eA4yQtIhsG+JSkuyoXaouV9T5GRO3f14HJZEMgm5ty2rgEWFJwZPkAWbJvnvY+8dAWD7JvwpeBvnxwcmNAvTLHsPHJjZn15vdh8z6h2uI2ptd3ANe0dzsq2MbupJNSwPbAH4Bj27tNldhXU5kRbL4nVMt5H3cEuhQ8n052B9p2b1drvo9p//xIen4x8JNmx9DeG6ENN/ZnyK4CWQh8N007AzgjPRfZPxhZCDwHDCtY9hfAcuB9sm/V09u7Pa3ZRuDjZIeDzwLV6fGZ9m5PK7fxAODPqY1zgR+0d1sqsa8WrGOzTe5lvo/9UqKcAzxfu+zm+Cgz5wwGZqX99SFgl+bW79sPmJnl0NYy5m5mtlVxcjczyyEndzOzHHJyNzPLISd3M7MccnI3M8shJ3czsxz6Pz9AVvjOiLJzAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xmin = np.min(np.append(sym_losses,asym_losses))\n",
    "xmax = np.percentile(np.append(sym_losses, asym_losses), 90)\n",
    "bins = np.linspace(xmin, xmax, 25)\n",
    "plt.hist(sym_losses,bins, alpha=0.5, label='symmetric')\n",
    "plt.hist(asym_losses, bins, alpha=0.5, label='asymmetric')\n",
    "plt.title(\"MSE of confusion matrices from rte data\")\n",
    "plt.legend(loc='upper right')\n",
    "plt.savefig(\"pic/rte_symmetry.png\",dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAtRklEQVR4nO3df3xW9X338dcnvyFAAQUVwg9b6Rqk2il1G2a3zZxa2ntq13YzdnU1iKUbqR13C9TYe/Vug5qiaxfbMlcy+yvRdq3WSimo0LmobYENFUx1VKAEREL5IQSuJCSf+49zEq/EJFwBruuEw/v5eFyPXOf3J+c61+f6nu/5nu8xd0dEROIrK+oAREQkvZToRURiToleRCTmlOhFRGJOiV5EJOaU6EVEYk6JfgizwL+Z2X4z+3UGt3u5mf2PmR02s+tPYj0rzexvT2FokTGzZWb2hQxu70NmtiP8DP4wU9sNt/0+M2saYPqDZvblDGxnUPv8eOvrY/5fmNktqc5/OlOiT2Jm28yszczO7jV+o5m5mU0Nh4vM7EdmttfMDprZi2b2iXDa1HDew71ef30CIZUAVwFF7n7ZSf57g/H/gPvdfYS7P3qiK3H32e7+7VMX1qlnZp8ws4bjzefu89z9S5mIKbQUmB9+Bv+dwe0OGRHs836FueHPo47jROVEHcAQtBUoA2oAzOzdwLBe83wXeB6YArQC7wbO7TXPaHc/dpKxTAG2uXvLSa7nRLa7OcPbHLLMLNvdOzK82X4/AzPLOQXH1mlN+2BwVKJ/q+8CNyUN/y3wnV7zvBd40N1b3P2Yu/+3u688kY2Z2QQze8zM9pnZFjObG46fA3wL+JPwjODOfpafa2aNZnbIzF4ys0vC8cXhqekBM9tsZtcmLfOgmX3dzFaEy/3KzN4RTvst8Hbgp+F283uXZszsi2b2vfB9gZl9z8x+H25rnZmdE07rPjU2sywzu8PMtpvZHjP7jpm9LZzWdRb0t2b2u/BMqXKAffagmX0jrBo6bGbPmNm5ZvbVsJrrN8nVHWa22Mx+m7SPPtS1j4BlSfv4QNL6v2lmPzOzFqC0d3WFmV0Xnum9Ea77/eH4t5nZcjN7zcx2mtmXzSw7nHaBmf1HeBa418we7uN/yzezw0A28Hz4eXSVKBeZ2QtAi5nlmNm14Wd7INzXxUnr2WZmnzOzF8ysJYzpnHCfHTKzJ81sTH/7OFzH7WGc28zsY/3M85YzovCzvCDp/1kafq6vW1AdM6zX/H1uJ3mfW1gtE+6D3cC/mdmwcJ79ZvYSwfdyoP/nqvDYOGhm9wOWNO0dZrYmPI73mtn3zWx0OO27wGTe/E4sDMf/0Mx2h+t72swuHGj7kXJ3vcIXsA34c+BloJjgy7aDoHTlwNRwvieBZ4AbgMm91jE1nDcnxW3+B/ANoAB4D9AMXBlO+wTQMMCyHwV2EhzgBlwQxpoLbAFuB/KAPwMOAX8QLvcgsA+4jOCs7vvAQ733wwDDXwS+F77/JPBTYHi4vy4FRoXTfgHcEr4vD2N6OzAC+DHw3V777F8Jzp4uJjhTKu7n/34Q2BtuqwBYQ3AmdlMYw5eBtb320wSCgs1fAy3Aef3t43D9B4HLw2UKwnFfDqdfFk6/Kpw+EXhXOO1R4F+AQmA88Gvgk+G0eqAyaZ0lA3y2DlzQ6zPYCEwK99E7w//jqvDzXhju37yk+X8JnBPGtwf4L+APgfxwn/1jP9t+H3AMuC+c94pwW8nHz5cH2H/dsQNfBR4DxgIjCY6Vu05gO13z3hPOOwy4G/jPcN2TgE1AUz//09nAG8BHwv31D+H6uo7PC8J9mQ+MA54GvtrfdyDpmB4ZLvNVYGPUOay/l0r0fesq1V8F/IYgmSb7KMEB9gVga1iy612a2BuWtLpexb2mY2aTCOrhF7l7wt03EpTiP55inLcA1e6+zgNb3H078McEyfRud29z9zXA4wRVUl1+7O6/9uD09/sEPzInoh04i+CL3eHuG9z9jT7m+xhwn7u/6u6Hgc8DN5hZcvXhne5+1N2fJ6gau3iA7T4SbisBPAIk3P07HlSxPEyQ0ABw9x+6+y5373T3h4H/IUjWA/mJuz8TLpPoNW0OUOvuT4TTd7r7byw4k5kNfMaDs709wD8RFAi69tUUYEL4eR/32kAv/+zuO9z9KMEP1oowhnaCOv1hwKyk+Wvc/XV330lwvP7Kg7PP1nCfHe8i7xfcvdXd/wNYAfzVYII1MwPmAv/g7vvc/RCwhDf3x2C300nw49Qa7oO/AqrCde8A/nmAcD4AvOTu/x7ur68Cu7smht+dJ8J1NxP8+Fwx0P/n7rXufijcn18ELrbwLHWoUaLv23eBGwlKK72rbXD3/e6+2N0vJCgxbQQeDQ/sLme7++ikV2Mf25kAdH0BumwnKIGlYhLw237Wu8PdOwdY7+6k90cIfhhOxHeBVcBDZrbLzKrNLLefmLb3iieHYP+dSEyvJ70/2sdw97JmdlP4Y3wgrJ6ZQVDCG8iOAab1t9+7zqZeS9rWvxCU7CEodRvw67DKpfw4MQwUU4/9GX7WO+j5Gae8j/qw33teG9oebnMwxhGc6W1I2h8/D8efyHaae/3oTqDnPtlO/3rM60GRvHvYzMab2UNhddsbwPcY4Bgxs2wzuzustnuDoMTPQMtESYm+D2GpeCtBKeDHx5l3L0FpagLBKeRg7ALGmtnIpHGTeesZRH92AO/oZ72TzCz58x3MentrIfjCdum+8Ozu7e5+p7tPJyhN/m96XuNIjmlKr3iO0TP5nHJmNoWgSmg+cJa7jyY4xe/6Ue6v+9aBunXtb7/vIKhySv6RHxUWCHD33e4+190nEFR5faOrLjtFyTH12J9hIWMSJ/4Z9zbGzAqThieH2+ytx7FhZsmNEvYS/KBcmLQ/3ubuyT8wqW4H3vqZvEbwPycv258e8ybtry53heu/yN1HAX9DUh1+H9u+EbiOoKr3bQTVj/RaZshQou/fHODPvI8WL2Z2j5nNCC+IjQQ+BWxx998PZgPh6eazwF0WXNS8KNzu91NcxbeAz5rZpRa4IExsvyL4Ai40s1wzex/wF8BDg4kvyUaCapZcM5tJUM8JgJmVmtm7Lbjg+AZB9URfLVTqgX8ws/PNbATBKfzDnv6WE4UEX9LmMN6bCUr0XV4HiswsbxDrXA7cbGZXWnCReaKZvcvdXwNWA/ea2ahw2jvM7Ipw2x81s6JwHfvDuE60Nc8PgA+GMeQC/4fgR+bZE1xfX+40szwz+1OCH/Af9jHP88CFZvYeMysgqMIAus8y/hX4JzMbDxDuq2tOYDt9+QHweTMbE+7XigHmXRHG+ZdhdeGn6dlSbiRwGDhgZhOBz/Va/nWC60vJ87cCvyf4oVuSYsyRUKLvh7v/1t3X9zN5OEEd5wHgVYKS1bW95jlgPdvRL+hnXWUEpYFd4Tr/0d2fSDHGHwJVQB3BxdZHgbHu3hbGM5ugVPUN4CZ3/00q6+3DFwhKsPuBO8PtdTkX+HeCJN9IcHH5e32so5agmudpgrOlBAN/MU8Jd38JuBd4juDL+m6CC+ld1hA0Y9xtZntTXOevgZsJ6t8PEvzPXaXrmwgugL9EsL/+HTgvnPZe4FcWtKp5DLjN3bee4P/1MkGps4bgM/4L4C/Cz/5U2E0Q/y6Cgse8vo4fd3+F4L6LJwmuffS+7rCI4CLxL8MqjieBPxjsdvpxJ0F1zVaCH9jv9jdjeOb9UYILuL8HptHzOLgTuITg81zBW8/k7wLuCKugPktQpbud4AzqJYIL30OWBVVVIiISVyrRi4jEnBK9iEjMKdGLiMScEr2ISMwNyU7Nzj77bJ86dWrUYYiInDY2bNiw193H9TVtSCb6qVOnsn59fy0bRUSkNzPr985gVd2IiMScEr2ISMwp0YuIxJwSvYhIzCnRi4jEnBK9iGRcfX09M2bMIDs7mxkzZlBfXx91SLE2JJtXikh81dfXU1lZyfLlyykpKaGhoYE5c+YAUFZWdpyl5UQMyd4rZ86c6WpHLxJPM2bM4Prrr+fRRx+lsbGR4uLi7uFNmzZFHd5py8w2uPvMvqapRC8iGfXSSy/R0tJCbW1td4m+vLyc7dsHehKgnAzV0YtIRuXl5VFRUUFpaSm5ubmUlpZSUVFBXt5gHvIlg6FELyIZ1dbWxv3338/atWtpb29n7dq13H///bS1naqHY0lvqroRkYyaPn06119/PRUVFd119DfeeCOPPvpo1KHFlkr0IpJRlZWV1NXVUVNTQyKRoKamhrq6OiorK6MOLbZUoheRjOpqQplcoq+qqlLTyjRS80oRkRgYqHmlqm5ERGJOiV5EJOaU6EUk49TXTWbpYqyIZJT6usk8XYwVkYyaMWMGNTU1lJaWdo9bu3YtFRUV6uvmJAx0MVaJXkQyKjs7m0QiQW5ubve49vZ2CgoK6OjoiDCy05ta3YjIkFFcXExDQ0OPcQ0NDRQXF0cUUfwp0YtIRlVWVjJnzpwefd3MmTNHd8amkS7GikhG6c7YzEupjt7M3g98DcgGvuXud/eafh3wJaATOAZ8xt0bUlm2L6qjFxEZnJOqozezbODrwGxgOlBmZtN7zfYUcLG7vwcoB741iGVFRCSNUqmjvwzY4u6vunsb8BBwXfIM7n7Y3zw1KAQ81WVFRCS9Ukn0E4EdScNN4bgezOxDZvYbYAVBqT7lZcPlbzWz9Wa2vrm5OZXYRUQkBakkeutj3Fsq9t39EXd/F3A9QX19ysuGyz/g7jPdfea4ceNSCEtETlfqAiGzUml10wRMShouAnb1N7O7P21m7zCzswe7rIjEn7pAyLxUSvTrgGlmdr6Z5QE3AI8lz2BmF5iZhe8vAfKA36eyrIicWaqqqli+fHmPh4MvX76cqqqqqEOLreOW6N39mJnNB1YRNJGsdffNZjYvnL4M+DBwk5m1A0eBvw4vzva5bJr+FxE5DTQ2NlJSUtJjXElJCY2NjRFFFH8p3TDl7j8DftZr3LKk9/cA96S6rIicubq6QEju1ExdIKSXukAQkYxSFwiZpy4QRCSj1AVC5qmbYhGRGFA3xSIiZzAlehHJON0wlVlK9DFVUVFBQUEBZkZBQQEVFRVRhyQCvHnDVE1NDYlEgpqaGiorK5Xs08ndh9zr0ksvdTlx8+fP95ycHL/33nu9paXF7733Xs/JyfH58+dHHZqIX3jhhb5mzZoe49asWeMXXnhhRBHFA7De+8mpuhgbQwUFBSxZsoQFCxZ0j7vvvvu4/fbbSSQSEUYmomfGposuxp5hWltbmTdvXo9x8+bNo7W1NaKIRN6kZ8ZmnhJ9DOXn57Ns2bIe45YtW0Z+fn5EEYm8STdMZZ5umIqhuXPn8rnPfY7q6mpef/11zjnnHJqbm/m7v/u7qEMT0Q1TEVCij6FZs2bx7W9/m3379gGwb98+CgsLmTVrVsSRiQTKysqU2DNIVTcxVFVVxU9+8hPa2tpwd9ra2vjJT36ibmBFzlBqdRNDatUgcuZRq5szjFo1iEgyJfoYUqsGEUmmi7ExVFZWxrPPPsvs2bNpbW0lPz+fuXPn6uKXyBlKJfoYqq+vZ8WKFaxcuZK2tjZWrlzJihUr1JeIyBlKF2NjaMaMGdTU1PR4VNvatWupqKhg06ZNEUYmIuky0MVYJfoYUqsbkTOPWt2cYdTqRkSSKdHHkFrdiEiylBK9mb3fzF42sy1mtriP6R8zsxfC17NmdnHStG1m9qKZbTQz1cdkQFlZGVVVVd0PH6moqFBfIjKk6AlTGdZfR/VdLyAb+C3wdiAPeB6Y3mueWcCY8P1s4FdJ07YBZx9vO8kvPXhEJL7q6ur8/PPP9zVr1nhbW5uvWbPGzz//fK+rq4s6tNMaAzx4JJUS/WXAFnd/1d3bgIeA63r9WDzr7vvDwV8CRSfz4yMi8VVVVcXFF1/M7NmzycvLY/bs2Vx88cXqiymNUkn0E4EdScNN4bj+zAFWJg07sNrMNpjZrf0tZGa3mtl6M1vf3NycQlgyEJ0ay1C1efNmHn/8cZYsWUJLSwtLlizh8ccfZ/PmzVGHFlupJHrrY1yfbTLNrJQg0S9KGn25u19CUKXz92b2v/pa1t0fcPeZ7j5z3LhxKYQl/dHDl2UoMzOuuOIKamtrGTlyJLW1tVxxxRWY9ZVq5FRIJdE3AZOShouAXb1nMrOLgG8B17n777vGu/uu8O8e4BGCqiBJo6qqKpYvX05paSm5ubmUlpayfPlynRrLkODurFmzhr1799LZ2cnevXtZs2ZN1zU9SYNUEv06YJqZnW9mecANwGPJM5jZZODHwMfd/ZWk8YVmNrLrPXA1oFsz06yxsZGSkpIe40pKSmhsbIwoIpGe8vPzGTZsGFlZWQwbNkyPuUyz4yZ6dz8GzAdWAY3AD9x9s5nNM7OuJ1D/X+As4Bu9mlGeAzSY2fPAr4EV7v7zU/5fSA+6YUqGuvb2dioqKjh06BAVFRW0t7dHHVK89dccJ8qXmleeHDVfk6EM8Guvvdbz8/Md8Pz8fL/22ms9SEdyojjJ5pVymikrK+ODH/xgj+ZrH/zgB3XDlAwJRUVFrFu3rkfvquvWraOoSK2y00WJPobq6+t5+OGHOe+888jKyuK8887j4YcfVqsbGRKqq6s5duwY5eXlFBQUUF5ezrFjx6iuro46tNhSoo+hhQsXkpOTQ21tLYlEgtraWnJycli4cGHUoYlQVlbG1772NQoLCwEoLCzka1/7ms4400jdFMeQmbF69Wquuuqq7nFPPPEEV199tZqwicSUuikWETmDKdHHUFFRETfddFOPbopvuukmXewSOUMp0cdQdXU1HR0dlJeXk5+fT3l5OR0dHbrYJXKGUqKPoeSLXWami10iZzgl+pgqKytj06ZNdHR0sGnTJiV5GVLUu2pm5UQdgIicWbp6V12+fDklJSU0NDQwZ84cABVI0kTNK0Uko2bMmEFNTQ2lpaXd49auXUtFRQWbNqnPwxOl5pUiMmQ0NjbS1NTUo+qmqalJvaumkapuRCSjJkyYwMKFC6mrq+uuurnxxhuZMGFC1KHFlhK9iGRcIpGgvLyc7du3M2XKFBKJBCNGjIg6rNhS1Y2IZNTOnTvJzc0F6H58YG5uLjt37owyrFhToheRjMrLy2Px4sVs3bqVjo4Otm7dyuLFi8nLy4s6tNhSoheRjGpra6OmpqZHFx01NTW0tbVFHVpsqY5eRDJq+vTpXH/99VRUVNDY2EhxcTEf+9jHePTRR6MOLbZUoheRjKqsrKSuro6amhoSiQQ1NTXU1dVRWVkZdWixpRK9iGRU192vySX6qqoq3RWbRirRi4jEXEqJ3szeb2Yvm9kWM1vcx/SPmdkL4etZM7s41WVF5MzS1ddNctVNZWWlOjZLo+P2dWNm2cArwFVAE7AOKHP3l5LmmQU0uvt+M5sNfNHd/yiVZfuivm5E4kt93aTHyfZ1cxmwxd1fdfc24CHguuQZ3P1Zd98fDv4SKEp1WUkPdQMrQ1VjYyMlJSU9xpWUlKivmzRKJdFPBHYkDTeF4/ozB1g52GXN7FYzW29m65ubm1MIS/qjU2MZyoqLi2loaOgxrqGhgeLi4ogiir9UEr31Ma7P+h4zKyVI9IsGu6y7P+DuM9195rhx41IIS/pTVVXF8uXLKS0tJTc3l9LSUpYvX05VVVXUoYlQWVnJnDlzetwwNWfOHDWvTKNUmlc2AZOShouAXb1nMrOLgG8Bs93994NZVk4tnRrLUKbmlZmXSqJfB0wzs/OBncANwI3JM5jZZODHwMfd/ZXBLCunXtepcfLFLp0ay1BSVlamxJ5Bx626cfdjwHxgFdAI/MDdN5vZPDObF872f4GzgG+Y2UYzWz/Qsmn4PySJTo1FJJkeJRhT9fX1VFVVdZ8aV1ZWqgQlEmMDNa9UohcRiQE9M1ZEhhTd55FZ6tRMRDKq6z6P5cuXdz8zds6cOQCqXkwTVd2ISEapC4T0UB29iAwZ2dnZJBKJ7ufGArS3t1NQUEBHR0eEkZ3eVEcvIkOGukDIPCV6Ecko3eeReboYKyIZVVZWxrPPPsvs2bNpbW0lPz+fuXPn6kJsGqlELyIZVV9fz4oVK1i5ciVtbW2sXLmSFStWqIllGulirIhklFrdpIcuxorIkNHY2EhTU1OPG6aamprUu2oaqY5eRDJqwoQJLFy4kLq6uu4bpm688UYmTJgQdWixpRK9iGScmQ04LKeWEr2IZNSuXbu45557qKiooKCggIqKCu655x527dIzidJFVTciklHFxcUUFRX1uPC6du1a3TCVRirRi0hG6YapzFOJXkQySs+MzTy1oxcRiQG1oxcROYMp0YtIxukJU5mlOnoRySg9YSrzVEcvIhmlvm7S46Tr6M3s/Wb2spltMbPFfUx/l5k9Z2atZvbZXtO2mdmLZrbRzJS9Rc5wjY2NlJSU9BhXUlKivm7S6LiJ3syyga8Ds4HpQJmZTe812z7g08DSflZT6u7v6e/XRkTOHHrCVOalUqK/DNji7q+6exvwEHBd8gzuvsfd1wHtaYhRRGJEN0xlXioXYycCO5KGm4A/GsQ2HFhtZg78i7s/0NdMZnYrcCvA5MmTB7F6ETmd6IapzEsl0ffVrdxgruBe7u67zGw88ISZ/cbdn37LCoMfgAcguBg7iPWLyGmmrKxMiT2DUqm6aQImJQ0XASl3M+fuu8K/e4BHCKqCREQkQ1JJ9OuAaWZ2vpnlATcAj6WycjMrNLORXe+BqwG1nxIRyaDjVt24+zEzmw+sArKBWnffbGbzwunLzOxcYD0wCug0s88QtNA5G3gkfKhADlDn7j9Py38iIiJ9SunOWHf/GfCzXuOWJb3fTVCl09sbwMUnE6CIiJwc9XUjIhJzSvQiknFdjxE0s+7HCUr6KNGLSEZVVFSwbNkylixZQktLC0uWLGHZsmVK9mmkTs1EJKMKCgpYsmQJCxYs6B533333cfvtt5NIJCKM7PQ2UKdmSvQiklFmRktLC8OHD+8ed+TIEQoLCxmK+eh0oSdMiciQkZ+fz9VXX92jjv7qq68mPz8/6tBiS4leRDLqne98J8888wzXXHMNzc3NXHPNNTzzzDO8853vjDq02NITpkQko1555RUuv/xyVq1axbhx48jPz+fyyy9H1bXpo0QvIhnV2trK6tWr+6yjl/RQ1Y2IZFR+fj7Lli3rMW7ZsmWqo08jlehFJKPmzp3LokWLAJg3bx7Lli1j0aJFzJs3L+LI4kvNK0Uk46655hqeeOIJ3B0z46qrrmLVqlVRh3VaU/NKERky6uvree6558jJCSoUcnJyeO6556ivr484svhSoheRjJo/fz5Hjhzh7rvvpqWlhbvvvpsjR44wf/78qEOLLdXRi0hG7du3j+rq6u4uEBYsWEBHRwcLFy6MOLL4UoleRDJu7969zJgxg+zsbGbMmMHevXujDinWlOhFJKOysrJYunQp5eXlHDp0iPLycpYuXUpWltJRumjPxlR9fX2PEpMudMlQMXr0aNyd6upqCgsLqa6uxt0ZPXp01KHFlhJ9DNXX13PbbbfR0tICQEtLC7fddpuSvQwJBw4c4Morr2TPnj0A7NmzhyuvvJIDBw5EG1iMKdHH0MKFC8nJyaG2tpZEIkFtbS05OTm62CVDwoQJE9i0aRNPPfUUbW1tPPXUU2zatIkJEyZEHVpsqdVNDDU1NbF69WpKS0sBKC0t5dvf/jZXX311xJGJBBKJBOXl5Wzfvp0pU6aQSCQYMWJE1GHFVkolejN7v5m9bGZbzGxxH9PfZWbPmVmrmX12MMtKeqxZs6ZHHf2aNWuiDkkEgJ07d9LR0cHOnTtx9x7Dkh7HTfRmlg18HZgNTAfKzGx6r9n2AZ8Glp7AsnKKjR07lurq6h6tGqqrqxk7dmzUoYmQnZ1NW1tbj3FtbW1kZ2dHFFH8pVKivwzY4u6vunsb8BBwXfIM7r7H3dcB7YNdVk694cOHU1BQwOLFiyksLGTx4sUUFBT06BZWJCrHjh2jtbWVW265hQMHDnDLLbfQ2trKsWPHog4ttlJJ9BOBHUnDTeG4VKS8rJndambrzWx9c3NziquXvuzcuZP29nba24Pf3a73OjWWoaK0tJSnn36asWPH8vTTT3dfT5L0SCXRWx/jUu3yMuVl3f0Bd5/p7jPHjRuX4uqlP+3t7YwZM4asrCzGjBnTnfRFhoINGzbQ0tKCu9PS0sKGDRuiDinWUkn0TcCkpOEiYFeK6z+ZZeUEdXX9escdd3Do0CHuuOMOzIyh2CW1nHmysrJ444032LZtG+7Otm3beOONN3RnbBqlsmfXAdPM7HwzywNuAB5Lcf0ns6ychI985CPU1tYycuRIamtr+chHPhJ1SCIA5ObmAnQn9q6/XePl1DtuO3p3P2Zm84FVQDZQ6+6bzWxeOH2ZmZ0LrAdGAZ1m9hlguru/0deyafpfJMkjjzyCmdHZ2ckrr7zCyy+/HHVIIkDwzNjs7Gw6OjoA6OzsJDs7m9bW1ogjiy89YSqGcnNz+2zBkJOTo7p6iZyZMWbMGH70ox9RUlJCQ0MDH/7wh9m/f7+qF0/CQE+Y0p2xMdRVUkp1vEimHTlyhPLycn73u98xefJkjhw5EnVIsaarHzHk7gwfPpypU6eSlZXF1KlTGT58uEpLMmS0trZy8OBBOjs7OXjwoKpt0kyJPqamTJnCa6+9RmdnJ6+99hpTpkyJOiQRIKhCzMvL4/DhwwAcPnyYvLy87mfIyqmnRB9TjY2N3aWk1tZWGhsbI45IJNDR0cGoUaOYOHEiZsbEiRMZNWqUqhbTSIleRDJq+vTpfPKTn6SwsBAzo7CwkE9+8pNMn65usNJFiV5EMqqyspK6ujpqampIJBLU1NRQV1dHZWVl1KHFlhJ9TOXl5fW4GJuXlxd1SCIAlJWVMW3aNK688kry8vK48sormTZtGmVlZVGHFltK9DHV1tZGRUUFhw4doqKi4i3dwopEpaKigieffJLx48cDMH78eJ588kkqKioijiy+dMNUDJn11ZdcYCh+3nJmyc3NJT8/n3HjxnU/Yaq5uZnW1lbd0HcSBrphSiX6GCosLBzUeJFMOnbsWJ9PmFJ/9OmjRB9D/d1lqLsPZahIJBKcddZZZGVlcdZZZ5FIJKIOKdaU6GOoq3rm3HPPJSsri3PPPbfHeJGhYM+ePXR2drJnz56oQ4k9JfqYuuiiizjrrLMAOOuss7jooosijkikp87Ozh5/JX10z3FMvfDCC4wZM4bOzk527drF/v37ow5JpIeCggISiUT3X0kflehj7ODBgz3+igwlo0ePJisri9GjR0cdSuypRB9jOjWWocrM2L17NwC7d+/Woy7TTCX6mLrkkku629ObGZdccknEEYm8qeu5xoCSfAYo0cfQ2LFjef7551m6dCktLS0sXbqU559/nrFjx0Ydmki3ruSuJJ9+SvQxNHz4cEaMGEFNTQ0jR46kpqaGESNGMHz48KhDEwEgOzt7wGE5tZToY2jXrl28973vZfv27XR2drJ9+3be+973smvXrqhDEwGgqKiI/Px8APLz8ykqKoo4onhToo+h0aNH89RTTzF+/HjMjPHjx/PUU0+pdYMMGdu3bycrK6v7tX379qhDirWUEr2Zvd/MXjazLWa2uI/pZmb/HE5/wcwuSZq2zcxeNLONZqaeyjLgwIEDgxovEoWjR4/S2dnJ0aNHow4l9o7bvNLMsoGvA1cBTcA6M3vM3V9Kmm02MC18/RHwzfBvl1J333vKopYBdXZ2MmzYMPbt24e7s2/fPgoKCvSFkiElKyuLzs7O7r+SPqmU6C8Dtrj7q+7eBjwEXNdrnuuA73jgl8BoMzvvFMcqg5CTk8OqVatoa2tj1apVevCyDClZWVndF2Czs7PJylItcjql8u2fCOxIGm6iZ2m9v3kmAq8BDqw2Mwf+xd0fOPFwJVWHDh3ixhtvZM+ePYwfP55Dhw5FHZJIt+QSfEdHh0r0aZZKou/rKRa9G74ONM/l7r7LzMYDT5jZb9z96bdsxOxW4FaAyZMnpxCWHE9zczOdnZ00NzdHHYqIRCiV86UmYFLScBHQu51ev/O4e9ffPcAjBFVBb+HuD7j7THefOW7cuNSilz7l5OQwYsQIJk2ahJkxadIkRowYoeobGVKS74yV9Eol0a8DppnZ+WaWB9wAPNZrnseAm8LWN38MHHT318ys0MxGAphZIXA1sOkUxi996OjowMx6PMHHzOjo6Ig6NBGJwHETvbsfA+YDq4BG4AfuvtnM5pnZvHC2nwGvAluAfwX+Lhx/DtBgZs8DvwZWuPvPT/H/IL1MnDixx/M329vbaW1tZeLEiRFHJhIoKCjoccZZUFAQdUixpoeDx9CIESNoaWnhU5/6FHfddRef//zn+eY3v0lhYSGHDx+OOjw5w5kZZkZWVhYdHR1kZ2fT2dmJu6vfm5Ogh4OfYVpaWpg1axa1tbWMHj2a2tpaZs2aRUtLS9ShiVBUVERubm53VWJHRwe5ubnqBiGNlOhjasuWLaxcuZK2tjZWrlzJli1bog5JBAgeUt/e3t7jmcbt7e16eH0aKdHHVO+7YHVXrAwV+/btY9SoURQUFODuFBQUMGrUKPbt2xd1aLGlRB9TLS0tlJWVkZeXR1lZmaptZEiprKxk69atdHZ2snXrViorK6MOKdbUsDqGLrzwQoYNG8aGDRsA2LNnD5deeqlK9TJk3HfffcycOZOSkhIaGhq47777og4p1lSij6HS0lI2btzY4wlTGzdupLS0NOrQRCgqKuLo0aOUl5dTUFBAeXk5R48e1cXYNFKij6G1a9eyaNEiamtrGTlyJLW1tSxatIi1a9dGHZoI1dXV5OXlAW8+RjAvL4/q6uoow4o1taOPoezsbBKJBLm5ud3j2tvbKSgo0N2xMiTU19dTVVVFY2MjxcXFVFZWUlZWFnVYp7WB2tGrjj6GiouLaWho6FFV09DQQHFxcYRRibyprKxMiT2DlOhjqLKykg984AMkEonucQUFBdTW1kYYlYhERXX0MfTggw+SSCS6H+aQlZVFIpHgwQcfjDYwEYmEEn0MrV69mtzc3B5P8MnNzWX16tURRyYSqK+vZ8aMGWRnZzNjxgzq6+ujDinWVHUTU+3t7d2JvrOzUxdhZcior6+nvLy8u2px8+bNlJeXA6jePk3U6iaGBnqQw1D8vOXM0tW7au+Hg6t31ZOj3itFZMhoaWnBzPjKV75CS0sLX/nKVzAzddORRkr0IpJxN998MwsWLGD48OEsWLCAm2++OeqQYk1VNzHUVXVz7rnnsmfPHsaPH8/u3bsBVd1I5p3oM2F1rA6Oqm7OAF1P7Un+Uu3evZvOzs7uJN97Pj2UWTKh68lRXa+u427MmDE9/ppZj/nk1FGiP82MHTv2Lcn6ZBJ2X+saO3bsKYxYzhT9HZu9X11JfP/+/T3+dv0IDPTSsXli1LzyNLPv0x3AqDRvRU0xZfB0bA5dSvSnmy8eTGm2yZMns2PHju7hSZMm8bvf/S5dUYmkfGwmSy7hS/oo0cfE8apvduzY0ec8+pJJuh3v2Oxvuo7NUyelOnoze7+ZvWxmW8xscR/Tzcz+OZz+gpldkuqycmr0vuCV6ksk3XRsRu+4id7MsoGvA7OB6UCZmU3vNdtsYFr4uhX45iCWFRGRNEqlRH8ZsMXdX3X3NuAh4Lpe81wHfMcDvwRGm9l5KS4rIiJplEqinwjsSBpuCselMk8qywJgZrea2XozW9/c3JxCWCIikopUEn1fV0p6V6D1N08qywYj3R9w95nuPnPcuHEphCUiIqlIpdVNEzApabgI2JXiPHkpLCsiImmUSol+HTDNzM43szzgBuCxXvM8BtwUtr75Y+Cgu7+W4rIiIpJGxy3Ru/sxM5sPrAKygVp332xm88Lpy4CfAR8AtgBHgJsHWjYt/4mIiPRJvVeKiMTAQL1XDslEb2bNwPao44iJs4G9UQch0g8dn6fOFHfvsyXLkEz0cuqY2fr+fuVFoqbjMzPUTbGISMwp0YuIxJwSffw9EHUAIgPQ8ZkBqqMXEYk5lehFRGJOiV5EJOaU6AUAM7t+oGcFmNk8M7spkzGJ9MXM3mdmswaYfq0ectST6ugFADN7EHjc3f+9j2k57n4s81GJvJWZfRE47O5L+5imY7UPSvQRMbNC4AcEPXpmA18CbnD3D4XTrwI+5e5/aWaHCZ7U9efAfuB2oBqYDHzG3R8zs08A14frmgHcS9B76MeBVuAD7r7PzN4RrmscQb9Ec4GxwOPAwfD1YWA58CxwOUFHdCMJv1xmdgGwLFxHB/BRd/9tevaUZJKZPUrQ42wB8DWC42A5MJOgi/FaguPhh+5+SbjMNOAhd7/UzLYBdUApkEvwxLm7gAuAr7j7MjN7H3An8DrwHuDHwIvAbcAw4Hp3/62ZjSM4ziaH4X0G2An8kuC4awYqgDnAPuAPgf8K1zXT3eeb2TnhOt4eruNT7v7sKdpdp48TfZ6jXif3Ikim/5o0/DbgN8C4cLgO+IvwvQOzw/ePAKsJvkQXAxvD8Z8g6FRuJEECPgjMC6f9E8EPAsBTwLTw/R8Ba8L3DwIfSYrnF8A3koa/CHw2fP8r4EPh+wJgeNT7U69TdlyODf8OAzYBlwJPJE0fHf5dC7wnfL8EqAjfbyNIpl3H3QtJx+SecPz7gAPAeUA+QfK+M5x2G/DV8H0dUBK+nww09j4Ww+EHCQoq2eHwJ4D7w/cPJx372cDbot7HUbxS6Y9e0uNFYKmZ3UNQZfKfZvZd4G/M7N+APwG66sTbgJ8nLdfq7u1m9iIwNWmda939EHDIzA4CP01a5iIzGwHMAn5o1v1MmPwBYny49wgzGwlMdPdHANw9MZh/Woa8T5vZh8L3kwjOCt9uZjXACoJCBsC3gJvNbAHw1wSPDe3S1RX5i8CIpGMyYWajw2nrPOjKHDP7bdJ6XyQ4G4DgDHZ60rE6Kjz++vJDd+/oY/yfEX6PwukHB/rn40qJPiLu/oqZXUrQvfNdZraa4MvzUyBBcOB21TW2e1gkAToJqmJw904zS/4MW5PedyYNdxJ81lnAAXd/T4phtvQxrq+nhkkMhFUqfw78ibsfMbNfEBQELgauAf4e+CugHPgR8I/AGmCDu/8+aVXJx13vYzKn1zy950ueJyuM5WivOPsKv69jVUJqdRMRM5sAHHH37wFLgUvcfRfBE7juIDgdPaXc/Q1gq5l9NIzBzOzicPIhglPsVNbRZGbXh+vIN7PhpzpWicTbgP1hkn8X8McEvUtmufuPgC8Al0D3mdwq4JvAv6UpntXA/K4BM3tP+DalYzX0FPCpcPlsMxt1KgM8XSjRR+fdwK/NbCNQCXw5HP99YIe7v5Sm7X4MmGNmzwObgevC8Q8BnzOz/w4v2A7k4wSn+C8QXLA9N02xSmb9HMgJP9cvEVz0nAj8IjxOHwQ+nzT/9wmuH60mPT4NzDSzF8zsJWBeOP6nwIfMbKOZ/elx1nEbUBpWc24ALkxTrEOaWt0MMWZ2P/Df7r486lhEBmJmnyW4uPmFqGORgSnRDyFmtoGgrvEqd2893vwiUTGzR4B3AH/m7npwyBCnRC8iEnOqoxcRiTklehGRmFOiFxGJOSV6EZGYU6IXEYm5/w9iSi6ir/bhcgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.boxplot([sym_losses, asym_losses], labels=['symmetric', 'asymmetric'])\n",
    "plt.title(\"MSE of confusion matrices from rte data\")\n",
    "plt.savefig(\"pic/rte_symmetry_box.png\", dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the plot we can see if we symmetricalize the matrices and tensors, the MSE of the confusion matrices can be reduced."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "19\n",
      "29\n",
      "39\n",
      "49\n",
      "59\n",
      "69\n",
      "79\n",
      "89\n",
      "99\n"
     ]
    }
   ],
   "source": [
    "# symmetric or not\n",
    "iternum = 100\n",
    "new_losses = np.zeros(iternum)\n",
    "original_losses = np.zeros(iternum)\n",
    "for i in range(iternum):\n",
    "    if i % 10 == 9:\n",
    "        print(i)\n",
    "    new_losses[i] = confusion_matrix_loss(get_confusion_matrix(\n",
    "        k, labels, sym=True, L=10, N=10), C_true)\n",
    "    original_losses[i] = confusion_matrix_loss(get_confusion_matrix2(\n",
    "        k, labels, sym=True, L=10, N=10), C_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAcbUlEQVR4nO3de7jVZZ338fdHRLfjCQ9oKMjG1CQ1EdFMBfNAj1mKeY2NzkxCo1Jaz1PONAY2VpqNVmReOdMklkke85imNTNkEZopgaApaHhAIZGT4ikh0O/zx+/etFjstffaa6+1177h87quda3D7/Rd91rrs+7f/VsHRQRmZpafzZpdgJmZ1cYBbmaWKQe4mVmmHOBmZplygJuZZcoBbmaWKQd4k6jwI0mvSJrRg9s9QtJ8SW9IOrkb6/mFpLF1LK1pJH1f0oU9uL2PSVqYHoODemq7adsflLSog+nXSrqkB7bTpTbvbH3tzD9N0lnVzp+rTSbAJS2Q9BdJO5fdPkdSSGpN1wdKul3SckmvSvqDpHFpWmua942y09/VUNKRwGhgYEQc2s271xUXA/8REdtExE9rXUlEfDgiptSvrPqTNE7SA53NFxGfjoiv9URNySTgs+kxmN2D2+01mtDmFaVsOK7ZddRi82YX0MOeA04HrgSQdACwVdk81wGPAoOB1cABwLvK5ukXEWu7WctgYEFEvNnN9dSy3Sd6eJu9lqQ+EfF2D2+24mMgafM6PLey5jao3ibTA0+uA84ouT4W+HHZPIcA10bEmxGxNiJmR8QvatmYpN0k3S3pZUlPSzo73X4m8APgA6kHf1GF5c+WNE/S65LmShqebh+adhFXSnpC0kkly1wr6T8l3ZuWe1jSu9O0Z4A9gZ+l7W5Z3vuQ9FVJ16fLLZKul7Qibev3knZN09btokraTNK/SXpe0lJJP5a0fZrWttcyVtILac/mSx202bWSvpeGaN6Q9FtJ75J0RRpuerJ02EHSBEnPlLTRx9raCPh+SRuvLFn/f0n6uaQ3gaPLhw0kjUl7Zq+ldR+fbt9e0g8lLZb0J0mXSOqTpu0l6Tdpr225pJ+0c9+2lPQG0Ad4ND0ebT3AL0p6DHhT0uaSTkqP7crU1kNL1rNA0r9KekzSm6mmXVObvS7pl5J2qNTGaR0XpDoXSPqHCvNssAeTHsu9Su7PpPS4LlExLLJV2fztbqe0zZWGR1IbvAT8SNJWaZ5XJM2leF12dH9Gp+fGq5L+A1DJtHdL+lV6Hi+XdIOkfmnadcAe/PU1cX66/VZJL6X1TZe0X0fbb5qI2CROwALgOOApYCjFi2ghRW8ogNY03y+B3wKnAXuUraM1zbt5ldv8DfA9oAUYBiwDjk3TxgEPdLDsqcCfKJ64AvZKtfYFngYuALYAjgFeB96TlrsWeBk4lGIP6wbg5vJ26OD6V4Hr0+VPAT8D/ia118HAdmnaNOCsdPmfUk17AtsAdwDXlbXZ1RR7OwdS7NkMrXC/rwWWp221AL+i2HM6I9VwCfDrsnbajaIz8nfAm8CASm2c1v8qcERapiXddkmafmiaPjpN3x3YN037KXAVsDWwCzAD+FSadhPwpZJ1HtnBYxvAXmWPwRxgUGqjfdL9GJ0e7/NT+25RMv9DwK6pvqXAI8BBwJapzb5SYdsfBNYCl6d5j0rbKn3+XNJB+62rHbgCuBvYEdiW4rlyaQ3baZv3G2nerYDLgPvTugcBjwOLKtynnYHXgL9N7XVeWl/b83Ov1JZbAv2B6cAVlV4DJc/pbdMyVwBzmp1h7Z02tR44/LUXPhp4kiIkS51K8cS5EHgu9cTK3/2Xp55R22lo2XQkDaIY5/5iRKyKiDkUve5PVFnnWcA3I+L3UXg6Ip4HDqMIycsi4i8R8SvgHoqhoTZ3RMSMKHZDb6B486jFGmAnihfs2xExKyJea2e+fwAuj4hnI+INYCJwmqTSIbqLIuKtiHiUYojqwA62e2fa1irgTmBVRPw4iqGOn1AEFQARcWtEvBgR70TET4D5FCHckbsi4rdpmVVl084EromIqWn6nyLiSRV7Hh8GPh/F3tlS4DsUb/RtbTUY2C093p2OvZf5bkQsjIi3KN6I7k01rKEYM98KOLxk/isjYklE/Ini+fpwFHuLq1ObdXZw9MKIWB0RvwHuBT7elWIlCTgbOC8iXo6I14F/56/t0dXtvEPxprM6tcHHga+ndS8EvttBOScAcyPittReVwAvtU1Mr52pad3LKN5Ujuro/kXENRHxemrPrwIHKu1V9iabaoD/PUXvonz4hIh4JSImRMR+FD2cOcBP0xO2zc4R0a/kNK+d7ewGtD2x2zxP0WOqxiDgmQrrXRgR73Sw3pdKLv+ZIvBrcR3wP8DNkl6U9E1JfSvU9HxZPZtTtF8tNS0pufxWO9fXLSvpjPQmuzINk+xP0SPryMIOplVq97a9n8Ul27qKoicORS9ZwIw09PFPndTQUU3rtWd6rBey/mNcdRu145VY/9jL82mbXdGfYs9sVkl7/He6vZbtLCt7M92N9dvkeSpbb94outDrrkvaRdLNadjrNeB6OniOSOoj6bI0fPYaRQ+djpZplk0uwFMv9jmKd+07Opl3OUXvZzeKXbmueBHYUdK2JbftwYY9/koWAu+usN5Bkkofu66st9ybFC/ENusO2EbEmoi4KCLeS9H7+yjrH0MorWlwWT1rWT9U6k7SYIqhmc8CO0VEP4pd7bY320o/tdnRT3BWaveFFEM/pW/e26U3eiLipYg4OyJ2oxh6+l7bWHGVSmtarz1T52EQtT/G5XaQtHXJ9T3SNsut99yQVHowfznFG8V+Je2xfUSUvnFUux3Y8DFZTHGfS5etZL15S9qrzaVp/e+LiO2Af6RkjLydbf89MIZiyHV7imFAypbpFTa5AE/OBI6Jdj4BIukbkvZPB5K2Bc4Bno6IFV3ZQNrtexC4VMXBwPel7d5Q5Sp+AHxB0sEq7JUC62GKF9b5kvpK+iBwInBzV+orMYdiuKOvpBEU44gASDpa0gEqDtS9RjFM0N4nNm4CzpM0RNI2FLvSP4nGf5Jga4oX37JU7ycpeuBtlgADJW3RhXX+EPikpGNVHJzdXdK+EbEY+F/g25K2S9PeLemotO1TJQ1M63gl1VXrp1tuAT6SaugL/AvFm8eDNa6vPRdJ2kLSSIo35lvbmedRYD9JwyS1UAwlAOv2Cq4GviNpF4DUVv+nhu205xZgoqQdUrv+3w7mvTfVeUoatvt/rP/JsW2BN4CVknYH/rVs+SUUx29K518NrKB4A/v3KmvucZtkgEfEMxExs8Lkv6EYQ1wJPEvREzqpbJ6VWv9z4P9cYV2nU7x7v5jW+ZWImFpljbcCXwdupDhI+VNgx4j4S6rnwxS9oO8BZ0TEk9Wstx0XUvQ4XwEuSttr8y7gNorwnkdxUPb6dtZxDcVwy3SKvZtVdPyCq4uImAt8G/gdxYvwAIoD0G1+RfFxvZckLa9ynTOAT1KMb79KcZ/besNnUBw4nkvRXrcBA9K0Q4CHVXzK5G7gcxHxXI336ymKXuKVFI/xicCJ6bGvh5co6n+RokPx6faePxHxR4rvDfyS4thC+bj+FykOrj6Uhhp+Cbynq9up4CKKYZPnKN44r6s0Y9pTPpXiwOcKYG/Wfx5cBAyneDzvZcM970uBf0tDQV+gGFp9nmKPZy7FAeNeScVwkZmZ5WaT7IGbmW0MHOBmZplygJuZZcoBbmaWqR79Maudd945Wltbe3KTZmbZmzVr1vKI6F9+e48GeGtrKzNnVvr0npmZtUdSu99E9RCKmVmmHOBmZplygJuZZWpT+0ceM2uSNWvWsGjRIlatKv8FX2vT0tLCwIED6du3vR/93JAD3Mx6xKJFi9h2221pbW1l/V9nNij+XGfFihUsWrSIIUOGVLWMh1DMrEesWrWKnXbayeFdgSR22mmnLu2hVNUDl7SA4hfx3gbWRsQISTtS/DtKK8UPnn88Il7pYs1mtglxeHesq+3TlR740RExLCJGpOsTgPsiYm/gvnTdzMx6SHfGwMdQ/BkpwBSKP7n9YjfrMbNNxHem/rGu6ztv9D51XV8Oqg3wAP5XUgBXRcRkYNf0LyVExOK2f+UoJ2k8MB5gjz06+lekjtX7wW6zKT7oZrZxqHYI5YiIGE7xLzCfkTSq2g1ExOSIGBERI/r33+Cr/GZmPWbBggUMHTqUs88+m/32248PfehDvPXWWzzzzDMcf/zxHHzwwYwcOZInn3ySt99+mz333JOIYOXKlWy22WZMnz4dgJEjR/L00083+d5UGeAR8WI6X0rx12CHAkskDQBI50sbVaSZWb3Mnz+fz3zmMzzxxBP069eP22+/nfHjx3PllVcya9YsJk2axLnnnkufPn3YZ599mDt3Lg888AAHH3ww999/P6tXr2bRokXstVdX/rO6MTodQkn/Kr1ZRLyeLn+I4n/y7gbGUvwP3VjgrkYWamZWD0OGDGHYsGEAHHzwwSxYsIAHH3yQU089dd08q1evBoqe9vTp03nuueeYOHEiV199NUcddRSHHHJIM0rfQDVj4LsCd6aPt2wO3BgR/y3p98Atks4EXqD4U1Ezs15tyy23XHe5T58+LFmyhH79+jFnzpwN5h05ciTf//73efHFF7n44ov51re+xbRp0xg1qupR5IbqdAglIp6NiAPTab+I+Hq6fUVEHBsRe6fzlxtfrplZfW233XYMGTKEW2+9FSi+Efnoo48C8P73v58HH3yQzTbbjJaWFoYNG8ZVV13FyJEjm1nyOv4qvZk1RW/6BNgNN9zAOeecwyWXXMKaNWs47bTTOPDAA9lyyy0ZNGgQhx12GFD0yG+66SYOOOCAJldcUET02MZGjBgRtf6hgz9GaJa3efPmMXTo0GaX0eu1106SZpV8iXId/xaKmVmmHOBmZplygJuZZcoBbmaWKQe4mVmmHOBmZpny58DNrDl+fWl913f0xLqt6oQTTuDGG2+kX79+Fef58pe/zKhRozjuuOO6vP5p06YxadIk7rnnnm5U6QA3M1snIogIfv7zn3c678UXX9wDFXXMQyhmtkm5/PLL2X///dl///254oor1v3E7Lnnnsvw4cNZuHAhra2tLF++HICvfe1r7LvvvowePZrTTz+dSZMmATBu3Dhuu+02AFpbW/nKV77C8OHDOeCAA3jyyScBmDFjBocffjgHHXQQhx9+OE899VRd74sD3Mw2GbNmzeJHP/oRDz/8MA899BBXX301r7zyCk899RRnnHEGs2fPZvDgwevmnzlzJrfffjuzZ8/mjjvuoKNvku+888488sgjnHPOOetCft9992X69OnMnj2biy++mAsuuKCu98dDKGa2yXjggQf42Mc+xtZbbw3AKaecwv3338/gwYPX/d5J+fxjxoxhq622AuDEE0+suO5TTjkFKH6i9o477gDg1VdfZezYscyfPx9JrFmzpq73xz1wM9tkVPrtp7ZAr3b+9rT9TG2fPn1Yu3YtABdeeCFHH300jz/+OD/72c9YtWpVFyvuWDY98MNemNygNU9q0HrNrLcZNWoU48aNY8KECUQEd955J9dddx2TJ7efL0ceeSSf+tSnmDhxImvXruXee+/l7LPPrnp7r776KrvvvjsA1157bT3uwnqyCXAz28jU8WN/1Ro+fDjjxo3j0EMPBeCss85ihx12qDj/IYccwkknncSBBx7I4MGDGTFiBNtvv33V2zv//PMZO3Ysl19+Occcc0y36y+Xzc/J/u6HX6hzNYUPnOkeuFlPyPXnZN944w222WYb/vznPzNq1CgmT57M8OHDG7a9rvycrHvgZmYdGD9+PHPnzmXVqlWMHTu2oeHdVQ5wM7MO3Hjjjc0uoSJ/CsXMekxPDtnmqKvt4wA3sx7R0tLCihUrHOIVRAQrVqygpaWl6mU8hGJmPWLgwIEsWrSIZcuWNbuUXqulpYWBAwdWPb8D3Mx6RN++fRkyZEizy9ioeAjFzCxTDnAzs0w5wM3MMuUANzPLlAPczCxTDnAzs0w5wM3MMuUANzPLlAPczCxTVQe4pD6SZku6J13fUdJUSfPTeeVfRTczs7rrSg/8c8C8kusTgPsiYm/gvnTdzMx6SFUBLmkg8BHgByU3jwGmpMtTgJPrWpmZmXWo2h74FcD5wDslt+0aEYsB0vku7S0oabykmZJm+lfIzMzqp9MAl/RRYGlEzKplAxExOSJGRMSI/v3717IKMzNrRzU/J3sEcJKkE4AWYDtJ1wNLJA2IiMWSBgBLG1momZmtr9MeeERMjIiBEdEKnAb8KiL+EbgbGJtmGwvc1bAqzcxsA935HPhlwGhJ84HR6bqZmfWQLv0jT0RMA6alyyuAY+tfkpmZVcPfxDQzy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsUw5wM7NMdelz4Buj70z9Y0PWe97ofRqyXjOzNu6Bm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaY2+f/EPOyFyQ1a86QGrdfMrOAeuJlZphzgZmaZcoCbmWXKAW5mlqlOA1xSi6QZkh6V9ISki9LtO0qaKml+Ot+h8eWamVmbanrgq4FjIuJAYBhwvKTDgAnAfRGxN3Bfum5mZj2k0wCPwhvpat90CmAMMCXdPgU4uREFmplZ+6oaA5fUR9IcYCkwNSIeBnaNiMUA6XyXhlVpZmYbqCrAI+LtiBgGDAQOlbR/tRuQNF7STEkzly1bVmOZZmZWrkufQomIlcA04HhgiaQBAOl8aYVlJkfEiIgY0b9//+5Va2Zm61TzKZT+kvqly1sBxwFPAncDY9NsY4G7GlSjmZm1o5rfQhkATJHUhyLwb4mIeyT9DrhF0pnAC8CpDazTzMzKdBrgEfEYcFA7t68Ajm1EUWZm1jl/E9PMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFOdBrikQZJ+LWmepCckfS7dvqOkqZLmp/MdGl+umZm1qaYHvhb4l4gYChwGfEbSe4EJwH0RsTdwX7puZmY9pNMAj4jFEfFIuvw6MA/YHRgDTEmzTQFOblCNZmbWji6NgUtqBQ4CHgZ2jYjFUIQ8sEuFZcZLmilp5rJly7pZrpmZtak6wCVtA9wOfD4iXqt2uYiYHBEjImJE//79a6nRzMzaUVWAS+pLEd43RMQd6eYlkgak6QOApY0p0czM2lPNp1AE/BCYFxGXl0y6GxibLo8F7qp/eWZmVsnmVcxzBPAJ4A+S5qTbLgAuA26RdCbwAnBqQyo0M7N2dRrgEfEAoAqTj61vOWZmVi1/E9PMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QA3M8tUpwEu6RpJSyU9XnLbjpKmSpqfzndobJlmZlaumh74tcDxZbdNAO6LiL2B+9J1MzPrQZ0GeERMB14uu3kMMCVdngKcXN+yzMysM7WOge8aEYsB0vkulWaUNF7STEkzly1bVuPmzMysXMMPYkbE5IgYEREj+vfv3+jNmZltMmoN8CWSBgCk86X1K8nMzKpRa4DfDYxNl8cCd9WnHDMzq1Y1HyO8Cfgd8B5JiySdCVwGjJY0HxidrpuZWQ/avLMZIuL0CpOOrXMtZmbWBf4mpplZphzgZmaZcoCbmWXKAW5mlikHuJlZphzgZmaZcoCbmWXKAW5mlikHuJlZphzgZmaZcoCbmWXKAW5mlikHuJlZphzgZmaZcoCbmWXKAW5mlikHuJlZphzgZmaZcoCbmWXKAW5mlikHuJlZphzgZmaZcoCbmWXKAW5mlikHuJlZphzgZmaZcoCbmWXKAW5mlqnNm13ARuvXl9Z/nUdPrP86zSxb7oGbmWXKAW5mlikHuJlZpjwG3iC/e3ZF/Vf67Bfqv07gA2dOash6G3IcoFEadXyhUW3g4yFGN3vgko6X9JSkpyVNqFdRZmbWuZoDXFIf4D+BDwPvBU6X9N56FWZmZh3rTg/8UODpiHg2Iv4C3AyMqU9ZZmbWGUVEbQtKfwscHxFnpeufAN4fEZ8tm288MD5dfQ/wVO3l1t3OwPJmF1GBa6uNa6uNa6tNT9U2OCL6l9/YnYOYaue2Dd4NImIyMLkb22kYSTMjYkSz62iPa6uNa6uNa6tNs2vrzhDKImBQyfWBwIvdK8fMzKrVnQD/PbC3pCGStgBOA+6uT1lmZtaZmodQImKtpM8C/wP0Aa6JiCfqVlnP6JVDO4lrq41rq41rq01Ta6v5IKaZmTWXv0pvZpYpB7iZWaY2mgDv7Gv9Knw3TX9M0vCSaddIWirp8bJldpQ0VdL8dL5DL6rtq5L+JGlOOp3Qk7VJGiTp15LmSXpC0udKlmlqu3VSW7PbrUXSDEmPptouKlmmLu3WwPqa2nYl0/tImi3pnpLbmv5a7aC2urRbuyIi+xPFQdRngD2BLYBHgfeWzXMC8AuKz68fBjxcMm0UMBx4vGyZbwIT0uUJwDd6UW1fBb7QrHYDBgDD0+VtgT+2Ldvsduuktma3m4Bt0uW+wMPAYfVqtwbX19S2K5n+z8CNwD296bXaQW3dbrdKp42lB17N1/rHAD+OwkNAP0kDACJiOvByO+sdA0xJl6cAJ/ei2uqh5toiYnFEPJJqfB2YB+xeskzT2q2T2uqhO7VFRLyR5umbTlGyTHfbrZH11UO3Xg+SBgIfAX7QzjJNfa12UFvDbCwBvjuwsOT6IjZ8wVYzT7ldI2IxQDrfpRfVBvDZtBt3TY27jHWpTVIrcBBFbw16Ubu1Uxs0ud3SbvYcYCkwNSLq2W6NrA+a/5y7AjgfeKdsmd7wnKtUG3S/3dq1sQR4NV/rr+qr/w3QqNr+C3g3MAxYDHy7y5XVoTZJ2wC3A5+PiNdqqKGna2t6u0XE2xExjOLby4dK2r+GGjrSqPqa2naSPgosjYhZNWy3Go2qrR7t1q6NJcCr+Vp/LV/9X1KyezSAokfSK2qLiCXphfYOcDXF7l+P1iapL0VA3hARd5TM0/R2q1Rbb2i3klpWAtOA49NN9Wi3htXXC9ruCOAkSQsohjeOkXR9mqfZz7mKtdWp3dpX70H1ZpwovlH6LDCEvx582K9sno+w/sGHGWXTW9nwQOG3WP/AyDd7UW0DSi6fB9zck7Wl6z8GrmhnvU1tt05qa3a79Qf6pctbAfcDH61XuzW4vqa2Xdk8H2T9A4W94rVaobZut1vFmuu1omafKI4O/5HiKPKX0m2fBj6dLoviDyieAf4AjChZ9iaKXZs1FO+wZ6bbdwLuA+an8x17UW3XpXkfo/gNmgE9WRtwJMXu5WPAnHQ6oTe0Wye1Nbvd3gfMTtt/HPhyyTrr0m4NrK+pbVe2jg+yfkg2/bXaQW11abf2Tv4qvZlZpjaWMXAzs02OA9zMLFMOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTP1/pHGCADevkiAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xmin = np.min(np.append(sym_losses, asym_losses))\n",
    "xmax = np.percentile(np.append(sym_losses, asym_losses), 95)\n",
    "bins = np.linspace(xmin, xmax, 15)\n",
    "plt.hist(new_losses, bins, alpha=0.5, label='new')\n",
    "plt.hist(original_losses, bins, alpha=0.5, label='original')\n",
    "plt.title(\"MSE of confusion matrices from rte data\")\n",
    "plt.legend(loc='upper right')\n",
    "plt.savefig(\"pic/rte_singular.png\",dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjb0lEQVR4nO3de3wc5X3v8c/XsmQ5xokhOBfwBTdAkKIEElyaE9QkSgsHpyc1aZOC4ZQmErdDLWhzUqBnkzQ0UQl99RKqQinBAnKxIMkB160hQA7bEOEm2ARIjAWpgzEWhmAnBoyLLFn8zh8zclZCl5WwdqXR9/167UuamWdmfjs7+9tnnmcuigjMzCy7ZpQ7ADMzm1hO9GZmGedEb2aWcU70ZmYZ50RvZpZxTvRmZhnnRD+JKXGjpN2SHijhek+W9J+SXpJ0+mtYzp2S/ugghlY2kq6T9NkSru+jkrann8G7S7XedN0flNQ1wvSbJH2xBOsZ0zYfbXlDlP93SecWW34qc6IvIOlJST2SDh80/mFJIemodHiBpP8raZekFyT9RNIn0mlHpWVfGvQ6Yxwh1QOnAAsi4qTX+PbG4i+Bf4yIQyJizXgXEhHLIuLmgxfWwSfpE5I6RisXERdGxBdKEVPqb4CV6WfwUAnXO2mUYZsPK80Nv13uOMZrZrkDmIS2AiuAVgBJ7wRmDyrzNeARYDGwD3gn8JZBZeZFxP7XGMti4MmI2PsalzOe9T5a4nVOWpIqIqKvxKsd9jOQNPMg7FtTmrfB2LhG/2pfA84pGP4j4KuDyvw6cFNE7I2I/RHxUETcOZ6VSTpC0lpJv5S0RdJ56fgm4Abgv6VHBFcMM/95kjol7ZG0WdJ70vE16aHp85IelfS7BfPcJOkaSevS+X4o6W3ptJ8Bvwb8a7reWYNrM5I+L+nr6f/Vkr4u6RfpujZIenM67cChsaQZkj4jaZuk5yR9VdIb0mn9R0F/JOmp9EgpN8I2u0nStWnT0EuS7pf0FklfTpu5Hits7pB0uaSfFWyjj/ZvI+C6gm38fMHy/0nSHZL2Ag2DmyskLU+P9F5Ml31aOv4NklZJekbS05K+KKkinXa0pO+lR4G7JN06xHubJekloAJ4JP08+muUl0n6MbBX0kxJv5t+ts+n27qmYDlPSvozST+WtDeN6c3pNtsj6buSDh1uG6fL+D9pnE9KOnuYMq86Iko/y6ML3s/fpJ/rz5U0x8weVH7I9RRuc6XNMuk2eBa4UdLstMxuSZtJvpcjvZ9T0n3jBUn/CKhg2tsk3Zvux7skfUPSvHTa14BF/Oo7cWk6/luSnk2Xd5+kd4y0/rKKCL/SF/Ak8NvA40ANyZdtO0ntKoCj0nLfBe4HzgQWDVrGUWnZmUWu83vAtUA1cAKwE/itdNongI4R5v048DTJDi7g6DTWSmAL8H+AKuBDwB7g7el8NwG/BE4iOar7BnDL4O0wwvDnga+n/18A/CvwunR7nQi8Pp3278C56f+NaUy/BhwC3AZ8bdA2+wrJ0dPxJEdKNcO875uAXem6qoF7SY7Ezklj+CKQH7SdjiCp2JwB7AXeOtw2Tpf/AnByOk91Ou6L6fST0umnpNOPBI5Lp60B/hmYA7wJeAC4IJ3WDuQKllk/wmcbwNGDPoOHgYXpNjo2fR+npJ/3pen2rSoo/wPgzWl8zwE/At4NzEq32V8Ms+4PAvuBv0vLfiBdV+H+88URtt+B2IEvA2uBw4C5JPvKleNYT3/Zq9Kys4EvAd9Pl70Q2AR0DfOeDgdeBD6Wbq8/TZfXv38enW7LWcB84D7gy8N9Bwr26bnpPF8GHi53Dhvu5Rr90Ppr9acAj5Ek00IfJ9nBPgtsTWt2g2sTu9KaVv+rZtB0JC0kaYe/LCK6I+Jhklr8HxYZ57nAX0fEhkhsiYhtwHtJkumXIqInIu4F/o2kSarfbRHxQCSHv98g+ZEZj17gjSRf7L6IeDAiXhyi3NnA30XEExHxEvDnwJmSCpsPr4iIlyPiEZKmseNHWO/t6bq6gduB7oj4aiRNLLeSJDQAIuJbEbEjIl6JiFuB/yRJ1iP5l4i4P52ne9C0JqAtIu5Jpz8dEY8pOZJZBvxJJEd7zwF/T1Ih6N9Wi4Ej0s971L6BQf4hIrZHxMskP1jr0hh6Sdr0ZwPvKyjfGhE/j4inSfbXH0Zy9Lkv3WajdfJ+NiL2RcT3gHXAH4wlWEkCzgP+NCJ+GRF7gL/iV9tjrOt5heTHaV+6Df4AaEmXvR34hxHC+TCwOSK+nW6vLwPP9k9Mvzv3pMveSfLj84GR3l9EtEXEnnR7fh44XulR6mTjRD+0rwFnkdRWBjfbEBG7I+LyiHgHSY3pYWBNumP3Ozwi5hW8OodYzxFA/xeg3zaSGlgxFgI/G2a52yPilRGW+2zB//9F8sMwHl8D7gJukbRD0l9Lqhwmpm2D4plJsv3GE9PPC/5/eYjhA/NKOif9MX4+bZ6pI6nhjWT7CNOG2+79R1PPFKzrn0lq9pDUugU8kDa5NI4Sw0gxDdie6We9nYGfcdHbaAi7Y2Df0LZ0nWMxn+RI78GC7fGddPx41rNz0I/uEQzcJtsY3oCykVTJDwxLepOkW9LmtheBrzPCPiKpQtKX0ma7F0lq/Iw0Tzk50Q8hrRVvJakF3DZK2V0ktakjSA4hx2IHcJikuQXjFvHqI4jhbAfeNsxyF0oq/HzHstzB9pJ8Yfsd6HiOiN6IuCIiaklqk/+DgX0chTEtHhTPfgYmn4NO0mKSJqGVwBsjYh7JIX7/j/Jwt28d6bauw2337SRNToU/8q9PKwRExLMRcV5EHEHS5HVtf1t2kQpjGrA900rGQsb/GQ92qKQ5BcOL0nUONmDfkFR4UsIukh+UdxRsjzdEROEPTLHrgVd/Js+QvOfCeYczoGzB9up3Zbr8d0XE64H/SUEb/hDrPgtYTtLU+waS5kcGzTNpONEPrwn4UAxxxoukqyTVpR1ic4H/BWyJiF+MZQXp4eZ64EolnZrvStf7jSIXcQPwaUknKnF0mth+SPIFvFRSpaQPAh8BbhlLfAUeJmlmqZS0lKSdEwBJDZLeqaTD8UWS5omhzlBpB/5U0hJJh5Acwt8aE3/mxBySL+nONN5PktTo+/0cWCCpagzLXAV8UtJvKelkPlLScRHxDHA38LeSXp9Oe5ukD6Tr/rikBekydqdxjfdsnm8Cv5PGUAn8b5IfmfXjXN5QrpBUJek3SX7AvzVEmUeAd0g6QVI1SRMGcOAo4yvA30t6E0C6rf77ONYzlG8Cfy7p0HS7No9Qdl0a5++lzYUXM/BMubnAS8Dzko4E/mzQ/D8n6V8qLL8P+AXJD91fFRlzWTjRDyMifhYRG4eZ/DqSNs7ngSdIala/O6jM8xp4Hv2nhlnWCpLawI50mX8REfcUGeO3gBZgNUln6xrgsIjoSeNZRlKruhY4JyIeK2a5Q/gsSQ12N3BFur5+bwG+TZLkO0k6l78+xDLaSJp57iM5Wupm5C/mQRERm4G/Bf6D5Mv6TpKO9H73kpzG+KykXUUu8wHgkyTt7y+QvOf+2vU5JB3gm0m217eBt6bTfh34oZKzatYCl0TE1nG+r8dJap2tJJ/xR4CPpJ/9wfAsSfw7SCoeFw61/0TET0muu/guSd/H4H6Hy0g6iX+QNnF8F3j7WNczjCtImmu2kvzAfm24gumR98dJOnB/ARzDwP3gCuA9JJ/nOl59JH8l8Jm0CerTJE2620iOoDaTdHxPWkqaqszMLKtcozczyzgnejOzjHOiNzPLOCd6M7OMm5Q3NTv88MPjqKOOKncYZmZTxoMPPrgrIuYPNW1SJvqjjjqKjRuHO7PRzMwGkzTslcFuujEzyzgnejOzjHOiNzPLOCd6M7OMc6I3M8s4J3ozK7n29nbq6uqoqKigrq6O9vb2coeUaZPy9Eozy6729nZyuRyrVq2ivr6ejo4OmpqaAFixYsUoc9t4TMq7Vy5dujR8Hr1ZNtXV1dHa2kpDQ8OBcfl8nubmZjZt2lTGyKY2SQ9GxNIhpznRm1kpVVRU0N3dTWXlr5442dvbS3V1NX19430Oi42U6N1Gb2YlVVNTQ0fHwOeTdHR0UFNTU6aIss+J3sxKKpfL0dTURD6fp7e3l3w+T1NTE7lcrtyhZZY7Y82spPo7XJubm+ns7KSmpoaWlhZ3xE4gt9GbmWWA2+jNzKYxJ3ozs4xzos8oX3loZv3cGZtBvvLQzAq5MzaDfOWh2fTjK2OnGV95aDb9+KybacZXHppZISf6DPKVh2ZWyJ2xGeQrD82skNvozcwy4DW30Us6TdLjkrZIunyI6csl/VjSw5I2Sqovdl4zM5tYoyZ6SRXANcAyoBZYIal2ULH/BxwfEScAjcANY5jXzMwmUDE1+pOALRHxRET0ALcAywsLRMRL8as2oDlAFDuvmZlNrGIS/ZHA9oLhrnTcAJI+KukxYB1Jrb7oedP5z0+bfTbu3LmzmNjNzKwIxSR6DTHuVT24EXF7RBwHnA58YSzzpvNfHxFLI2Lp/PnziwjLzMyKUUyi7wIWFgwvAHYMVzgi7gPeJunwsc5rZmYHXzGJfgNwjKQlkqqAM4G1hQUkHS1J6f/vAaqAXxQzr5mZTaxRL5iKiP2SVgJ3ARVAW0Q8KunCdPp1wO8D50jqBV4Gzkg7Z4ecd4Lei5mZDcEXTJmZZYBvamZmNo050ZuZZZwTvZlZxjnRm5llnBO9mVnGOdGbWcm1t7dTV1dHRUUFdXV1tLe3lzukTPODR8yspNrb28nlcqxatYr6+no6OjpoamoC8MNxJojPozezkqqrq6O1tZWGhoYD4/L5PM3NzWzatKmMkU1tI51H70RvZiVVUVFBd3c3lZWVB8b19vZSXV1NX19fGSOb2nzBlJlNGjU1NXR0dAwY19HRQU1NTZkiyj4nejMrqVwuR1NTE/l8nt7eXvL5PE1NTeRyuXKHllnujDWzkurvcG1ubqazs5OamhpaWlrcETuB3EZvZpYBbqM3M5vGnOjNzDLOid7MLOOc6M3MMs6J3sws45zozcwyzonezCzjnOjNzDLOid7MLOOKSvSSTpP0uKQtki4fYvrZkn6cvtZLOr5g2pOSfiLpYUm+3NXMrMRGvdeNpArgGuAUoAvYIGltRGwuKLYV+EBE7Ja0DLge+I2C6Q0Rsesgxm1mZkUqpkZ/ErAlIp6IiB7gFmB5YYGIWB8Ru9PBHwALDm6YZmY2XsUk+iOB7QXDXem44TQBdxYMB3C3pAclnT/cTJLOl7RR0sadO3cWEZaZmRWjmNsUa4hxQ97yUlIDSaKvLxh9ckTskPQm4B5Jj0XEfa9aYMT1JE0+LF26dPLdUtPMbIoqpkbfBSwsGF4A7BhcSNK7gBuA5RHxi/7xEbEj/fsccDtJU5CZmZVIMYl+A3CMpCWSqoAzgbWFBSQtAm4D/jAiflowfo6kuf3/A6cCfvqvmVkJjdp0ExH7Ja0E7gIqgLaIeFTShen064DPAW8ErpUEsD+9Af6bgdvTcTOB1RHxnQl5J2ZmNqSizqOPiDsi4tiIeFtEtKTjrkuTPBFxbkQcGhEnpK+l6fgnIuL49PWO/nlt4rW3t1NXV0dFRQV1dXW0t7eXOyQzKxM/MzaD2tvbyeVyrFq1ivr6ejo6OmhqagLwcznNpiE/MzaD6urqaG1tpaGh4cC4fD5Pc3Mzmza5i8Qsi0Z6ZqwTfQZVVFTQ3d1NZWXlgXG9vb1UV1fT19dXxsjMbKL44eDTTE1NDR0dHQPGdXR0UFNTU6aIzKycnOgzKJfL0dTURD6fp7e3l3w+T1NTE7lcrtyhmVkZuDM2g1asWMH69etZtmwZ+/btY9asWZx33nnuiDWbplyjz6D29nbWrVvHnXfeSU9PD3feeSfr1q3zKZZm05Q7YzOorq6O008/nTVr1tDZ2UlNTc2BYZ91Y5ZNI3XGuukmgzZv3szevXtpa2s7cB59Y2Mj27ZtK3doZlYGTvQZVFVVxcknn0xzc/OBGv3JJ5/MM888U+7QzKwM3EafQfv27ePWW2+lsbGRPXv20NjYyK233sq+ffvKHZqZlYETfQbNmjWLM844g7a2NubOnUtbWxtnnHEGs2bNKndoZlYGTvQZ1NPTw/33309rayvd3d20trZy//3309PTU+7QzKwM3EafQbW1tZx++ukD2ujPPvts1qxZU+7QzKwMXKPPoFwux+rVqwfU6FevXu0rY82mKdfoM6j/CtjCGn1LS4uvjDWbpnzBlJlZBvjulWZm05gTvZlZxjnRm5llnBO9mVnGFZXoJZ0m6XFJWyRdPsT0syX9OH2tl3R8sfOamdnEGjXRS6oArgGWAbXACkm1g4ptBT4QEe8CvgBcP4Z5zcxsAhVToz8J2BIRT0RED3ALsLywQESsj4jd6eAPgAXFzmtm0097ezt1dXVUVFRQV1fnh+JMsGIS/ZHA9oLhrnTccJqAO8c5r5llXHt7O5dccgl79+4FYO/evVxyySVO9hOomESvIcYNeZWVpAaSRH/ZOOY9X9JGSRt37txZRFhmNhVdeumlzJw5k7a2Nrq7u2lra2PmzJlceuml5Q4ts4pJ9F3AwoLhBcCOwYUkvQu4AVgeEb8Yy7wAEXF9RCyNiKXz588vJnYzm4K6urq4+eabaWhooLKykoaGBm6++Wa6urrKHVpmFZPoNwDHSFoiqQo4E1hbWEDSIuA24A8j4qdjmdfMzCbWqIk+IvYDK4G7gE7gmxHxqKQLJV2YFvsc8EbgWkkPS9o40rwT8D7MbIpYsGAB55xzDvl8nt7eXvL5POeccw4LFiwYfWYbF9/UzMxKqr8zds6cOWzbto3Fixezd+9err76at9h9TXwTc3MbNJYsWIFV199NXPmzEESc+bMcZKfYK7Rm5llgGv0ZmbTmBO9mVnGOdGbmWWcE72ZWcY50ZuZZZwTvZlZxjnRm5llnBO9mVnGOdGbmWWcE72ZWcY50ZuZZZwTvZlZxjnRm5llnBO9mVnGOdGbmWWcE72ZWcY50ZuZZZwTvZlZxjnRm5llnBO9mVnGFZXoJZ0m6XFJWyRdPsT04yT9h6R9kj49aNqTkn4i6WFJfuK3mVmJzRytgKQK4BrgFKAL2CBpbURsLij2S+Bi4PRhFtMQEbteY6xmZjYOxdToTwK2RMQTEdED3AIsLywQEc9FxAagdwJiNDOz16CYRH8ksL1guCsdV6wA7pb0oKTzhysk6XxJGyVt3Llz5xgWb2ZmIykm0WuIcTGGdZwcEe8BlgF/LOn9QxWKiOsjYmlELJ0/f/4YFm9mZiMpJtF3AQsLhhcAO4pdQUTsSP8+B9xO0hRkZmYlUkyi3wAcI2mJpCrgTGBtMQuXNEfS3P7/gVOBTeMN1szMxm7URB8R+4GVwF1AJ/DNiHhU0oWSLgSQ9BZJXcCngM9I6pL0euDNQIekR4AHgHUR8Z2JejNmNjW0t7dTV1dHRUUFdXV1tLe3lzukTBv19EqAiLgDuGPQuOsK/n+WpElnsBeB419LgGaWLe3t7eRyOVatWkV9fT0dHR00NTUBsGLFijJHl02KGEu/amksXbo0Nm70tVVmWVRXV0draysNDQ0HxuXzeZqbm9m0yS274yXpwYhYOuQ0J3ozK6WKigq6u7uprKw8MK63t5fq6mr6+vrKGNnUNlKi971uzKykampq6OjoGDCuo6ODmpqaMkWUfU70GeXOLpuscrkcTU1N5PN5ent7yefzNDU1kcvlyh1aZhXVGWtTizu7bDLr3webm5vp7OykpqaGlpYW75sTyG30GeTOLrPpx52x04w7u8ymH3fGTjPu7DKzQk70GeTOLpvsfLJAabkzNoPc2WWTmU8WKD3X6M2spFpaWli1ahUNDQ1UVlbS0NDAqlWraGlpKXdomeUafQa5xmSTWWdnJ/X19QPG1dfX09nZWaaIss81+gxqaWnhrLPOorm5merqapqbmznrrLNcY7JJoaamhiuuuGJAG/0VV1zhkwUmkBN9Bm3evJnVq1fT2tpKd3c3ra2trF69ms2bN48+s9kEa2ho4KqrrqKxsZE9e/bQ2NjIVVddNeC6Dzu4nOgzqKqqipUrVw5oA125ciVVVVXlDs2MfD7PZZddRltbG3PnzqWtrY3LLruMfD5f7tAyyxdMZdCMGTNYvHgxbW1tB9roGxsb2bZtG6+88kq5w7Npzhf0TQxfMDXN1NbWcvbZZw9ooz/77LOpra0td2hmvqCvDHzWTQblcrkhz7pxZ6xNBrlcjjPOOIM5c+bw1FNPsWjRIvbu3cvVV19d7tAyy4k+g3zBlE0Vk7HpOIvcRm9mJeW7q04M373SzCYNd8ZODHfGmtmk4QumSq+oRC/pNEmPS9oi6fIhph8n6T8k7ZP06bHMa2bTS0NDA1deeSW7du3ilVdeYdeuXVx55ZW+YGoCjZroJVUA1wDLgFpghaTB5+n9ErgY+JtxzGtm08iaNWuYO3cus2fPZsaMGcyePZu5c+eyZs2acoeWWcXU6E8CtkTEExHRA9wCLC8sEBHPRcQGoHes85rZ9NLV1cVFF13EnDlzAJgzZw4XXXQRXV1dZY4su4pJ9EcC2wuGu9JxxSh6XknnS9ooaePOnTuLXLwNxw92sMnsxhtvHHAvphtvvLHcIWVaMefRa4hxxZ6qU/S8EXE9cD0kZ90UuXwbgm9TbJPZzJkz6enpGTCup6eHmTN9Wc9EKaZG3wUsLBheAOwocvmvZV4bJz/YwSazvr4+ZsyYQWNjI9XV1TQ2NjJjxgyfWjmBikn0G4BjJC2RVAWcCawtcvmvZV4bJz/YwSaz2tpaLrjgggFt9BdccIHvxTSBRk30EbEfWAncBXQC34yIRyVdKOlCAElvkdQFfAr4jKQuSa8fbt6JejOW8E2jbDLL5XJDPi/BD6+fQBEx6V4nnnhi2PitXr06lixZEvfee2/09PTEvffeG0uWLInVq1eXOzSziIhYuXJlzJo1K4CYNWtWrFy5stwhTXnAxhgmp7r3I4N8UzObzNrb21m3bh133nnngJMF3ve+93kfnSC+142ZlZRvajYxfFMzM5s0fFOzieGbmpnZpOGTBUrPbfRmVlK5XI7TTz+dl19+md7eXiorK5k9ezbXXXdduUPLLNfozayk1q9fz549ew48XSoi2LNnD+vXry9zZNnlRG9mJXXddddx6KGHcvfdd9PT08Pdd9/NoYce6hr9BHKiN7OS2r9/P+eeey7Nzc1UV1fT3NzMueeey/79+8sdWma5jd7MSu6GG27g29/+9oHz6D/2sY+VO6RMc43ezEqqoqKC3bt389BDD9Hb28tDDz3E7t27qaioKHdomeXz6M2spGbMmMEhhxxCd3f3gbNuqqureemll3jllVfKHd6U5fPozWzSqK2t5eKLL+bYY49lxowZHHvssVx88cW+e+UEcqI3s5Ly3StLz52xZlZSvule6blGn1F+ZqxNZitWrGDTpk309fWxadMmJ/kJ5hp9BvmZsWZWyGfdZJBvA2s2/fg2xdOMbwNrNv349MppxreBNbNCTvQZlMvlaGpqIp/P09vbSz6fp6mpyaevmU1T7ozNIJ++ZmaFimqjl3QacDVQAdwQEV8aNF3p9A8D/wV8IiJ+lE57EtgD9AH7h2tDKuQ2ejOzsRmpjX7UGr2kCuAa4BSgC9ggaW1EbC4otgw4Jn39BvBP6d9+DRGxa5zxm5nZa1BMG/1JwJaIeCIieoBbgOWDyiwHvhqJHwDzJL31IMdqZmbjUEyiPxLYXjDclY4rtkwAd0t6UNL54w3UzLLDV26XVjGdsRpi3OCG/ZHKnBwROyS9CbhH0mMRcd+rVpL8CJwPsGjRoiLCMrOpyFdul14xNfouYGHB8AJgR7FlIqL/73PA7SRNQa8SEddHxNKIWDp//vziojezKaelpYVVq1bR0NBAZWUlDQ0NrFq1ipaWlnKHllnFJPoNwDGSlkiqAs4E1g4qsxY4R4n3Ai9ExDOS5kiaCyBpDnAq4Gvwzaaxzs5O6uvrB4yrr6+ns7OzTBFl36iJPiL2AyuBu4BO4JsR8aikCyVdmBa7A3gC2AJ8BbgoHf9moEPSI8ADwLqI+M5Bfg9mNoX4yu3SK+qCqYi4gySZF467ruD/AP54iPmeAI5/jTGaWYb0X7k9uI3eTTcTx1fGmllJ+crt0vPdK83MMsB3rzQzm8ac6M2s5HzBVGm5jd7MSsoXTJWe2+jNrKT8qMuJ4TZ6M5s0Ojs76erqGtB009XV5QumJpCbbsyspI444gguvfRSVq9efaDp5qyzzuKII44od2iZ5URvZiXX3d1NY2Mj27ZtY/HixXR3d3PIIYeUO6zMctONmZXU008/TWVlJQDJw+mgsrKSp59+upxhZZoTvZmVVFVVFZdffjlbt26lr6+PrVu3cvnll1NVVVXu0DLLid7MSqqnp4fW1lby+Ty9vb3k83laW1vp6ekpd2iZ5USfUb4gxSar2tpaTjjhBJYtW0ZVVRXLli3jhBNOoLa2ttyhZZYTfQb1X5DS2tpKd3c3ra2t5HI5J3ubFBoaGli7di3z5s1DEvPmzWPt2rUDzqu3g8sXTGWQL0ixyWzhwoW89NJLzJs3j6eeeopFixbx/PPPc8ghh7B9+/bRF2BDGumCKSf6DKqoqKC7u/vAmQ0Avb29VFdX09fXV8bIzJIzbe6++25OOeWUA+PuueceTj31VCZjPpoqfGXsNOMn+JhZIV8wlRH95yP3+9CHPlRUOdegbKIN3ucATj311FHLet88eFyjn2IOO+wwJL3qNV5DLeuwww47iBHbdOF9c/JyjX6K+eXFfcDrJ3gtbse3sfO+OXk50U81n39hTMUl+RDYSmOM+yZ4/ywVJ/qMGOkQeaRp/pLZRBut+Wa46d43D56i2uglnSbpcUlbJF0+xHRJ+od0+o8lvafYee3giIhxvcwmmvfN8hs10UuqAK4BlgG1wApJg69VXgYck77OB/5pDPOamdkEKqZGfxKwJSKeiIge4BZg+aAyy4GvRuIHwDxJby1yXjMzm0DFJPojgcLrkrvSccWUKWZeACSdL2mjpI07d+4sIiwzMytGMYl+qJ6SwQ1ow5UpZt5kZMT1EbE0IpbOnz+/iLDMzKwYxZx10wUsLBheAOwoskxVEfOamdkEKqZGvwE4RtISSVXAmcDaQWXWAuekZ9+8F3ghIp4pcl4zM5tAo9boI2K/pJXAXUAF0BYRj0q6MJ1+HXAH8GFgC/BfwCdHmndC3omZmQ3Jtyk2M8uAKXc/ekk7gW3ljiMjDgd2lTsIs2F4/zx4FkfEkGeyTMpEbwePpI3D/cqblZv3z9LwbYrNzDLOid7MLOOc6LPv+nIHYDYC758l4DZ6M7OMc43ezCzjnOjNzDLOid7MJoSkOyTNG6XMX0r67XEu/4OS/m1cwU0zfpSgmR1USp4NqIj48GhlI+JzJQhp2nONfgqTdJSkTklfkfSopLslzZb0NknfkfSgpO9LOk5ShaQn0hvPzZP0iqT3p8v5vqSjy/1+bOqQ9ClJm9LXnxTsi9cCPwIWSnpS0uFp+c9KekzSPZLaJX06HX+TpI+l/z8p6QpJP5L0E0nHpeNPkrRe0kPp37eX631PVU70U98xwDUR8Q7geeD3SU5Za46IE4FPA9dGRB/wU5JHOtYDDwK/KWkWsCAitpQjeJt6JJ1IcuPC3wDeC5wHHAq8neRJc++OiG0F5ZeS7JfvBn4PGOlK2F0R8R6Sx5F+Oh33GPD+iHg38Dngrw7uO8o+N91MfVsj4uH0/weBo4D3Ad9KjqABmJX+/T7wfmAJcCXJF/R7JLeTNitWPXB7ROwFkHQb8JvAtvRRokOV/5eIeDkt/68jLPu29O+DJD8KAG8AbpZ0DMmDiypf+1uYXlyjn/r2FfzfBxwGPB8RJxS8atLp3yf5Qp5EcmvpecAHgftKF65lwFBPjgPYO8byQ+nfn/v4VUX0C0A+IuqAjwDVY1ie4USfRS8CWyV9HJKOMUnHp9N+SFLbfyUiuoGHgQtIfgDMinUfcLqk10maA3yUkfehDuAjkqolHQL8zhjX9wbg6fT/T4w1WHOiz6qzgSZJjwCPAssBImIfycPa+w+vvw/MBX5SjiBtaoqIHwE3AQ+QVB5uAHaPUH4DyZPlHiFpmtkIvDCGVf41cKWk+0keYGRj5FsgmNmEk3RIRLwk6XUkRwTnpz8YVgLujDWzUrheUi1J+/rNTvKl5Rq9mVnGuY3ezCzjnOjNzDLOid7MLOOc6M3MMs6J3sws4/4/x3XLVgAriZ0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.boxplot([new_losses, original_losses], labels=['new', 'original'])\n",
    "plt.title(\"MSE of confusion matrices from rte data\")\n",
    "plt.savefig(\"pic/rte_singular_box.png\", dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "19\n",
      "29\n",
      "39\n",
      "49\n",
      "59\n",
      "69\n",
      "79\n",
      "89\n",
      "99\n"
     ]
    }
   ],
   "source": [
    "# balance or not\n",
    "iternum = 100\n",
    "balance_losses = np.zeros(iternum)\n",
    "inbalance_losses = np.zeros(iternum)\n",
    "for i in range(iternum):\n",
    "    if i % 10 == 9:\n",
    "        print(i)\n",
    "    balance_losses[i] = confusion_matrix_loss(get_confusion_matrix(\n",
    "        k, labels, sym=True, groups=np.random.permutation([i % 3 for i in range(m)]), L=10, N=10), C_true)\n",
    "    inbalance_losses[i] = confusion_matrix_loss(get_confusion_matrix(\n",
    "        k, labels, sym=True, L=10, N=10), C_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdNklEQVR4nO3de7xUdb3/8df7AIInMeVmKCiYZoSiJqCmv8SUstT0FJ70WGFeyApTz/Gk5ul3pF8dzW6EHY9SecBLXsJKs1+nH966mIoQiAjeQUURtnhFRUE+vz/Wd9PsYfbes2fPvnzh/Xw89mNm1vqutT7znZn3rPVdM7MVEZiZWX7+rqsLMDOz2jjAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5QDvIir8t6SXJM3pxO0eJOkxSWskHduO9fxO0sQ6ltZlJF0u6RuduL1/kPRMegz27aztpm2Pk7S8hfkzJH2rE7bTpj5vbX0V2t8l6dRq2+dqiwlwScskvS1pQNn0BZJC0rB0e4ikmyS9IOkVSQ9KOinNG5barin7+0wNJR0MjAeGRMTYdt69tvgm8OOI2CYifl3rSiLi4xExs35l1Z+kkyT9ubV2EXF6RPyfzqgp+R4wOT0G8ztxu91GF/R5s1I2HN7VddSiZ1cX0MmWAicAlwJI2gvYuqzN1cADwC7AW8BewHvK2mwXEevbWcsuwLKIeL2d66lluw918ja7LUk9IuKdTt5ss4+BpJ51eG5lzX1QvS1mDzy5Gvh8ye2JwFVlbcYAMyLi9YhYHxHzI+J3tWxM0o6SbpH0oqTHJZ2Wpp8C/BQ4MO3BT2lm+dMkLZH0mqTFkj6Ypo9Ih4gvS3pI0idLlpkh6T8l/TYtd5+k96Z5TwC7Ar9J2+1dvvch6UJJ16TrfSRdI2l12tb9knZI8zYeokr6O0n/JukpSaskXSXp3Wle41HLRElPpyObC1rosxmSLktDNGsk3S3pPZKmpuGmh0uHHSSdJ+mJkj76h8Y+Ai4v6eOXS9b/X5L+r6TXgUPLhw0kHZOOzF5N6z4iTX+3pJ9JWiHpWUnfktQjzdtN0h/SUdsLkm6ocN96S1oD9AAeSI9H4x7guZIWAq9L6inpk+mxfTn19YiS9SyT9K+SFkp6PdW0Q+qz1yTdJmn75vo4rePrqc5lkk5sps0mRzDpsdyt5P58Lz2uK1UMi2xd1r7idkr7XGl4JPXB88B/S9o6tXlJ0mKK12VL92d8em68IunHgErmvVfSHel5/IKkayVtl+ZdDezM314TX0vTfyHp+bS+P0oa2dL2u0xEbBF/wDLgcOARYATFi+gZir2hAIaldrcBdwPHAzuXrWNYatuzym3+AbgM6APsAzQAh6V5JwF/bmHZ44BnKZ64AnZLtfYCHge+DmwFfAR4DdgjLTcDeBEYS3GEdS1wfXk/tHD7QuCadP2LwG+Av0/9tR+wbZp3F3Bqun5yqmlXYBvgl8DVZX32E4qjnb0pjmxGNHO/ZwAvpG31Ae6gOHL6fKrhW8CdZf20I8XOyGeA14HBzfVxWv8rwEFpmT5p2rfS/LFp/vg0fyfg/Wner4ErgHcBg4A5wBfTvOuAC0rWeXALj20Au5U9BguAoamP3pfux/j0eH8t9e9WJe3vBXZI9a0C/grsC/ROffbvzWx7HLAe+EFqe0jaVunz51st9N/G2oGpwC1AP6AvxXPlohq209j2O6nt1sDFwJ/SuocCi4DlzdynAcCrwITUX2en9TU+P3dLfdkbGAj8EZja3Gug5DndNy0zFVjQ1RlW6W9L2wOHv+2FjwcepgjJUsdRPHG+ASxNe2Ll7/4vpD2jxr8RZfORNJRinPvciFgbEQso9ro/V2WdpwKXRMT9UXg8Ip4CDqAIyYsj4u2IuAO4lWJoqNEvI2JOFIeh11K8edRiHdCf4gX7TkTMi4hXK7Q7EfhBRDwZEWuA84HjJZUO0U2JiDcj4gGKIaq9W9jur9K21gK/AtZGxFVRDHXcQBFUAETELyLiuYjYEBE3AI9RhHBLbo6Iu9Mya8vmnQJcGRGz0/xnI+JhFUceHwfOiuLobBXwQ4o3+sa+2gXYMT3erY69l5kWEc9ExJsUb0S/TTWsoxgz3xr4UEn7SyNiZUQ8S/F8vS+Ko8W3Up+1dnL0GxHxVkT8Afgt8I9tKVaSgNOAsyPixYh4DfgP/tYfbd3OBoo3nbdSH/wj8O207meAaS2U8wlgcUTMSv01FXi+cWZ67cxO626geFM5pKX7FxFXRsRrqT8vBPZWOqrsTrbUAP8nir2L8uETIuKliDgvIkZS7OEsAH6dnrCNBkTEdiV/SypsZ0eg8Ynd6CmKPaZqDAWeaGa9z0TEhhbW+3zJ9TcoAr8WVwO/B66X9JykSyT1aqamp8rq6UnRf7XUtLLk+psVbm9cVtLn05vsy2mYZE+KPbKWPNPCvOb6vfHoZ0XJtq6g2BOHYi9ZwJw09HFyKzW0VFOT/kyP9TM0fYyr7qMKXoqm516eSttsi4EUR2bzSvrjf9L0WrbTUPZmuiNN++QpmtekbRS70BtvSxok6fo07PUqcA0tPEck9ZB0cRo+e5ViD52WlukqW1yAp73YpRTv2r9spe0LFHs/O1IcyrXFc0A/SX1Lpu3Mpnv8zXkGeG8z6x0qqfSxa8t6y71O8UJstPGEbUSsi4gpEfEBir2/o2h6DqG0pl3K6llP01CpO0m7UAzNTAb6R8R2FIfajW+2zf3UZks/wdlcvz9DMfRT+ua9bXqjJyKej4jTImJHiqGnyxrHiqtUWlOT/kw7D0Op/TEut72kd5Xc3jlts1yT54ak0pP5L1C8UYws6Y93R0TpG0e124FNH5MVFPe5dNnmNGlb0l+NLkrrHxUR2wKfpWSMvMK2/wk4hmLI9d0Uw4CULdMtbHEBnpwCfCQqfAJE0nck7ZlOJPUFvgQ8HhGr27KBdNj3F+AiFScDR6XtXlvlKn4KnCNpPxV2S4F1H8UL62uSekkaBxwNXN+W+kosoBju6CVpNMU4IgCSDpW0l4oTda9SDBNU+sTGdcDZkoZL2obiUPqG6PhPEryL4sXXkOr9AsUeeKOVwBBJW7VhnT8DviDpMBUnZ3eS9P6IWAH8P+D7krZN894r6ZC07eMkDUnreCnVVeunW24Ejkw19AL+heLN4y81rq+SKZK2kvS/KN6Yf1GhzQPASEn7SOpDMZQAbDwq+AnwQ0mDAFJffayG7VRyI3C+pO1Tv57RQtvfpjo/lYbtvkrTT471BdYAL0vaCfjXsuVXUpy/KW3/FrCa4g3sP6qsudNtkQEeEU9ExNxmZv89xRjiy8CTFHtCnyxr87Kafg78n5tZ1wkU797PpXX+e0TMrrLGXwDfBn5OcZLy10C/iHg71fNxir2gy4DPR8TD1ay3gm9Q7HG+BExJ22v0HmAWRXgvoTgpe02FdVxJMdzyR4qjm7W0/IKri4hYDHwfuIfiRbgXxQnoRndQfFzveUkvVLnOOcAXKMa3X6G4z417w5+nOHG8mKK/ZgGD07wxwH0qPmVyC3BmRCyt8X49QrGXeCnFY3w0cHR67OvheYr6n6PYoTi90vMnIh6l+N7AbRTnFsrH9c+lOLl6bxpquA3Yo63bacYUimGTpRRvnFc31zAdKR9HceJzNbA7TZ8HU4APUjyev2XTI++LgH9LQ0HnUAytPkVxxLOY4oRxt6RiuMjMzHKzRe6Bm5ltDhzgZmaZcoCbmWXKAW5mlqlO/TGrAQMGxLBhwzpzk2Zm2Zs3b94LETGwfHqnBviwYcOYO7e5T++ZmVklkip+E9VDKGZmmXKAm5llygFuZpapLe0/8phZHa1bt47ly5ezdm35r/JaLfr06cOQIUPo1avSj35uygFuZjVbvnw5ffv2ZdiwYTT9xWVrq4hg9erVLF++nOHDh1e1jIdQzKxma9eupX///g7vOpBE//7923Q04wA3s3ZxeNdPW/vSAW5mlimPgZtZ3fxw9qN1Xd/Z49/Xaptly5Zx1FFHsWjRoqrWedJJJ3HUUUcxYcKE1ht3c9kEeL2fGI2qeYKYmXVHHkIxs+ytX7+eiRMnMmrUKCZMmMAbb7zBN7/5TcaMGcOee+7JpEmTqPTPa5prM27cOM4991zGjh3L+973Pv70pz8B8M4773DOOeew1157MWrUKC699FIA5s2bxyGHHMJ+++3Hxz72MVasWNEp99sBbmbZe+SRR5g0aRILFy5k22235bLLLmPy5Mncf//9LFq0iDfffJNbb711k+VaarN+/XrmzJnD1KlTmTJlCgDTp09n6dKlzJ8/n4ULF3LiiSeybt06zjjjDGbNmsW8efM4+eSTueCCCzrlfmczhGJm1pyhQ4dy0EEHAfDZz36WadOmMXz4cC655BLeeOMNXnzxRUaOHMnRRx/dZLk777yz2Taf+tSnANhvv/1YtmwZALfddhunn346PXsW0dmvXz8WLVrEokWLGD9+PFDspQ8ePJjO4AA3s+yVf/xOEl/+8peZO3cuQ4cO5cILL9zk89Vr165tsU3v3r0B6NGjB+vXrweKL9uUbysiGDlyJPfcc09H3LUWeQjFzLL39NNPbwzQ6667joMPPhiAAQMGsGbNGmbNmrXJMo1h3VKbch/96Ee5/PLLNwb6iy++yB577EFDQ8PG7a9bt46HHnqoLverNd4DN7O66apPdY0YMYKZM2fyxS9+kd13350vfelLvPTSS+y1114MGzaMMWPGbLLMdtttx2mnndZim3Knnnoqjz76KKNGjaJXr16cdtppTJ48mVmzZvHVr36VV155hfXr13PWWWcxcuTIjrirTajSmdmOMnr06Kj1Hzrc87Nz6lxN4cBTvtch6zXbEixZsoQRI0Z0dRmblUp9KmleRIwub+shFDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy5c+Bm1n93HlRfdd36PmtNvnQhz7EX/7ylxbbDBs2jLlz5zJgwICqNjtjxgzmzp3Lj3/846radxXvgZtZ1loL782ZA9zMsrbNNtsAcNdddzFu3DgmTJjA+9//fk488cQmPyH73e9+l7FjxzJ27Fgef/xxAH7zm9+w//77s++++3L44YezcuXKTdbfXJsLL7yQk08+mXHjxrHrrrsybdq0jctcddVVjBo1ir333pvPfe5zADQ0NPDpT3+aMWPGMGbMGO6+++5233cHuJltNubPn8/UqVNZvHgxTz75ZJOQ3HbbbZkzZw6TJ0/mrLPOAuDggw/m3nvvZf78+Rx//PFccsklm6yzpTYPP/wwv//975kzZw5TpkzZ+Dso3/72t7njjjt44IEH+NGPfgTAmWeeydlnn83999/PTTfdxKmnntru++sxcDPbbIwdO5YhQ4YAsM8++7Bs2bKNP2x1wgknbLw8++yzAVi+fDmf+cxnWLFiBW+//TbDhw/fZJ0ttTnyyCPp3bs3vXv3ZtCgQaxcuZI77riDCRMmbBxv79evH1D8FO3ixYs3Lvvqq6/y2muv0bdv35rvb1V74JKWSXpQ0gJJc9O0fpJmS3osXW5fcxVmZnXQ+BOw0PRnYKHpT842Xj/jjDOYPHkyDz74IFdcccUmPznbWptK26v0k7MAGzZs4J577mHBggUsWLCAZ599tl3hDW0bQjk0IvYp+UGV84DbI2J34PZ028ysW7rhhhs2Xh544IEAvPLKK+y0004AzJw5s+Jy1bQpddhhh3HjjTeyevVqoPjJWSh+irb0Uy0LFiyo7Y6UaM8QyjHAuHR9JnAXcG476zGznFXxsb+u8tZbb7H//vuzYcMGrrvuOqA4EXncccex0047ccABB7B06dJNlqumTamRI0dywQUXcMghh9CjRw/23XdfZsyYwbRp0/jKV77CqFGjWL9+PR/+8Ie5/PLL23Wfqvo5WUlLgZeAAK6IiOmSXo6I7UravBQRmwyjSJoETALYeeed93vqqadqKtQ/J2vW/fjnZOuvLT8nW+0e+EER8ZykQcBsSQ9XW0xETAemQ/F74NUuZ2ZmLatqDDwinkuXq4BfAWOBlZIGA6TLVR1VpJmZbarVAJf0Lkl9G68DHwUWAbcAE1OzicDNHVWkmXVfnflfvTZ3be3LaoZQdgB+lT4W0xP4eUT8j6T7gRslnQI8DRzXxlrNLHN9+vRh9erV9O/fv+JH56x6EcHq1avp06dP1cu0GuAR8SSwd4Xpq4HD2lShmW1WhgwZwvLly2loaOjqUjYLffr02fhFpGr4m5hmVrNevXpV/PaidQ7/FoqZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llquoAl9RD0nxJt6bb/STNlvRYuty+48o0M7NybdkDPxNYUnL7POD2iNgduD3dNjOzTlJVgEsaAhwJ/LRk8jHAzHR9JnBsXSszM7MWVbsHPhX4GrChZNoOEbECIF0OqrSgpEmS5kqa29DQ0J5azcysRKsBLukoYFVEzKtlAxExPSJGR8TogQMH1rIKMzOroGcVbQ4CPinpE0AfYFtJ1wArJQ2OiBWSBgOrOrJQMzNrqtU98Ig4PyKGRMQw4Hjgjoj4LHALMDE1mwjc3GFVmpnZJtrzOfCLgfGSHgPGp9tmZtZJqhlC2Sgi7gLuStdXA4fVvyQzM6uGv4lpZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmXKAm5llygFuZpYpB7iZWaYc4GZmmerZ1QV0uTsv6pj1Hnp+x6zXzCzxHriZWaYc4GZmmXKAm5llygFuZpapVgNcUh9JcyQ9IOkhSVPS9H6SZkt6LF1u3/HlmplZo2r2wN8CPhIRewP7AEdIOgA4D7g9InYHbk+3zcysk7Qa4FFYk272Sn8BHAPMTNNnAsd2RIFmZlZZVWPgknpIWgCsAmZHxH3ADhGxAiBdDmpm2UmS5kqa29DQUKeyzcysqgCPiHciYh9gCDBW0p7VbiAipkfE6IgYPXDgwBrLNDOzcm36FEpEvAzcBRwBrJQ0GCBdrqp3cWZm1rxqPoUyUNJ26frWwOHAw8AtwMTUbCJwcwfVaGZmFVTzWyiDgZmSelAE/o0Rcauke4AbJZ0CPA0c14F1mplZmVYDPCIWAvtWmL4aOKwjijIzs9b5m5hmZplygJuZZcoBbmaWKQe4mVmmHOBmZplygJuZZcoBbmaWKQe4mVmmHOBmZpmq5qv0m7V7nlzdIes98NAOWa2Z2UbeAzczy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsU60GuKShku6UtETSQ5LOTNP7SZot6bF0uX3Hl2tmZo2q2QNfD/xLRIwADgC+IukDwHnA7RGxO3B7um1mZp2k1QCPiBUR8dd0/TVgCbATcAwwMzWbCRzbQTWamVkFbRoDlzQM2Be4D9ghIlZAEfLAoGaWmSRprqS5DQ0N7SzXzMwaVR3gkrYBbgLOiohXq10uIqZHxOiIGD1w4MBaajQzswqqCnBJvSjC+9qI+GWavFLS4DR/MLCqY0o0M7NKqvkUioCfAUsi4gcls24BJqbrE4Gb61+emZk1p2cVbQ4CPgc8KGlBmvZ14GLgRkmnAE8Dx3VIhWZmVlGrAR4RfwbUzOzD6luOmZlVy9/ENDPLlAPczCxTDnAzs0w5wM3MMuUANzPLlAPczCxTDnAzs0w5wM3MMuUANzPLlAPczCxTDnAzs0w5wM3MMuUANzPLlAPczCxTDnAzs0w5wM3MMuUANzPLlAPczCxTDnAzs0w5wM3MMuUANzPLlAPczCxTDnAzs0w5wM3MMuUANzPLlAPczCxTDnAzs0w5wM3MMtVqgEu6UtIqSYtKpvWTNFvSY+ly+44t08zMylWzBz4DOKJs2nnA7RGxO3B7um1mZp2o1QCPiD8CL5ZNPgaYma7PBI6tb1lmZtaaWsfAd4iIFQDpclBzDSVNkjRX0tyGhoYaN2dmZuU6/CRmREyPiNERMXrgwIEdvTkzsy1GrQG+UtJggHS5qn4lmZlZNWoN8FuAien6RODm+pRjZmbVquZjhNcB9wB7SFou6RTgYmC8pMeA8em2mZl1op6tNYiIE5qZdVidazEzszbwNzHNzDLlADczy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsUw5wM7NMOcDNzDLlADczy5QD3MwsUw5wM7NMtfpPja1Gd15U/3Ueen7912lm2fIeuJlZphzgZmaZcoCbmWXKAW5mlikHuJlZphzgZmaZ8scIc9IRH00EfzwxQ/f87JwOWe+Bp3yvQ9ZrHaNde+CSjpD0iKTHJZ1Xr6LMzKx1NQe4pB7AfwIfBz4AnCDpA/UqzMzMWtaePfCxwOMR8WREvA1cDxxTn7LMzKw17RkD3wl4puT2cmD/8kaSJgGT0s01kh5pxzbrbQDwQlcX0YxOrO3rbV3A/Vab7l/bqd/v6joq6f791vF2qTSxPQGuCtNikwkR04Hp7dhOh5E0NyJGd3Udlbi22ri22ri22nR1be0ZQlkODC25PQR4rn3lmJlZtdoT4PcDu0saLmkr4HjglvqUZWZmral5CCUi1kuaDPwe6AFcGREP1a2yztEth3YS11Yb11Yb11abLq1NEZsMW5uZWQb8VXozs0w5wM3MMrXZBHhrX+tXYVqav1DSB0vmXSlplaRFZcv0kzRb0mPpcvtuVNuFkp6VtCD9faIza5M0VNKdkpZIekjSmSXLdGm/tVJbV/dbH0lzJD2QaptSskxd+q0D6+vSviuZ30PSfEm3lkzr8tdqC7XVpd8qiojs/yhOoj4B7ApsBTwAfKCszSeA31F8fv0A4L6SeR8GPggsKlvmEuC8dP084DvdqLYLgXO6qt+AwcAH0/W+wKONy3Z1v7VSW1f3m4Bt0vVewH3AAfXqtw6ur0v7rmT+PwM/B27tTq/VFmprd78197e57IFX87X+Y4CronAvsJ2kwQAR8UfgxQrrPQaYma7PBI7tRrXVQ821RcSKiPhrqvE1YAnFt3Mbl+myfmultnpoT20REWtSm17pL0qWaW+/dWR99dCu14OkIcCRwE8rLNOlr9UWauswm0uAV/paf/kLtpo25XaIiBUA6XJQN6oNYHI6jLuyxkPGutQmaRiwL8XeGnSjfqtQG3Rxv6XD7AXAKmB2RNSz3zqyPuj659xU4GvAhrJlusNzrrnaoP39VtHmEuDVfK2/qq/+d4COqu2/gPcC+wArgFp+xKLdtUnaBrgJOCsiXq2hhs6urcv7LSLeiYh9KL69PFbSnjXU0JKOqq9L+07SUcCqiJhXw3ar0VG11aPfKtpcAryar/XX8tX/lSWHR4Mp9ki6RW0RsTK90DYAP6E4/OvU2iT1ogjIayPilyVturzfmqutO/RbSS0vA3cBR6RJ9ei3DquvG/TdQcAnJS2jGN74iKRrUpuufs41W1ud+q2yeg+qd8UfxTdKnwSG87eTDyPL2hxJ05MPc8rmD2PTE4XfpemJkUu6UW2DS66fDVzfmbWl21cBUyust0v7rZXaurrfBgLbpetbA38CjqpXv3VwfV3ad2VtxtH0RGG3eK02U1u7+63Zmuu1oq7+ozg7/CjFWeQL0rTTgdPTdVH8A4ongAeB0SXLXkdxaLOO4h32lDS9P3A78Fi67NeNars6tV1I8Rs0gzuzNuBgisPLhcCC9PeJ7tBvrdTW1f02Cpiftr8I+N8l66xLv3VgfV3ad2XrGEfTkOzy12oLtdWl3yr9+av0ZmaZ2lzGwM3MtjgOcDOzTDnAzcwy5QA3M8uUA9zMLFMOcDOzTDnAzcwy9f8Bvc8JbIrRtfYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xmin = np.min(np.append(sym_losses, asym_losses))\n",
    "xmax = np.percentile(np.append(sym_losses, asym_losses), 95)\n",
    "bins = np.linspace(xmin, xmax, 15)\n",
    "plt.hist(balance_losses, bins, alpha=0.5, label='balance')\n",
    "plt.hist(inbalance_losses, bins, alpha=0.5, label='inbalance')\n",
    "plt.title(\"MSE of confusion matrices from rte data\")\n",
    "plt.legend(loc='upper right')\n",
    "# plt.savefig(\"pic/rte_singular.png\",dpi=300)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.24380891340979552"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(balance_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAi+0lEQVR4nO3df5xcdX3v8dc7yyaB8ENiAgoBkwqtS1axsOVW2KprpSVyLfRehSzeRs1KSh+y3pZrgXZqC7cutjysogGk6Kb80GysVLhQiBHvrnrXiGRzRQxZ0QjhZoFIAuFXarKb8Ll/nLPL7DC7ObubndmcvJ+Pxzxm5pzv95zPnDnzme/5nl+KCMzMLL+mVTsAMzObXE70ZmY550RvZpZzTvRmZjnnRG9mlnNO9GZmOedEP4Up8S+Sdkh6sILzPUvSLyS9LOn8CUxntaQP78fQqkbSTZI+VcH5/bGkLel38NuVmm8673dL6htl/C2SPl2B+Yxpme9remXKf1fSx7KWP5A50ReRtFlSv6Q5JcMfkhSS5qfv50n6N0nbJb0g6aeSPpKOm5+WfbnkceE4QmoEzgbmRcQZE/x4Y/E/gesj4vCIuGu8E4mIRRFx6/4La/+T9BFJ3fsqFxGXRMTfVyKm1GeBS9Pv4McVnO+UUYVlPqI0N7y32nGM1yHVDmAKehxoBpYDSHorcGhJmduBnwBvAnYDbwXeUFLmdRGxZ4KxvAnYHBE7Jzid8cz3kQrPc8qSVBMReys82xG/A0mH7Id164DmZTA2btG/1u3AkqL3HwZuKynzO8AtEbEzIvZExI8jYvV4ZibpOEl3S3pO0iZJF6fDW4CvAO9ItwiuHqH+xZJ6Jb0kaaOk09Lhdemm6fOSHpH0R0V1bpF0g6R703o/kvTmdNwvgd8A7knnO6O0NSPpKklfTV/PlPRVSc+m81on6dh03NCmsaRpkv5G0hOSnpF0m6Sj0nGDW0EflvT/0i2lwijL7BZJN6ZdQy9L+oGkN0i6Lu3m+llxd4ekKyX9smgZ/fHgMgJuKlrGzxdN/0uS7pO0E2gq7a6QdF66pfdiOu1z0uFHSWqX9LSkJyV9WlJNOu4kSd9LtwK3S/p6mc82Q9LLQA3wk/T7GGxRXiHpYWCnpEMk/VH63T6fLuu6oulslvSXkh6WtDON6dh0mb0k6TuSjh5pGafT+Os0zs2SPjRCmddsEaXf5UlFn+ez6ff6KyXdMYeWlC87n+JlrrRbJl0GW4F/kXRoWmaHpI0kv8vRPs/Z6brxgqTrARWNe7OkznQ93i7pa5Jel467HTiRV38Tl6fDvyFpazq970taONr8qyoi/EgfwGbgvcCjQB3Jj20LSesqgPlpue8APwAWAyeWTGN+WvaQjPP8HnAjMBN4O7AN+P103EeA7lHqfhB4kmQFF3BSGmstsAn4a2A68B7gJeC30nq3AM8BZ5Bs1X0NWFW6HEZ5fxXw1fT1nwL3AIely+t04Mh03HeBj6Wvl6Yx/QZwOPBN4PaSZfZlkq2nU0m2lOpG+Ny3ANvTec0EOkm2xJakMXwa6CpZTseRNGwuBHYCbxxpGafTfwE4K60zMx326XT8Gen4s9PxxwNvScfdBfwzMAs4BngQ+NN0XAdQKJpm4yjfbQAnlXwHDwEnpMvoN9PPcXb6fV+eLt/pReUfAI5N43sG+L/AbwMz0mX2dyPM+93AHuBzadl3pfMqXn8+PcryG4oduA64G5gNHEGyrnxmHPMZLPuPadlDgX8A/k867ROADUDfCJ9pDvAi8IF0ef1FOr3B9fOkdFnOAOYC3weuG+k3ULROH5HWuQ54qNo5bKSHW/TlDbbqzwZ+RpJMi32QZAX7FPB42rIrbU1sT1tag4+6kvFIOoGkH/6KiNgVEQ+RtOL/JGOcHwOujYh1kdgUEU8Av0uSTP8hIvojohP4d5IuqUHfjIgHI9n8/RrJn8x4DACvJ/lh742I9RHxYplyHwI+FxGPRcTLwF8BiyUVdx9eHRG/joifkHSNnTrKfO9M57ULuBPYFRG3RdLF8nWShAZARHwjIp6KiFci4uvAL0iS9Wj+V0T8IK2zq2RcC7AiIu5Pxz8ZET9TsiWzCPjzSLb2ngE+T9IgGFxWbwKOS7/vfe4bKPHFiNgSEb8m+cO6N41hgKRP/1DgzKLyyyPiVxHxJMn6+qNItj53p8tsXzt5PxURuyPie8C9wAVjCVaSgIuBv4iI5yLiJeAaXl0eY53PKyR/TrvTZXAB0JZOewvwxVHCeR+wMSLuSJfXdcDWwZHpb+f+dNrbSP583jXa54uIFRHxUro8rwJOVbqVOtU40Zd3O3ARSWultNuGiNgREVdGxEKSFtNDwF3pij1oTkS8rujRW2Y+xwGDP4BBT5C0wLI4AfjlCNPdEhGvjDLdrUWv/4Pkj2E8bgfWAKskPSXpWkm1I8T0REk8h5Asv/HE9Kui178u836orqQl6Z/x82n3TD1JC280W0YZN9JyH9yaerpoXv9M0rKHpNUt4MG0y2XpPmIYLaZhyzP9rrcw/DvOvIzK2BHD9w09kc5zLOaSbOmtL1oe30qHj2c+20r+dI9j+DJ5gpENKxtJk3zovaRjJK1Ku9teBL7KKOuIpBpJ/5B2271I0uJntDrV5ERfRtoqfpykFfDNfZTdTtKaOo5kE3IsngJmSzqiaNiJvHYLYiRbgDePMN0TJBV/v2OZbqmdJD/YQUM7niNiICKujohTSFqT/5nh+ziKY3pTSTx7GJ589jtJbyLpEroUeH1EvI5kE3/wT3mky7eOdlnXkZb7FpIup+I/+SPTBgERsTUiLo6I40i6vG4c7MvOqDimYcszbWScwPi/41JHS5pV9P7EdJ6lhq0bkooPSthO8oeysGh5HBURxX8wWecDr/1Onib5zMV1RzKsbNHyGvSZdPpvi4gjgf9GUR9+mXlfBJxH0tV7FEn3IyV1pgwn+pG1AO+JMke8SPpHSfXpDrEjgD8DNkXEs2OZQbq5uRb4jJKdmm9L5/u1jJP4CvBJSacrcVKa2H5E8gO8XFKtpHcD7wdWjSW+Ig+RdLPUSmog6ecEQFKTpLcq2eH4Ikn3RLkjVDqAv5C0QNLhJJvwX4/JP3JiFsmPdFsa70dJWvSDfgXMkzR9DNNsBz4q6feV7GQ+XtJbIuJp4NvAP0k6Mh33ZknvSuf9QUnz0mnsSOMa79E8/wqcm8ZQC/wPkj+ZteOcXjlXS5ou6fdI/sC/UabMT4CFkt4uaSZJFwYwtJXxZeDzko4BSJfVH45jPuX8K/BXko5Ol2vrKGXvTeP8L2l34ScYfqTcEcDLwPOSjgf+sqT+r0j2LxWX3w08S/JHd03GmKvCiX4EEfHLiOgZYfRhJH2czwOPkbSs/qikzPMafhz9ZSNMq5mkNfBUOs2/i4j7M8b4DaANWEmys/UuYHZE9KfxLCJpVd0ILImIn2WZbhmfImnB7gCuTuc36A3AHSRJvpdk5/JXy0xjBUk3z/dJtpZ2MfoPc7+IiI3APwE/JPmxvpVkR/qgTpLDGLdK2p5xmg8CHyXpf3+B5DMPtq6XkOwA30iyvO4A3piO+x3gR0qOqrkb+O8R8fg4P9ejJK3O5STf8fuB96ff/f6wlST+p0gaHpeUW38i4uck5118h2TfR+l+hytIdhI/kHZxfAf4rbHOZwRXk3TXPE7yB3v7SAXTLe8PkuzAfRY4meHrwdXAaSTf5728dkv+M8DfpF1QnyTp0n2CZAtqI8mO7ylLSVeVmZnllVv0ZmY550RvZpZzTvRmZjnnRG9mlnNT8qJmc+bMifnz51c7DDOzA8b69eu3R8TccuOmZKKfP38+PT0jHdloZmalJI14ZrC7bszMcs6J3sws55zozcxyzonezCznnOjNzHLOid7MKq6jo4P6+npqamqor6+no6Oj2iHl2pQ8vNLM8qujo4NCoUB7ezuNjY10d3fT0tICQHNz8z5q23hkatFLOkfSo0puXn1lmfHnKbkJ8UOSeiQ1Zq1rZgeXtrY22tvbaWpqora2lqamJtrb22lra6t2aLm1z8sUpzeU+DnJ/VP7gHVAc3qd78EyhwM7IyLSm2f8a0S8JUvdchoaGsInTJnlU01NDbt27aK29tU7Tg4MDDBz5kz27h3vfVhM0vqIaCg3LkuL/gySuyc9lt7UYBXJLbSGRMTL8eo/xuAdfTLVNbODS11dHd3dw+9P0t3dTV1dXZUiyr8sif54ht+At48yN6+W9MeSfkZyd5alY6mb1l+Wdvv0bNu2LUvsZnYAKhQKtLS00NXVxcDAAF1dXbS0tFAoFKodWm5l2Rlb7ma3r+nviYg7gTslvRP4e5Kb5maqm9a/GbgZkq6bDHGZ2QFocIdra2srvb291NXV0dbW5h2xkyhLou9j+N3S5zHyXdqJiO+nN0SeM9a6ZnZwaG5udmKvoCxdN+uAkyUtkDQdWExyY+Mhkk6SpPT1aSQ3R342S10zM5tc+2zRR8QeSZcCa4AaYEVEPCLpknT8TcB/BZZIGgB+DVyY7pwtW3eSPouZmZWxz8Mrq8GHV5qZjc1ED680M7MDmBO9mVnOOdGbmeWcE72ZWc450ZuZ5ZwTvZlZzjnRm5nlnBO9mVnOOdGbmeWcE72ZWc450ZuZ5ZwTvZlZzjnRm5nlnBO9mVnOOdGbmeWcE72ZWc450ZuZ5ZwTvZlZzjnRm5nlnBO9mVnOOdGbmeWcE72ZWc450ZuZ5ZwTvZlZzmVK9JLOkfSopE2Sriwz/kOSHk4fayWdWjRus6SfSnpIUs/+DN7MzPbtkH0VkFQD3ACcDfQB6yTdHREbi4o9DrwrInZIWgTcDPynovFNEbF9P8ZtZmYZZWnRnwFsiojHIqIfWAWcV1wgItZGxI707QPAvP0bppmZjVeWRH88sKXofV86bCQtwOqi9wF8W9J6SctGqiRpmaQeST3btm3LEJaZmWWRJdGrzLAoW1BqIkn0VxQNPisiTgMWAR+X9M5ydSPi5ohoiIiGuXPnZgjLRtPR0UF9fT01NTXU19fT0dFR7ZDMrEr22UdP0oI/oej9POCp0kKS3gZ8BVgUEc8ODo+Ip9LnZyTdSdIV9P2JBG2j6+jooFAo0N7eTmNjI93d3bS0tADQ3Nxc5ejMrNKytOjXASdLWiBpOrAYuLu4gKQTgW8CfxIRPy8aPkvSEYOvgT8ANuyv4K28trY22tvbaWpqora2lqamJtrb22lra6t2aGZWBfts0UfEHkmXAmuAGmBFRDwi6ZJ0/E3A3wKvB26UBLAnIhqAY4E702GHACsj4luT8klsSG9vL42NjcOGNTY20tvbW6WIzKyasnTdEBH3AfeVDLup6PXHgI+VqfcYcGrpcJtcdXV1dHd309TUNDSsu7uburq6KkZlZtXiM2NzqFAo0NLSQldXFwMDA3R1ddHS0kKhUKh2aGZWBZla9HZgGdzh2traSm9vL3V1dbS1tXlHrNlBShFlj5SsqoaGhujp8dUSzMyykrQ+3Tf6Gu66MTPLOSd6M7Occ6I3M8s5J3ozs5xzojczyzknejOznHOiNzPLOSd6M7Occ6I3M8s5J3ozs5xzojczyzknejOznHOiNzPLOSd6M7Occ6I3M8s5J3ozs5xzojczyzknejOznHOiNzPLOSd6M7Occ6I3M8s5J3ozs5zLlOglnSPpUUmbJF1ZZvyHJD2cPtZKOjVrXTMzm1z7TPSSaoAbgEXAKUCzpFNKij0OvCsi3gb8PXDzGOqamdkkytKiPwPYFBGPRUQ/sAo4r7hARKyNiB3p2weAeVnrmpnZ5MqS6I8HthS970uHjaQFWD3WupKWSeqR1LNt27YMYZmZWRZZEr3KDIuyBaUmkkR/xVjrRsTNEdEQEQ1z587NEJaZmWVxSIYyfcAJRe/nAU+VFpL0NuArwKKIeHYsdc3MbPJkadGvA06WtEDSdGAxcHdxAUknAt8E/iQifj6WumZmNrn22aKPiD2SLgXWADXAioh4RNIl6fibgL8FXg/cKAlgT9oNU7buJH0WMzMrQxFlu8yrqqGhIXp6eqodhpnZAUPS+ohoKDfOZ8aameWcE72ZVVxHRwf19fXU1NRQX19PR0dHtUPKtSxH3ZiZ7TcdHR0UCgXa29tpbGyku7ublpYWAJqbm6scXT65j97MKqq+vp7ly5fT1NQ0NKyrq4vW1lY2bNhQxcgObKP10TvRm1lF1dTUsGvXLmpra4eGDQwMMHPmTPbu3VvFyA5s3hlrZlNGXV0d3d3dw4Z1d3dTV1dXpYjyz4nezCqqUCjQ0tJCV1cXAwMDdHV10dLSQqFQqHZoueWdsWZWUc3Nzaxdu5ZFixaxe/duZsyYwcUXX+wdsZPILXozq6iOjg7uvfdeVq9eTX9/P6tXr+bee+/1IZaTyDtjzayifNTN5PBRN2Y2Zfiom8nho27MbMrwUTeV50RvZhXlo24qz0fdmFlFDR5d09raSm9vL3V1dbS1tfmom0nkPnozsxxwH72Z2UHMid7MLOec6M3Mcs6J3sws55zozcxyzonezCznnOjNzHLOid7MLOec6M3Mci5Topd0jqRHJW2SdGWZ8W+R9ENJuyV9smTcZkk/lfSQJJ/uamZWYfu81o2kGuAG4GygD1gn6e6I2FhU7DngE8D5I0ymKSK2TzBWMzMbhywt+jOATRHxWET0A6uA84oLRMQzEbEOGJiEGM3MbAKyJPrjgS1F7/vSYVkF8G1J6yUtG6mQpGWSeiT1bNu2bQyTNzOz0WRJ9CozbCyXvDwrIk4DFgEfl/TOcoUi4uaIaIiIhrlz545h8mZmNposib4POKHo/TzgqawziIin0udngDtJuoLMzKxCsiT6dcDJkhZImg4sBu7OMnFJsyQdMfga+APAd/81M6ugfR51ExF7JF0KrAFqgBUR8YikS9LxN0l6A9ADHAm8IunPgVOAOcCdkgbntTIivjUpn8TMzMrKdCvBiLgPuK9k2E1Fr7eSdOmUehE4dSIBmpnZxPjMWDOznHOiNzPLOSd6M7Occ6I3M8s5J3ozs5xzojczyzknejOznHOiNzPLOSd6M7Occ6I3M8s5J3ozs5xzojczyzknejOznHOiNzPLOSd6M7Occ6I3M8s5J/qc6ujooL6+npqaGurr6+no6Kh2SGZWJZnuMGUHlo6ODgqFAu3t7TQ2NtLd3U1LSwsAzc3NVY7OzCpNEVHtGF6joaEhenp6qh3GAau+vp7ly5fT1NQ0NKyrq4vW1lY2bPC92c3ySNL6iGgoN85dNznU29tLX1/fsK6bvr4+ent7qx2amVWBE30OHXfccVxxxRUsX76cXbt2sXz5cq644gqOO+64aodmBngfUqW5jz6nSrvkpmIXnR2cvA+pCiJiyj1OP/30sPGbNm1a3HbbbbFw4cKYNm1aLFy4MG677baYNm1atUMzi4ULF0ZnZ+ewYZ2dnbFw4cIqRZQPQE+MkFPdos+huro65s2bN2zHa1dXF3V1dVWMyizR29tLY2PjsGGNjY3ehzSJ3EefQ4VCgZaWFrq6uhgYGKCrq4uWlhYKhUK1QzOjrq6O7u7uYcO6u7vdEJlEmRK9pHMkPSppk6Qry4x/i6QfStot6ZNjqWv7X3NzM+eeey6LFi1i+vTpLFq0iHPPPdf9nzYluCFSBSP16Qw+gBrgl8BvANOBnwCnlJQ5BvgdoA345Fjqlnu4j35iVq5cGQsWLIjOzs7o7++Pzs7OWLBgQaxcubLaoZlFRLKOFu9D8ro5cYzSR7/PE6YkvQO4KiL+MH3/V+kfxGfKlL0KeDkiPjvWusV8wtTE+IQps4PPRE+YOh7YUvS+Lx2WRea6kpZJ6pHUs23btoyTt3K8s8vMimVJ9CozLOtB2ZnrRsTNEdEQEQ1z587NOHkrxzu7zKxYlkTfB5xQ9H4e8FTG6U+kro2Td3aZWbEsx9GvA06WtAB4ElgMXJRx+hOpa+M0eHRNa2srvb291NXV0dbW5qNuzA5Sma5eKel9wHUkR9GsiIg2SZcARMRNkt4A9ABHAq8AL5McXfNiubr7mp93xpqZjc1oO2N9mWIzsxzwZYrNzA5iTvRmVnG+THFl+aJmZlZRvkxx5bmP3swqymduTw730R+EvGlsU5XP3K48J/ocGtw0Lr6VYKFQcLK3KcFnblfBSFc7q+bDV6+cGN/Bx6YyX111cjCRq1dWg/voJ6ampoZdu3ZRW1s7NGxgYICZM2eyd+/eKkZmlujo6KCtrW3ozO1CoeAdsRM0Wh+9j7rJocFN4+KdXd40tqmkubnZib2C3EefQ4VCgQsvvJAFCxZQU1PDggULuPDCC31RM5syfLBAZblFn3NTsWvODm4+jr7y3EefQ/X19Zx//vncddddQ32gg+99nLJVm9fPyeGLmh1kpk2bxqxZs9i9ezcDAwPU1tYyY8YMdu7cySuvvFLt8OwgN23aNObPn/+aFv3mzZu9fk6AT5g6yEybNo2dO3cye/ZsAGbPns3OnTuZNs1ft1Xf9OnTOfPMM2ltbWXmzJm0trZy5plnMn369GqHllv+5edQ8SGUksoON6uW/v5+Vq1axdKlS3nppZdYunQpq1ator+/v9qh5ZYTfU4ddthhHHrooUji0EMP5bDDDqt2SGZA0qJfvHgxK1as4IgjjmDFihUsXrzYLfpJ5ESfU3v27Bn1vVm19Pf3s3bt2mGX6Fi7dq1b9JPIh1fm1O7du9m8eTPA0LPZVHDKKadw/vnnD7un8UUXXcRdd91V7dByyy36HCrul88y3KySCoUCK1euHNaiX7lypU/om0Ru0edQRCCJY445hmeeeWboeSoeSmsHn8GToopb9G1tbT5ZahK5RZ9TF1xwAXPmzEESc+bM4YILLqh2SGZDmpub2bBhA3v37mXDhg1O8pPMLfqcuv/++7njjjuGTkj5wAc+UO2QzKxKnOhzaPbs2ezYsYOLLrpoqOtmx44dQydQmdnBxV03OXT99dczY8YMtm7dyiuvvMLWrVuZMWMG119/fbVDM7MqyJToJZ0j6VFJmyRdWWa8JH0xHf+wpNOKxm2W9FNJD0nyBWwqpLa2dujGI8Wvzezgs89EL6kGuAFYBJwCNEs6paTYIuDk9LEM+FLJ+KaIePtIF9yx/evyyy/n8MMPZ82aNfT397NmzRoOP/xwLr/88mqHZmZVkKVFfwawKSIei4h+YBVwXkmZ84Db0lsXPgC8TtIb93OsllFfXx+33norTU1N1NbW0tTUxK233kpfX1+1QzOzKsiS6I8HthS970uHZS0TwLclrZe0bKSZSFomqUdSz7Zt2zKEZWYHKt9hqrKyJPpyp1OWnnkzWpmzIuI0ku6dj0t6Z7mZRMTNEdEQEQ1z587NEJaNZN68eSxZsoSuri4GBgbo6upiyZIlzJs3r9qhmQ3dYar4zNhCoeBkP4myJPo+4ISi9/OAp7KWiYjB52eAO0m6gmwSXXvttezdu5elS5cyY8YMli5dyt69e7n22murHZoZbW1ttLe3D+tabG9vp62trdqh5VaWRL8OOFnSAknTgcXA3SVl7gaWpEff/C7wQkQ8LWmWpCMAJM0C/gDwvcImWXNzM1/4wheYNWsWkpg1axZf+MIXfPahTQm9vb00NjYOG9bY2Ehvb2+VIsq/fZ4wFRF7JF0KrAFqgBUR8YikS9LxNwH3Ae8DNgH/AXw0rX4scGd6Ma1DgJUR8a39/insNZqbm53YbUqqq6uju7ubpqamoWHd3d3U1dVVMap8y3RmbETcR5LMi4fdVPQ6gI+XqfcYcOoEYzSzHCkUCrS0tLzmnrHuupk8vgSCmVWUr15ZeZqKl65taGiInh6fRGtmlpWk9SOdlOpr3eSUj1M2s0FO9Dnk45RtqnNDpMIiYso9Tj/99LDxW7hwYXR2dg4b1tnZGQsXLqxSRGavWrlyZSxYsCA6Ozujv78/Ojs7Y8GCBbFy5cpqh3ZAA3pihJzqPvocqqmpYdeuXcOuWDkwMMDMmTPZu3dvFSMzg/r6epYvXz7s8Mquri5aW1vZsMGn2YyX++gPMoPHKRfzcco2VfiEqcpzos+hweOUi69109LSQqFQqHZoZm6IVIGPo88hH6dsU5lPmKo899GbWcV1dHTQ1tY21BApFApuiEzQaH30TvRmZjngnbFmZgcxJ3ozs5xzojezivOZsZXlo27MrKIGL9FRetQN4B2yk8Q7Y82sourr6zn55JNZvXo1u3fvZsaMGSxatIhf/OIXPjN2Arwz1symjI0bN3LPPfdwzTXXsHPnTq655hruueceNm7cWO3QcsuJ3swqbtmyZVx22WUcdthhXHbZZSxbtqzaIeWaE72ZVVREsHr16mGX6Fi9ejVTsRs5L7wz1swmlaTXDNu8eTPvec97Ri3rxL//uEWfE5LG9TDbX2bPnr1f17Fy05o9e/Z+jPjg4Rb9geaqo8oOjr87cr9Oj6teGN/07KD13Cf2AuNcDzPz/RTGw4dXHmAq0Qo/+uijee655yZ9PpYvXjera7TDK92iP8CM9Mc83h/ZVPyjtwOT182py4k+J/yjsKnK62b1ZdoZK+kcSY9K2iTpyjLjJemL6fiHJZ2Wta6ZmU2ufSZ6STXADcAi4BSgWdIpJcUWASenj2XAl8ZQ18zMJlGWFv0ZwKaIeCwi+oFVwHklZc4DbovEA8DrJL0xY10zM5tEWRL98cCWovd96bAsZbLUBUDSMkk9knq2bduWISwzM8siS6Ivt8u8dO/KSGWy1E0GRtwcEQ0R0TB37twMYZmZWRZZjrrpA04oej8PeCpjmekZ6pqZ2STK0qJfB5wsaYGk6cBi4O6SMncDS9Kjb34XeCEins5Y18zMJtE+W/QRsUfSpcAaoAZYERGPSLokHX8TcB/wPmAT8B/AR0erOymfxMzMypqSl0CQtA14otpx5MQcYHu1gzAbgdfP/edNEVF2B+eUTPS2/0jqGen6F2bV5vWzMnyZYjOznHOiNzPLOSf6/Lu52gGYjcLrZwW4j97MLOfcojczyzknejOznHOin4IkzZe0YQzlb5H0gcmMyQ5OktZmKLNZ0pwxTPMjkq6fWGQ2Fk70ZjaiiDiz2jHYxDnRT12HSLo1vWPXHZIOk/S3ktZJ2iDpZpW5GedIZSR9V9I/SnpQ0s8l/V46vEbSZyX9NJ1Xazr8dEnfk7Re0pr0/gJ2kJH0cvr87nQdukPSzyR9rWT9+8t03XpQ0klpnfdL+pGkH0v6jqRjy0y/bBlJV0lakc7zMUmfKKqzJF1XfyLp9nTYXEn/lq776ySdNakL5kATEX5MsQcwn+Ryzmel71cAnwRmF5W5HXh/+voW4APp65HKfBf4p/T1+4DvpK//DPg34JDB+kAtsBaYmw67kOQ6RVVfNn5UfF18OX1+N/ACyRVopwE/BBrTcZuBQvp6CfDv6eujefXIvo8VrX8fAa7fR5mr0nVwBsllEp5N18uFwKPAnLTc7PR5ZVE8JwK91V52U+nhm4NPXVsi4gfp668CnwAel3Q5cBhJQn4EuKekXtMoZb6ZPq8n+TMBeC9wU0TsAYiI5yTVA/XA/WmjrQZ4er9+OjsQPRgRfQCSHiJZh7rTcR1Fz59PX88Dvp5uDU4HHi8zzdHK3BsRu4Hdkp4BjgXeA9wREdshWV/Tsu8FTinayDhS0hER8dL4P25+ONFPXaUnOARwI9AQEVskXQXMLC4gaeY+yuxOn/fy6nevMvMS8EhEvGOiH8JyZXfR6+J1CIavQ4OvlwOfi4i7Jb2bpJVearQy5eZXbn2FZCvjHRHx6318hoOS++inrhMlDSbaZl5tOW2XdDhQ7iibmRnKlPo2cImkQwAkzSbZNJ47OH9JtZIWjvNz2MHhwqLnH6avjwKeTF9/eIR6WcoU+9/ABZJeD0PrKyTr8aWDhSS9PVPUBwm36KeuXuDDkv4Z+AXwJZL+zJ+S9ImuK60QEc9L+vJoZcr4CvCbwMOSBoAvR8T16eGaX5R0FMl6ch1JN5BZOTMk/Yik8dicDrsK+IakJ4EHgAVl6mUpMySSe2G0Ad+TtBf4MUmf/yeAGyQ9TLK+fh+4ZIKfKTd8CQQzs5xz142ZWc450ZuZ5ZwTvZlZzjnRm5nlnBO9mVnOOdGbmeWcE72ZWc79f9mfPuUcAU3UAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.boxplot([balance_losses, inbalance_losses],\n",
    "            labels=['balance', 'inbalance'])\n",
    "plt.title(\"MSE of confusion matrices from rte data\")\n",
    "plt.savefig(\"pic/rte_balance_box.png\", dpi=300)\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "7b4b4feff2f24a0f0a34464dbe537a36fda679851528fb8735cb41fa49dffb2d"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit ('base': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}