GraphEngine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cf90affb-1055-4a61-9e90-8b4eb7c40ebe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "import matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "18f72c65",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.makedirs('plots/TC', exist_ok=True)\n",
    "os.makedirs('plots/MSC', exist_ok=True)\n",
    "os.makedirs('processed/TC', exist_ok=True)\n",
    "os.makedirs('processed/MSC', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "94227915-2fdf-497e-aee3-58d50ce6acaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sheet = 'Multi Session Chat View 1'\n",
    "# sheet = 'Topical Chat View 1'\n",
    "# df = pd.read_excel('ITL_20230521.xlsx', sheet_name=sheet)\n",
    "# df = df.loc[df.Model != \"Gap\"]\n",
    "\n",
    "# Remove unnamed columns\n",
    "# df = df.loc[:, ~df.columns.str.contains('^Unnamed')]\n",
    "\n",
    "# New Results from data/evals.csv\n",
    "df = pd.read_csv('data/evals.csv')\n",
    "# dataset_prefix = \"MSC\"\n",
    "dataset_prefix = \"TC\"\n",
    "df = df.loc[df.dataset == dataset_prefix]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a8c7a037",
   "metadata": {},
   "outputs": [],
   "source": [
    "if dataset_prefix == \"MSC\":\n",
    "    # title_fontsize = 36\n",
    "    legend_fontsize = 18\n",
    "    label_fontsize = 24\n",
    "else:\n",
    "    # dataset_prefix = \"TC\"\n",
    "    # title_fontsize = \n",
    "    legend_fontsize = 20\n",
    "    label_fontsize = 20\n",
    "\n",
    "    # Rename \"Knowledge\" as \"Persona Signal\"\n",
    "    df = df.rename(columns={\"Knowledge\": \"Persona Signal\"})\n",
    "\n",
    "# Set to a common linux font\n",
    "fonts = {'family' : 'serif',\n",
    "        'serif' : 'Caladea',\n",
    "        # 'weight' : 'bold',\n",
    "        'size'   : 36\n",
    "        }\n",
    "# For some wierd reason you need to set the \n",
    "# font size like following\n",
    "# 36 for MSC\n",
    "# 40 for TC \n",
    "\n",
    "matplotlib.rc('font', **fonts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c709e3d7-3c8e-4fed-99ea-28d2886cefef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Short', 'Orig', 'Short (ppl)', 'Orig (ppl)'], dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Prompt Type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8de212a5-6939-4c1f-9fb3-8e7614976a6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Prompt Type'].replace(\"Orig(.*)\", \"FS-ICL\\\\1\", regex=True, inplace=True)\n",
    "df['Prompt Type'].replace(\"Short(.*)\", \"ZS-ICL\\\\1\", regex=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a2cb987e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop for values ppl1, ppl2, Exp1, Exp2. Or if it is null\n",
    "df = df.loc[~df['Prompt Type'].isin(['ppl1', 'ppl2', 'Exp1', 'Exp2'])]\n",
    "df = df.loc[~df['Prompt Type'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bd3ebaef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ZS-ICL', 'FS-ICL', 'ZS-ICL (ppl)', 'FS-ICL (ppl)'], dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Prompt Type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "99b967dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Full', 'PegasusFT', 'BART', 'Recent-2', 'Recent-4', 'Recent-8',\n",
       "       'Recent-10', 'Semantic-2', 'Semantic-4', 'Semantic-8',\n",
       "       'Semantic-10', 'None', 'Pegasus-CD', 'Recent-1', 'Semantic-1'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"History Signal\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "12545625-d414-4c50-bf68-115f0fb95759",
   "metadata": {},
   "outputs": [],
   "source": [
    "# MSC Rules\n",
    "df['History Signal'].replace(r\"Prev (\\d+)\", r\"Recent-\\1\", regex=True, inplace=True)\n",
    "df['History Signal'].replace(\"4 Semant.*\", \"Semantic-4\", regex=True, inplace=True)\n",
    "df['History Signal'].replace(r\"(\\d+) semantic sim \\(simcse\\)\", r\"Semantic-\\1\", regex=True, inplace=True)\n",
    "df['History Signal'].replace(\"semantic sim (simcse)\", \"Semantic-4\", inplace=True)\n",
    "df['History Signal'].replace(\"Pegasus cnn/dm\", \"Pegasus-CD\", inplace=True)\n",
    "df['Persona Signal'].replace(\"Pegasus cnn/dm\", \"Pegasus-CD\", inplace=True)\n",
    "# # TC Conversion Rules\n",
    "df['History Signal'].replace(\"4 semantic sim (simcse)\", \"Semantic-4\", inplace=True)\n",
    "df['History Signal'].replace(\"4 semantic sim\", \"Semantic-4\", regex=True, inplace=True)\n",
    "df['Model'].replace(\"T-k instruct\", \"Tk-Instruct\", inplace=True)\n",
    "df['Model'].replace(\"Tk-instruct\", \"Tk-Instruct\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3f7ee1a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Full', 'PegasusFT', 'BART', 'Recent-2', 'Recent-4', 'Recent-8',\n",
       "       'Recent-10', 'Semantic-2', 'Semantic-4', 'Semantic-8',\n",
       "       'Semantic-10', 'None', 'Pegasus-CD', 'Recent-1', 'Semantic-1'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"History Signal\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "18044f6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Model'].replace(\"text-davinci-003\", \"GPT-3\", inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f88e4633",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop rows with Persona Signal = PegasusFT\n",
    "df = df.loc[df['Persona Signal'] != \"PegasusFT\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c7e18ef9-ab0c-44a5-8cd4-f4936bd0acf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>Model</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>prompt_type</th>\n",
       "      <th>few_shot</th>\n",
       "      <th>background_knowledge</th>\n",
       "      <th>History Signal</th>\n",
       "      <th>history_k</th>\n",
       "      <th>wandb</th>\n",
       "      <th>num_gpus</th>\n",
       "      <th>...</th>\n",
       "      <th>Persona Signal</th>\n",
       "      <th>Prompt Type</th>\n",
       "      <th>Method</th>\n",
       "      <th>BLEU</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>Bert-p</th>\n",
       "      <th>Bert-r</th>\n",
       "      <th>Bert-f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>TC</td>\n",
       "      <td>T0</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Semantic-10</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>TC</td>\n",
       "      <td>T0</td>\n",
       "      <td>12</td>\n",
       "      <td>ppl</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Semantic-8</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294</th>\n",
       "      <td>TC</td>\n",
       "      <td>Tk-Instruct</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Full</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>TC</td>\n",
       "      <td>T0</td>\n",
       "      <td>12</td>\n",
       "      <td>ppl</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Recent-10</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>474</th>\n",
       "      <td>TC</td>\n",
       "      <td>GPT-3</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Recent-1</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>TC</td>\n",
       "      <td>Tk-Instruct</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>BART</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>ZS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462</th>\n",
       "      <td>TC</td>\n",
       "      <td>GPT-3</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>PegasusFT</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>Pegasus-CD</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>TC</td>\n",
       "      <td>GPT-3</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Recent-4</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>ZS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>TC</td>\n",
       "      <td>T0</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Recent-4</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>TC</td>\n",
       "      <td>T0</td>\n",
       "      <td>12</td>\n",
       "      <td>ppl</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Semantic-4</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>TC</td>\n",
       "      <td>flanT5-XL</td>\n",
       "      <td>12</td>\n",
       "      <td>ppl</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Full</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>TC</td>\n",
       "      <td>Tk-Instruct</td>\n",
       "      <td>12</td>\n",
       "      <td>ppl</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Semantic-10</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>FS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>TC</td>\n",
       "      <td>Tk-Instruct</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Recent-8</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>TC</td>\n",
       "      <td>GPT-3</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Recent-2</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>TC</td>\n",
       "      <td>flanT5-XL</td>\n",
       "      <td>12</td>\n",
       "      <td>manual</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Recent-10</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>ZS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    dataset        Model  batch_size prompt_type  few_shot  \\\n",
       "248      TC           T0          12      manual      True   \n",
       "261      TC           T0          12         ppl     False   \n",
       "294      TC  Tk-Instruct          12      manual      True   \n",
       "258      TC           T0          12         ppl     False   \n",
       "474      TC        GPT-3          12      manual      True   \n",
       "282      TC  Tk-Instruct          12      manual     False   \n",
       "462      TC        GPT-3          12      manual      True   \n",
       "440      TC        GPT-3          12      manual     False   \n",
       "242      TC           T0          12      manual      True   \n",
       "260      TC           T0          12         ppl     False   \n",
       "196      TC    flanT5-XL          12         ppl     False   \n",
       "332      TC  Tk-Instruct          12         ppl      True   \n",
       "299      TC  Tk-Instruct          12      manual      True   \n",
       "453      TC        GPT-3          12      manual      True   \n",
       "174      TC    flanT5-XL          12      manual     False   \n",
       "\n",
       "     background_knowledge History Signal  history_k  wandb  num_gpus  ...  \\\n",
       "248                 False    Semantic-10         10  False         1  ...   \n",
       "261                 False     Semantic-8          8  False         1  ...   \n",
       "294                 False           Full          4  False         1  ...   \n",
       "258                 False      Recent-10         10  False         1  ...   \n",
       "474                 False       Recent-1          1  False         1  ...   \n",
       "282                 False           BART          4  False         1  ...   \n",
       "462                  True      PegasusFT          4  False         8  ...   \n",
       "440                 False       Recent-4          4  False         8  ...   \n",
       "242                 False       Recent-4          4  False         1  ...   \n",
       "260                 False     Semantic-4          4  False         1  ...   \n",
       "196                 False           Full          4  False         1  ...   \n",
       "332                 False    Semantic-10         10  False         1  ...   \n",
       "299                 False       Recent-8          8  False         1  ...   \n",
       "453                 False       Recent-2          2  False         8  ...   \n",
       "174                 False      Recent-10         10  False         1  ...   \n",
       "\n",
       "     Persona Signal   Prompt Type  Method  BLEU  rouge1  rouge2 rougeL Bert-p  \\\n",
       "248            None        FS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "261            None  ZS-ICL (ppl)     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "294            None        FS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "258            None  ZS-ICL (ppl)     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "474            None        FS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "282            None        ZS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "462      Pegasus-CD        FS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "440            None        ZS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "242            None        FS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "260            None  ZS-ICL (ppl)     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "196            None  ZS-ICL (ppl)     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "332            None  FS-ICL (ppl)     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "299            None        FS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "453            None        FS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "174            None        ZS-ICL     0.5   0.5     0.5     0.5    0.5    0.5   \n",
       "\n",
       "     Bert-r  Bert-f1  \n",
       "248     0.5      0.5  \n",
       "261     0.5      0.5  \n",
       "294     0.5      0.5  \n",
       "258     0.5      0.5  \n",
       "474     0.5      0.5  \n",
       "282     0.5      0.5  \n",
       "462     0.5      0.5  \n",
       "440     0.5      0.5  \n",
       "242     0.5      0.5  \n",
       "260     0.5      0.5  \n",
       "196     0.5      0.5  \n",
       "332     0.5      0.5  \n",
       "299     0.5      0.5  \n",
       "453     0.5      0.5  \n",
       "174     0.5      0.5  \n",
       "\n",
       "[15 rows x 26 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2366f747",
   "metadata": {},
   "outputs": [],
   "source": [
    "used_cols = ['Model', 'Method', 'Persona Signal', 'History Signal', 'Prompt Type',\n",
    "       'BLEU', 'METEOR', 'rouge1', 'rouge2', 'rougeL', 'Bert-p', 'Bert-r',\n",
    "       'Bert-f1', 'DEB', 'Bleurt', 'output', 'prompt']\n",
    "df = df[used_cols]\n",
    "df['total_budget'] = df['output'] + df['prompt']\n",
    "df['BLEURT']=df['Bleurt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "843a7ba5-a852-41eb-a4a2-4584c5d4775e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values(['Model', 'Prompt Type'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "479dbdec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Method</th>\n",
       "      <th>Persona Signal</th>\n",
       "      <th>History Signal</th>\n",
       "      <th>Prompt Type</th>\n",
       "      <th>BLEU</th>\n",
       "      <th>METEOR</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>Bert-p</th>\n",
       "      <th>Bert-r</th>\n",
       "      <th>Bert-f1</th>\n",
       "      <th>DEB</th>\n",
       "      <th>Bleurt</th>\n",
       "      <th>output</th>\n",
       "      <th>prompt</th>\n",
       "      <th>total_budget</th>\n",
       "      <th>BLEURT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>GPT-3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>None</td>\n",
       "      <td>Full</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.156904</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.952048</td>\n",
       "      <td>0.363777</td>\n",
       "      <td>39.621147</td>\n",
       "      <td>798.293515</td>\n",
       "      <td>837.914662</td>\n",
       "      <td>0.363777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>GPT-3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>None</td>\n",
       "      <td>PegasusFT</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.134215</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.824261</td>\n",
       "      <td>0.336097</td>\n",
       "      <td>37.871058</td>\n",
       "      <td>271.986549</td>\n",
       "      <td>309.857607</td>\n",
       "      <td>0.336097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>GPT-3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>None</td>\n",
       "      <td>BART</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.134018</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.822749</td>\n",
       "      <td>0.336770</td>\n",
       "      <td>37.702655</td>\n",
       "      <td>270.801265</td>\n",
       "      <td>308.503919</td>\n",
       "      <td>0.336770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>GPT-3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>None</td>\n",
       "      <td>Recent-2</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.141298</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.895496</td>\n",
       "      <td>0.340845</td>\n",
       "      <td>37.341573</td>\n",
       "      <td>322.257394</td>\n",
       "      <td>359.598967</td>\n",
       "      <td>0.340845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>GPT-3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>None</td>\n",
       "      <td>Recent-4</td>\n",
       "      <td>FS-ICL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.144644</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.904315</td>\n",
       "      <td>0.342487</td>\n",
       "      <td>38.920052</td>\n",
       "      <td>410.100036</td>\n",
       "      <td>449.020087</td>\n",
       "      <td>0.342487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>flanT5-XL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>Pegasus-CD</td>\n",
       "      <td>PegasusFT</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.114752</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.792150</td>\n",
       "      <td>0.300725</td>\n",
       "      <td>48.384473</td>\n",
       "      <td>351.893535</td>\n",
       "      <td>400.278007</td>\n",
       "      <td>0.300725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>flanT5-XL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>Pegasus-CD</td>\n",
       "      <td>BART</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.114752</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.792150</td>\n",
       "      <td>0.300725</td>\n",
       "      <td>48.384473</td>\n",
       "      <td>351.893535</td>\n",
       "      <td>400.278007</td>\n",
       "      <td>0.300725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>flanT5-XL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>None</td>\n",
       "      <td>Pegasus-CD</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.122815</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.841282</td>\n",
       "      <td>0.360340</td>\n",
       "      <td>40.130991</td>\n",
       "      <td>141.813335</td>\n",
       "      <td>181.944326</td>\n",
       "      <td>0.360340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>flanT5-XL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>None</td>\n",
       "      <td>Recent-1</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.129332</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.783195</td>\n",
       "      <td>0.354326</td>\n",
       "      <td>37.009398</td>\n",
       "      <td>109.671032</td>\n",
       "      <td>146.680429</td>\n",
       "      <td>0.354326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>380</th>\n",
       "      <td>flanT5-XL</td>\n",
       "      <td>0.5</td>\n",
       "      <td>None</td>\n",
       "      <td>Semantic-1</td>\n",
       "      <td>ZS-ICL (ppl)</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.126044</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.833928</td>\n",
       "      <td>0.360522</td>\n",
       "      <td>31.756993</td>\n",
       "      <td>111.482496</td>\n",
       "      <td>143.239489</td>\n",
       "      <td>0.360522</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>238 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Model  Method Persona Signal History Signal   Prompt Type  BLEU  \\\n",
       "450      GPT-3     0.5           None           Full        FS-ICL   0.5   \n",
       "451      GPT-3     0.5           None      PegasusFT        FS-ICL   0.5   \n",
       "452      GPT-3     0.5           None           BART        FS-ICL   0.5   \n",
       "453      GPT-3     0.5           None       Recent-2        FS-ICL   0.5   \n",
       "454      GPT-3     0.5           None       Recent-4        FS-ICL   0.5   \n",
       "..         ...     ...            ...            ...           ...   ...   \n",
       "208  flanT5-XL     0.5     Pegasus-CD      PegasusFT  ZS-ICL (ppl)   0.5   \n",
       "209  flanT5-XL     0.5     Pegasus-CD           BART  ZS-ICL (ppl)   0.5   \n",
       "378  flanT5-XL     0.5           None     Pegasus-CD  ZS-ICL (ppl)   0.5   \n",
       "379  flanT5-XL     0.5           None       Recent-1  ZS-ICL (ppl)   0.5   \n",
       "380  flanT5-XL     0.5           None     Semantic-1  ZS-ICL (ppl)   0.5   \n",
       "\n",
       "       METEOR  rouge1  rouge2  rougeL  Bert-p  Bert-r  Bert-f1       DEB  \\\n",
       "450  0.156904     0.5     0.5     0.5     0.5     0.5      0.5  0.952048   \n",
       "451  0.134215     0.5     0.5     0.5     0.5     0.5      0.5  0.824261   \n",
       "452  0.134018     0.5     0.5     0.5     0.5     0.5      0.5  0.822749   \n",
       "453  0.141298     0.5     0.5     0.5     0.5     0.5      0.5  0.895496   \n",
       "454  0.144644     0.5     0.5     0.5     0.5     0.5      0.5  0.904315   \n",
       "..        ...     ...     ...     ...     ...     ...      ...       ...   \n",
       "208  0.114752     0.5     0.5     0.5     0.5     0.5      0.5  0.792150   \n",
       "209  0.114752     0.5     0.5     0.5     0.5     0.5      0.5  0.792150   \n",
       "378  0.122815     0.5     0.5     0.5     0.5     0.5      0.5  0.841282   \n",
       "379  0.129332     0.5     0.5     0.5     0.5     0.5      0.5  0.783195   \n",
       "380  0.126044     0.5     0.5     0.5     0.5     0.5      0.5  0.833928   \n",
       "\n",
       "       Bleurt     output      prompt  total_budget    BLEURT  \n",
       "450  0.363777  39.621147  798.293515    837.914662  0.363777  \n",
       "451  0.336097  37.871058  271.986549    309.857607  0.336097  \n",
       "452  0.336770  37.702655  270.801265    308.503919  0.336770  \n",
       "453  0.340845  37.341573  322.257394    359.598967  0.340845  \n",
       "454  0.342487  38.920052  410.100036    449.020087  0.342487  \n",
       "..        ...        ...         ...           ...       ...  \n",
       "208  0.300725  48.384473  351.893535    400.278007  0.300725  \n",
       "209  0.300725  48.384473  351.893535    400.278007  0.300725  \n",
       "378  0.360340  40.130991  141.813335    181.944326  0.360340  \n",
       "379  0.354326  37.009398  109.671032    146.680429  0.354326  \n",
       "380  0.360522  31.756993  111.482496    143.239489  0.360522  \n",
       "\n",
       "[238 rows x 19 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c8c4ad2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # set and increase Matplotlib font\n",
    "# plt.rcParams['font.family'] = 'sans-serif'\n",
    "# plt.rcParams['font.sans-serif'] = 'Arial'\n",
    "# plt.rcParams['font.size'] = 24\n",
    "\n",
    "# font = {'family' : 'normal',\n",
    "#         'weight' : 'bold',\n",
    "#         'size'   : 32}\n",
    "\n",
    "# matplotlib.rc('font', **font)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f85fa352",
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_nan_row_cols(df):\n",
    "    # print(\"Shape of df: {}\".format(df.shape))\n",
    "    # Print the columns/rows that will be dropped\n",
    "    # print(\"Columns with all nan: {}\".format(df.columns[df.isnull().all(axis=0)]))\n",
    "    # print(\"Rows with all nan: {}\".format(df.index[df.isnull().all(axis=1)]))\n",
    "    # Drop rows with all nan\n",
    "    df = df.dropna(axis=0, how='all')\n",
    "    # Drop columns with all nan\n",
    "    df = df.dropna(axis=1, how='all')\n",
    "    # print(\"Shape of df after dropping rows and columns with all nan: {}\".format(df.shape))\n",
    "    return df\n",
    "\n",
    "def scatter_plot_pivoted_df(pivoted_df, metric, save_path, is_roi=False):\n",
    "    # Print shape\n",
    "    # print(pivoted_df)\n",
    "\n",
    "    print(\"Metric: {}\".format(metric))\n",
    "    print(\"Save path: {}\".format(save_path))\n",
    "    \n",
    "    # Check if any row or column is all nan\n",
    "    # if pivoted_df.isnull().all(axis=1).any() or pivoted_df.isnull().all(axis=0).any():\n",
    "    #     print(\"Skipping plot because of all nan row or column for {} @ {}\".format(metric, save_path))\n",
    "    #     return\n",
    "\n",
    "    # # Drop rows with all nan\n",
    "    # pivoted_df = pivoted_df.dropna(axis=0, how='all')\n",
    "    # # Drop columns with all nan\n",
    "    # pivoted_df = pivoted_df.dropna(axis=1, how='all')\n",
    "\n",
    "\n",
    "    if pivoted_df.shape[0] == 0:\n",
    "        print(\"Skipping plot for {} @ {}\".format(metric, save_path))\n",
    "        return\n",
    "\n",
    "    colors = sns.color_palette('Paired', len(pivoted_df.columns))\n",
    "    markers = ['o', 'v', 's', 'p', 'P', '*', 'X', 'D', 'd', '1', '2', '3', '4', '8', 'h', 'H', 'x', 'X', 'D', 'd', '|', '_']\n",
    "\n",
    "    # Create scatter plot\n",
    "    fig, ax = plt.subplots(figsize=(8,6))\n",
    "    # print(pivoted_df.index.tolist())\n",
    "    # Scatter plot\n",
    "    models = pivoted_df.columns\n",
    "    for i, model in enumerate(models):\n",
    "        x = pivoted_df.index.tolist()\n",
    "        y = pivoted_df[model]\n",
    "        # y shouldn't be all nans\n",
    "        if y.isnull().all():\n",
    "            continue\n",
    "        lbl = f\"{model[0]} ({model[1].replace('-ICL', '')})\"\n",
    "        ax.scatter(x, y, color = colors[i], marker = markers[i], label=lbl, s=150)\n",
    "\n",
    "    # Set x-axis labels\n",
    "    ax.set_xticks(list(range(len(pivoted_df.index.tolist()))))\n",
    "    ax.set_xticklabels(pivoted_df.index.tolist(), rotation=90, fontsize=label_fontsize)\n",
    "\n",
    "    # Set y-axis tick font\n",
    "    ax.tick_params(axis='y', labelsize=label_fontsize)\n",
    "\n",
    "    # Add legend at the top of the figure\n",
    "    # show in a single row\n",
    "    # Smaller font size\n",
    "    if len(models) <= 5:\n",
    "        ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.26), ncol=2, fontsize=legend_fontsize)\n",
    "    else:\n",
    "        ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.43), ncol=2, fontsize=legend_fontsize)\n",
    "\n",
    "    ax.grid(True, axis='y')\n",
    "    # x-axis label\n",
    "    ax.set_xlabel('History Signal', fontsize=label_fontsize)\n",
    "    # y-axis label\n",
    "    if is_roi:\n",
    "        ax.set_ylabel(f'ROI ({metric})', fontsize=label_fontsize)\n",
    "    else:\n",
    "        ax.set_ylabel(f'{metric}', fontsize=label_fontsize)\n",
    "\n",
    "\n",
    "    # plt.tight_layout()\n",
    "    # Save such that legends are fitted within the figure\n",
    "    plt.savefig(save_path, bbox_inches='tight', dpi=300)\n",
    "    plt.close()\n",
    "\n",
    "    return\n",
    "\n",
    "    # Separate plot averaged over all models (two groups FS and ZS)\n",
    "    fig, ax = plt.subplots(figsize=(8,6))\n",
    "\n",
    "    # Scatter plot\n",
    "    models = pivoted_df.columns\n",
    "    fs_models = [model for model in models if \"FS\" in model[1]]\n",
    "    x = pivoted_df.index.tolist()\n",
    "    y = pivoted_df[fs_models].mean(axis=1)\n",
    "    ax.scatter(x, y, color = colors[0], marker = markers[0], label='Model Avg. (FS)', s=150)\n",
    "\n",
    "\n",
    "    zs_models = [model for model in models if \"ZS\" in model[1]]\n",
    "    y = pivoted_df[zs_models].mean(axis=1)\n",
    "    ax.scatter(x, y, color = colors[1], marker = markers[1], label='Model Avg. (ZS)', s=150)\n",
    "\n",
    "    # Set x-axis labels\n",
    "    ax.set_xticks(list(range(len(pivoted_df.index.tolist()))))\n",
    "    ax.set_xticklabels(pivoted_df.index.tolist(), rotation=90, fontsize=label_fontsize)\n",
    "\n",
    "    # Set y-axis tick font\n",
    "    ax.tick_params(axis='y', labelsize=label_fontsize)\n",
    "    \n",
    "    # Add legend at the top of the figure\n",
    "    # show in a single row\n",
    "    # Smaller font size\n",
    "    ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.16), ncol=2, fontsize=legend_fontsize)\n",
    "\n",
    "    ax.grid(True, axis='y')\n",
    "    # x-axis label\n",
    "    ax.set_xlabel('History Signal', fontsize=label_fontsize)\n",
    "    # y-axis label\n",
    "    if is_roi:\n",
    "        ax.set_ylabel(f'ROI ({metric})', fontsize=label_fontsize)\n",
    "    else:\n",
    "        ax.set_ylabel(f'{metric}', fontsize=label_fontsize)\n",
    "\n",
    "    \n",
    "    # plt.tight_layout()\n",
    "    # Save such that legends are fitted within the figure\n",
    "    plt.savefig(save_path.replace('.pdf', '_avg.pdf'), bbox_inches='tight', dpi=300)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cdbbf034-e258-436d-8971-a45448572559",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_ROI(df_perf, df_budget, metric_exp=10.0):\n",
    "    # print(df_perf)\n",
    "    assert df_perf.shape == df_budget.shape\n",
    "    # Find the rows with index \"None_0\" or \"No DH, no BI\", only one of these should exist\n",
    "    if \"None_0\" in df_perf.index:\n",
    "        X_perf = df_perf.loc['None_0']\n",
    "        X_budget = df_budget.loc['None_0']\n",
    "    # elif \"No DH, no BI\" in df_perf.index:\n",
    "    #     assert \"None_0\" not in df_perf.index\n",
    "    #     X_perf = df_perf.loc['No DH, no BI']\n",
    "    #     X_budget = df_budget.loc['No DH, no BI']\n",
    "    else:\n",
    "        raise ValueError(\"None_0 not found in df_perf.index.\")\n",
    "\n",
    "    # assert df_perf.index[0] == \"None\" or df_perf.index[0] == \"No DH, no BI\"\n",
    "    # Match index and column order\n",
    "    assert df_perf.index.tolist() == df_budget.index.tolist()\n",
    "    assert df_perf.columns.tolist() == df_budget.columns.tolist()\n",
    "\n",
    "    # Calculate ROI: w.r.t to best baseline in \"None\"\n",
    "    # df_delta = df_perf - X_perf\n",
    "    # df_delta_budget = df_budget - X_budget\n",
    "    # Unnormalized ROI\n",
    "    df_delta = (100*df_perf)**metric_exp\n",
    "    df_delta_budget = df_budget\n",
    "    df_roi = 100*df_delta / df_delta_budget\n",
    "    return df_roi.iloc[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2c4eb935-cd66-48f0-a026-d3f1b7557ed6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def proc_sub_df_persona(sub_df, baseline_df, signal, prompt_type_groups, metric, all_subframes, dataset_prefix):\n",
    "    for prompt_types_name, prompt_types in prompt_type_groups.items():\n",
    "        # prompt_sub_df = sub_df\n",
    "        prompt_sub_df = sub_df[sub_df['Prompt Type'].isin(prompt_types)]\n",
    "        # Remove old baselines\n",
    "        # prompt_sub_df = sub_df[~(sub_df['History Signal'] == \"None\")]\n",
    "        # Rename this baseline to \"None_1\"\n",
    "        prompt_sub_df.loc[prompt_sub_df['History Signal'] == 'None', 'History Signal'] = 'None_1'\n",
    "        \n",
    "        # Add common baseline - prompt type would be non-ppl\n",
    "        # prompt_baseline_df = baseline_df[baseline_df['Prompt Type'].isin(prompt_types)]\n",
    "        prompt_baseline_df = baseline_df[baseline_df['Prompt Type'].isin(prompt_type_groups['normal'])]\n",
    "        # overwrite prompt type of baseline\n",
    "        for i, row in prompt_baseline_df.iterrows():\n",
    "            if row['Prompt Type'] == 'FS-ICL':\n",
    "                prompt_baseline_df.at[i, 'Prompt Type'] = prompt_types[0]\n",
    "            elif row['Prompt Type'] == 'ZS-ICL':\n",
    "                prompt_baseline_df.at[i, 'Prompt Type'] = prompt_types[1]\n",
    "        # rename this baseline to \"None_0\"\n",
    "        prompt_baseline_df['History Signal'] = \"None_0\"\n",
    "        prompt_sub_df  = prompt_sub_df.append(prompt_baseline_df)\n",
    "        \n",
    "        print(\"History:\", prompt_sub_df['History Signal'].unique())\n",
    "        print(\"Prompt types:\", prompt_sub_df['Prompt Type'].unique())\n",
    "\n",
    "        # Remove blenderbot\n",
    "        prompt_sub_df = prompt_sub_df[~prompt_sub_df['Model'].isin([\"BlenderBot-3B\"])]\n",
    "        if len(prompt_sub_df) <= 0:\n",
    "            continue\n",
    "\n",
    "        pivoted_df = prompt_sub_df.pivot(index='History Signal', columns=['Model', 'Prompt Type'], values=metric)        \n",
    "        pivoted_budget = prompt_sub_df.pivot(index='History Signal', columns=['Model', 'Prompt Type'], values='total_budget')\n",
    "        pivoted_input_budget = prompt_sub_df.pivot(index='History Signal', columns=['Model', 'Prompt Type'], values='prompt')\n",
    "\n",
    "        # Fixed model order\n",
    "        history_order = [\"None_0\", \"None_1\", \"BART\", \"Pegasus-CD\", \"PegasusFT\", \n",
    "                         \"Recent-1\", \"Recent-2\", \"Recent-4\", \"Recent-8\", \"Recent-10\", \"Recent-16\", \n",
    "                         \"Semantic-1\", \"Semantic-2\", \"Semantic-4\", \"Semantic-8\", \"Semantic-10\", \"Semantic-16\",\"Full\"\n",
    "                         ]\n",
    "        # remove history signals that are not in the data\n",
    "        history_order = [h for h in history_order if h in prompt_sub_df['History Signal'].unique()]\n",
    "        pivoted_df = pivoted_df.reindex(history_order)\n",
    "        pivoted_budget = pivoted_budget.reindex(history_order)\n",
    "        pivoted_input_budget = pivoted_input_budget.reindex(history_order)\n",
    "\n",
    "        # ROI: based on first row (None)\n",
    "        ROI = calculate_ROI(pivoted_df, pivoted_budget)\n",
    "\n",
    "        models = ['flanT5-XL',\n",
    "                'T0',\n",
    "                'Tk-Instruct',\n",
    "                'GPT-3']\n",
    "        # remove models that are not in the data\n",
    "        models = [m for m in models if m in prompt_sub_df['Model'].unique()] \n",
    "        # icls = [\"ZS-ICL\", \"FS-ICL\", \"ZS-ICL (ppl)\", \"FS-ICL (ppl)\"]\n",
    "        icls = prompt_types\n",
    "        model_icl_pairs = list(itertools.product(models, icls))\n",
    "        pivoted_df = pivoted_df.reindex(model_icl_pairs, axis=1)\n",
    "        pivoted_budget = pivoted_budget.reindex(model_icl_pairs, axis=1)\n",
    "        pivoted_input_budget = pivoted_input_budget.reindex(model_icl_pairs, axis=1)\n",
    "        ROI = ROI.reindex(model_icl_pairs, axis=1)\n",
    "\n",
    "        # Clean the dataframes of empty columns/rows\n",
    "        pivoted_df = drop_nan_row_cols(pivoted_df)\n",
    "        pivoted_budget = drop_nan_row_cols(pivoted_budget)\n",
    "        pivoted_input_budget = drop_nan_row_cols(pivoted_input_budget)\n",
    "        ROI = drop_nan_row_cols(ROI)\n",
    "\n",
    "        all_subframes[signal + \"_\" + prompt_types_name] = pivoted_df\n",
    "        all_subframes[signal + \"_\" + prompt_types_name + \"_budget\"] = pivoted_budget\n",
    "        all_subframes[signal + \"_\" + prompt_types_name + \"_ibudget\"] = pivoted_input_budget\n",
    "        if \"None_0\" in pivoted_df.index:\n",
    "            all_subframes[signal + \"_\" + prompt_types_name + \"_ROI\"] = ROI\n",
    "        else:\n",
    "            print(\"Cannot calculate ROI without NoHistory baseline (i.e. None).\")\n",
    "\n",
    "        # Plot absolute metrics\n",
    "        f1 = f'plots/{dataset_prefix}/{metric}_{signal}_{prompt_types_name}.pdf'\n",
    "        scatter_plot_pivoted_df(pivoted_df, metric, f1)\n",
    "\n",
    "        # Plot ROI\n",
    "        if \"None\" in pivoted_df.index:\n",
    "            f2 = f'plots/{dataset_prefix}/{metric}_{signal}_{prompt_types_name}_ROI.pdf'\n",
    "            scatter_plot_pivoted_df(ROI, metric, f2, is_roi=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "0881fff0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_runs(framedict, metric, dataset_prefix, target=\"ppl\"):\n",
    "    assert target in [\"normal\", \"ppl\"], \"merge target must be either normal or ppl\"\n",
    "\n",
    "    # DF Merged, Only for None_ppl, PegasusFT_*\n",
    "    def is_budget(x):\n",
    "        return x.endswith(\"_budget\") or x.endswith(\"_ibudget\")\n",
    "    \n",
    "    def is_roi(x):\n",
    "        return x.endswith(\"_ROI\")\n",
    "    \n",
    "    def is_runlist(x):\n",
    "        # Set of runs that we want to keep\n",
    "        # No persona and PegasusFT-based persona summary\n",
    "        return x.startswith(\"None\") or x.startswith(\"PegasusFT\") or x.startswith(\"Pegasus-CD\")\n",
    "\n",
    "    modded_run_keys = list(\n",
    "        filter(\n",
    "            lambda x: is_runlist(x) and not is_roi(x) and not is_budget(x),\n",
    "            framedict.keys()\n",
    "        )\n",
    "    )\n",
    "    df_modded_runs_group = {k: v for k, v in framedict.items() if k in modded_run_keys}\n",
    "\n",
    "    budget_keys = list(\n",
    "        filter(lambda x: is_budget(x), framedict.keys())\n",
    "    )\n",
    "    df_budgets = {k.replace(\"_budget\", \"\"): v for k, v in framedict.items() if k in budget_keys}\n",
    "    df_budgets = {k: v for k, v in df_budgets.items() if k in df_modded_runs_group}\n",
    "\n",
    "    # Baseline is None_normal, index \"None\"\n",
    "    df_baseline = df_modded_runs_group[\"None_normal\"].iloc[:1]\n",
    "    df_baseline_budget = df_budgets[\"None_normal\"].iloc[:1]\n",
    "    # check that history is None\n",
    "    assert df_baseline.index[0] == \"None_0\" and df_baseline_budget.index[0] == \"None_0\", \"Baseline must be None_0\"\n",
    "\n",
    "    to_remove = []\n",
    "    if target == \"ppl\":\n",
    "        for k, v in df_modded_runs_group.items():\n",
    "            if k.endswith(\"_normal\"):\n",
    "                to_remove.append(k)\n",
    "    elif target == \"normal\":\n",
    "        for k, v in df_modded_runs_group.items():\n",
    "            if k.endswith(\"_ppl\"):\n",
    "                to_remove.append(k)\n",
    "        # del df_modded_runs_group[\"None_ppl\"]\n",
    "        # del df_budgets[\"None_ppl\"]\n",
    "    print(\"TO REMOVE:\", to_remove)\n",
    "    for k in to_remove:\n",
    "        del df_modded_runs_group[k]\n",
    "        del df_budgets[k]\n",
    "    print(\"REMAINING:\", df_modded_runs_group.keys())\n",
    "\n",
    "\n",
    "    # Remove (ppl) from column names and \"None\" history rows\n",
    "    df_modded_runs_group = {k: v.rename(columns=lambda x: x.replace(\" (ppl)\", \"\")) for k, v in df_modded_runs_group.items()}\n",
    "    # df_modded_runs_group = {k: v.drop(\"None\") for k, v in df_modded_runs_group.items()}\n",
    "\n",
    "    # Do the same for budgets\n",
    "    df_budgets = {k: v.rename(columns=lambda x: x.replace(\" (ppl)\", \"\")) for k, v in df_budgets.items()}\n",
    "    # df_budgets = {k: v.drop(\"None\") for k, v in df_budgets.items()}\n",
    "\n",
    "    # Create merged df\n",
    "    # Keys as a new column, and reset index, history as another new column\n",
    "    df_merged = pd.concat(df_modded_runs_group, keys=df_modded_runs_group.keys()).reset_index(level=1)\n",
    "    df_merged_budget = pd.concat(df_budgets, keys=df_budgets.keys()).reset_index(level=1)\n",
    "\n",
    "    # Reset index again, rename the old one to \"Config\"\n",
    "    df_merged = df_merged.reset_index().rename(columns={\"index\": \"Config\"})\n",
    "    df_merged_budget = df_merged_budget.reset_index().rename(columns={\"index\": \"Config\"})\n",
    "\n",
    "    # Add the baseline row at the top\n",
    "    # TODO: It don't think this is necessary anymore, so I commented it out (May 21, 2023)\n",
    "    # df_merged = df_merged.append(df_baseline.reset_index().rename(columns={\"index\": \"Config\"}))\n",
    "    # df_merged_budget = df_merged_budget.append(df_baseline_budget.reset_index().rename(columns={\"index\": \"Config\"}))\n",
    "\n",
    "    # Move last row to the top\n",
    "    df_merged = df_merged.iloc[-1:].append(df_merged.iloc[:-1]).reset_index(drop=True)\n",
    "    df_merged_budget = df_merged_budget.iloc[-1:].append(df_merged_budget.iloc[:-1]).reset_index(drop=True)\n",
    "\n",
    "    # Final names for DH\n",
    "    replacements_2 = {\n",
    "        # \"None\": \"No DH, no BI\",\n",
    "        \"None_1\": \"No DH, no BI\",\n",
    "        \"BART\": \"BART-D\",\n",
    "        \"PegasusFT\": \"Pegasus-DS\",\n",
    "        \"Pegasus-CD\": \"Pegasus-CD\",\n",
    "    }\n",
    "    df_merged[\"History Signal\"] = df_merged[\"History Signal\"].replace(replacements_2)\n",
    "    df_merged_budget[\"History Signal\"] = df_merged_budget[\"History Signal\"].replace(replacements_2)\n",
    " \n",
    "    if target==\"ppl\":\n",
    "        replace_dict = {\n",
    "            \"None_ppl\": \"\",\n",
    "            \"PegasusFT_ppl\": \" + BI(P_FT)\",\n",
    "            \"Pegasus-CD_ppl\": \" + BI\"\n",
    "        }\n",
    "    elif target==\"normal\":\n",
    "        replace_dict = {\n",
    "            \"None_normal\": \"\",\n",
    "            \"PegasusFT_normal\": \" + BI(P_FT)\",\n",
    "            \"Pegasus-CD_normal\": \" + BI\"\n",
    "        }\n",
    "\n",
    "\n",
    "    df_merged[\"Config\"] = df_merged[\"Config\"].fillna(\"\").replace(replace_dict)\n",
    "    df_merged_budget[\"Config\"] = df_merged_budget[\"Config\"].fillna(\"\").replace(replace_dict)\n",
    "\n",
    "    # Add config to history signal\n",
    "    df_merged[\"History Signal\"] = df_merged[\"History Signal\"] + df_merged[\"Config\"]\n",
    "    df_merged_budget[\"History Signal\"] = df_merged_budget[\"History Signal\"] + df_merged_budget[\"Config\"]\n",
    "\n",
    "    # One \"None_0 + BI\" might have been added, remove it -- None_0 is results from the orig none_none setup\n",
    "    df_merged = df_merged[df_merged[\"History Signal\"] != \"None_0 + BI\"]\n",
    "    df_merged_budget = df_merged_budget[df_merged_budget[\"History Signal\"] != \"None_0 + BI\"]\n",
    "\n",
    "    # Drop all None_0 **\n",
    "    # df_merged = df_merged[df_merged[\"History Signal\"] != \"None_0\"]\n",
    "    # df_merged_budget = df_merged_budget[df_merged_budget[\"History Signal\"] != \"None_0\"]\n",
    "\n",
    "    # No DH, No BI + BI -> Only BI\n",
    "    df_merged[\"History Signal\"] = df_merged[\"History Signal\"].replace({\"No DH, no BI + BI\": \"Only BI\"})\n",
    "    df_merged_budget[\"History Signal\"] = df_merged_budget[\"History Signal\"].replace({\"No DH, no BI + BI\": \"Only BI\"})\n",
    "    \n",
    "   # df_merged[\"History Signal\"] = df_merged[\"History Signal\"].replace({\"None\": \"No DH, No BI\"})\n",
    "    # df_merged_budget[\"History Signal\"] = df_merged_budget[\"History Signal\"].replace({\"None\": \"No DH, No BI\"})\n",
    "\n",
    "    # make history signal the index column\n",
    "    df_merged = df_merged.set_index(\"History Signal\")\n",
    "    df_merged_budget = df_merged_budget.set_index(\"History Signal\")\n",
    "\n",
    "    # drop config\n",
    "    df_merged = df_merged.drop(\"Config\", axis=1)\n",
    "    df_merged_budget = df_merged_budget.drop(\"Config\", axis=1)\n",
    "\n",
    "    assert (df_merged.index == df_merged_budget.index).all()\n",
    "\n",
    "    # Convert the column names to tuples by splitting on space\n",
    "    # df_merged.columns = [tuple(col.split()) for col in df_merged.columns]\n",
    "    df_merged = df_merged.reindex(framedict['None_normal'].columns, axis=1)\n",
    "    df_merged_budget = df_merged_budget.reindex(framedict['None_normal'].columns, axis=1)\n",
    "\n",
    "    # print(df_merged)\n",
    "\n",
    "    # Fixed model order\n",
    "    history_order = [\"None_0\", \"No DH, no BI\", \n",
    "                    #  \"Only BI\", \n",
    "                        \"BART-D\", \"Pegasus-CD\", \"Pegasus-DS\", \"Pegasus-DS + BI\", \n",
    "                        \"Recent-1\", \"Recent-2\", \"Recent-4\", \"Recent-8\", \"Recent-10\", \"Recent-16\", \n",
    "                        \"Semantic-1\", \"Semantic-2\", \"Semantic-4\", \"Semantic-8\", \"Semantic-10\", \"Semantic-16\",\"Full\", \n",
    "                        ]\n",
    "    # remove history signals that are not in the data\n",
    "    history_order_chk = [h for h in history_order if h in df_merged.index]\n",
    "    df_merged = df_merged.reindex(history_order_chk)\n",
    "    history_order_chk = [h for h in history_order if h in df_merged_budget.index]\n",
    "    df_merged_budget = df_merged_budget.reindex(history_order_chk)\n",
    "\n",
    "\n",
    "    # Plot the merged df using the scatter_plot_pivoted_df function\n",
    "    f3 = f'plots/{dataset_prefix}/{metric}_merge_{target}.pdf'\n",
    "    scatter_plot_pivoted_df(df_merged[~df_merged.index.isin([\"None_0\", \"No DH, no BI\"])],metric, f3)\n",
    "\n",
    "    # Get the ROI\n",
    "    # df_delta = df_merged - df_merged.iloc[0]\n",
    "    # df_delta_budget = df_merged_budget - df_merged_budget.iloc[0]\n",
    "    # df_merged_ROI = 10_000 * df_delta / df_delta_budget\n",
    "    df_merged_ROI = calculate_ROI(df_merged, df_merged_budget)\n",
    "\n",
    "    # Plot the ROI\n",
    "    f4 = f'plots/{dataset_prefix}/{metric}_merge_{target}_ROI.pdf'\n",
    "    scatter_plot_pivoted_df(df_merged[~df_merged.index.isin([\"None_0\", \"No DH, no BI\"])],metric, f4, is_roi=True)\n",
    "\n",
    "    return df_merged, df_merged_budget, df_merged_ROI\n",
    "\n",
    "def merge_runs_pro(df_merged_normal, df_merged_ppl, metric):\n",
    "    # Copy the df_merged_normal and df_merged_ppl\n",
    "    df_merged_normal = df_merged_normal.copy()\n",
    "    df_merged_ppl = df_merged_ppl.copy()\n",
    "\n",
    "    # If columns are tuples, convert them to strings\n",
    "    if isinstance(df_merged_normal.columns[0], tuple):\n",
    "        df_merged_normal.columns = [\" \".join(col) for col in df_merged_normal.columns]\n",
    "        df_merged_ppl.columns = [\" \".join(col) for col in df_merged_ppl.columns]\n",
    "        \n",
    "    # Average runs by models+FS+normal/ppl)\n",
    "    zs_runs_n = [c for c in df_merged_normal.columns if \"ZS\" in c]\n",
    "    fs_runs_n = [c for c in df_merged_normal.columns if \"FS\" in c]\n",
    "    \n",
    "    # Create new df\n",
    "    df_merged = pd.DataFrame()\n",
    "    df_merged[('Model Avg.','ZS+Manual')] = df_merged_normal[zs_runs_n].mean(axis=1)\n",
    "    df_merged[('Model Avg.','FS+Manual')] = df_merged_normal[fs_runs_n].mean(axis=1)\n",
    "    df_merged[('Model Avg.','ZS+PPL')] = df_merged_ppl[zs_runs_n].mean(axis=1)\n",
    "    df_merged[('Model Avg.','FS+PPL')] = df_merged_ppl[fs_runs_n].mean(axis=1)\n",
    "\n",
    "    # Plot the merged df using the scatter_plot_pivoted_df function\n",
    "    f1 = f'plots/{dataset_prefix}/{metric}_merge.avg.pdf'\n",
    "    scatter_plot_pivoted_df(df_merged[~df_merged.index.isin([\"None_0\", \"No DH, no BI\"])],metric, f1)\n",
    "\n",
    "    return df_merged\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "86e050ca-f46e-4c38-afe8-dfb79d9ec474",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_None_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_None_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_Pegasus-CD_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_Pegasus-CD_ppl.pdf\n",
      "TO REMOVE: ['None_normal', 'Pegasus-CD_normal']\n",
      "REMAINING: dict_keys(['None_ppl', 'Pegasus-CD_ppl'])\n",
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_merge_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_merge_ppl_ROI.pdf\n",
      "TO REMOVE: ['None_ppl', 'Pegasus-CD_ppl']\n",
      "REMAINING: dict_keys(['None_normal', 'Pegasus-CD_normal'])\n",
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_merge_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_merge_normal_ROI.pdf\n",
      "Metric: BLEU\n",
      "Save path: plots/TC/BLEU_merge.avg.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_None_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_None_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_Pegasus-CD_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_Pegasus-CD_ppl.pdf\n",
      "TO REMOVE: ['None_normal', 'Pegasus-CD_normal']\n",
      "REMAINING: dict_keys(['None_ppl', 'Pegasus-CD_ppl'])\n",
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_merge_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_merge_ppl_ROI.pdf\n",
      "TO REMOVE: ['None_ppl', 'Pegasus-CD_ppl']\n",
      "REMAINING: dict_keys(['None_normal', 'Pegasus-CD_normal'])\n",
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_merge_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_merge_normal_ROI.pdf\n",
      "Metric: METEOR\n",
      "Save path: plots/TC/METEOR_merge.avg.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_None_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_None_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_Pegasus-CD_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_Pegasus-CD_ppl.pdf\n",
      "TO REMOVE: ['None_normal', 'Pegasus-CD_normal']\n",
      "REMAINING: dict_keys(['None_ppl', 'Pegasus-CD_ppl'])\n",
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_merge_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_merge_ppl_ROI.pdf\n",
      "TO REMOVE: ['None_ppl', 'Pegasus-CD_ppl']\n",
      "REMAINING: dict_keys(['None_normal', 'Pegasus-CD_normal'])\n",
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_merge_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_merge_normal_ROI.pdf\n",
      "Metric: rougeL\n",
      "Save path: plots/TC/rougeL_merge.avg.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_None_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_None_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_Pegasus-CD_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_Pegasus-CD_ppl.pdf\n",
      "TO REMOVE: ['None_normal', 'Pegasus-CD_normal']\n",
      "REMAINING: dict_keys(['None_ppl', 'Pegasus-CD_ppl'])\n",
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_merge_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_merge_ppl_ROI.pdf\n",
      "TO REMOVE: ['None_ppl', 'Pegasus-CD_ppl']\n",
      "REMAINING: dict_keys(['None_normal', 'Pegasus-CD_normal'])\n",
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_merge_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_merge_normal_ROI.pdf\n",
      "Metric: Bert-f1\n",
      "Save path: plots/TC/Bert-f1_merge.avg.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_None_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_None_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_Pegasus-CD_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_Pegasus-CD_ppl.pdf\n",
      "TO REMOVE: ['None_normal', 'Pegasus-CD_normal']\n",
      "REMAINING: dict_keys(['None_ppl', 'Pegasus-CD_ppl'])\n",
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_merge_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_merge_ppl_ROI.pdf\n",
      "TO REMOVE: ['None_ppl', 'Pegasus-CD_ppl']\n",
      "REMAINING: dict_keys(['None_normal', 'Pegasus-CD_normal'])\n",
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_merge_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_merge_normal_ROI.pdf\n",
      "Metric: BLEURT\n",
      "Save path: plots/TC/BLEURT_merge.avg.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_None_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['Full' 'PegasusFT' 'BART' 'Recent-2' 'Recent-4' 'Recent-8' 'Recent-10'\n",
      " 'Semantic-2' 'Semantic-4' 'Semantic-8' 'Semantic-10' 'None_1'\n",
      " 'Pegasus-CD' 'Recent-1' 'Semantic-1' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_None_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL' 'ZS-ICL']\n",
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_Pegasus-CD_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n",
      "<ipython-input-21-64776c2fad98>:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  prompt_baseline_df['History Signal'] = \"None_0\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History: ['PegasusFT' 'BART' 'None_0']\n",
      "Prompt types: ['FS-ICL (ppl)' 'ZS-ICL (ppl)']\n",
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_Pegasus-CD_ppl.pdf\n",
      "TO REMOVE: ['None_normal', 'Pegasus-CD_normal']\n",
      "REMAINING: dict_keys(['None_ppl', 'Pegasus-CD_ppl'])\n",
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_merge_ppl.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_merge_ppl_ROI.pdf\n",
      "TO REMOVE: ['None_ppl', 'Pegasus-CD_ppl']\n",
      "REMAINING: dict_keys(['None_normal', 'Pegasus-CD_normal'])\n",
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_merge_normal.pdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bytestorm/miniconda3/lib/python3.9/site-packages/pandas/core/generic.py:4150: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_merge_normal_ROI.pdf\n",
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_merge.avg.pdf\n"
     ]
    }
   ],
   "source": [
    "prompt_type_groups = {\n",
    "    'normal': ['FS-ICL', 'ZS-ICL'],\n",
    "    'ppl': ['FS-ICL (ppl)', 'ZS-ICL (ppl)'],\n",
    "}\n",
    "metrics = ['BLEU', 'METEOR', 'rougeL', 'Bert-f1', 'BLEURT', 'DEB']\n",
    "\n",
    "# Start Processing\n",
    "filtered_df = df[df['History Signal'].notnull()]\n",
    "\n",
    "# Filter by metric\n",
    "for metric in metrics:\n",
    "    all_subframes = {}\n",
    "    if 'Persona Signal' in df:\n",
    "        # dataset_prefix = \"MSC\"\n",
    "        # Filter by persona\n",
    "        groups = filtered_df.groupby(['Persona Signal'])\n",
    "\n",
    "        # Find common baselines\n",
    "        # Persona = None, History = None, prompt_type=same\n",
    "        baseline_df = filtered_df[(filtered_df['Persona Signal'] == \"None\") & (filtered_df['History Signal'] == \"None\")]\n",
    "        for signal, sub_df in groups:\n",
    "            #if signal != \"None\":\n",
    "            #    _internal_baseline = sub_df[sub_df['History Signal'] == \"None\"]\n",
    "            #    if len(_internal_baseline) == 0:\n",
    "            #        sub_df_with_baseline = sub_df.append(baseline_df)\n",
    "            proc_sub_df_persona(sub_df, baseline_df, signal, prompt_type_groups, metric, all_subframes, dataset_prefix)\n",
    "            # if signal == \"None\":\n",
    "            #     raise \"Hell\"\n",
    "    else:\n",
    "        # dataset_prefix = \"TC\"\n",
    "        # Find common baselines\n",
    "        # Persona = None, History = None, prompt_type=same\n",
    "        baseline_df = filtered_df[(filtered_df['History Signal'] == \"None\")]\n",
    "\n",
    "        proc_sub_df_persona(filtered_df, baseline_df, \"NA\", prompt_type_groups, metric, all_subframes, dataset_prefix)\n",
    "\n",
    "    # raise Exception(\"Hell\")\n",
    "    # if dataset_prefix == \"MSC\":\n",
    "    if 'Persona Signal' in df:\n",
    "        df_merged_p, _, df_merged_p_ROI = merge_runs(all_subframes, metric, dataset_prefix, target=\"ppl\")\n",
    "        df_merged_n, _, df_merged_n_ROI = merge_runs(all_subframes, metric, dataset_prefix, target=\"normal\")\n",
    "        _ = merge_runs_pro(df_merged_n, df_merged_p, metric)\n",
    "    # Save all subframes to xlsx, one sheet per persona signal\n",
    "    # (signal, prompt_types) for all prompt type should be in same sheet for easy comparison\n",
    "\n",
    "    with pd.ExcelWriter(f'processed/{dataset_prefix}/{metric}.xlsx') as writer:\n",
    "        for key, value in all_subframes.items():\n",
    "            # remove multiindex \n",
    "            value.columns = [' '.join(col).strip() for col in value.columns.values]\n",
    "            value.to_excel(writer, sheet_name=key)\n",
    "\n",
    "        # if dataset_prefix == \"MSC\":\n",
    "        if \"Persona Signal\" in df:\n",
    "            # Save merged df\n",
    "            # fix column names\n",
    "            df_merged_p.columns = [' '.join(col).strip() for col in df_merged_p.columns.values]\n",
    "            df_merged_p.to_excel(writer, sheet_name=\"merged_ppl\")\n",
    "\n",
    "            # Save merged ROI\n",
    "            # fix column names\n",
    "            df_merged_p_ROI.columns = [' '.join(col).strip() for col in df_merged_p_ROI.columns.values]\n",
    "            df_merged_p_ROI.to_excel(writer, sheet_name=\"merged_ppl_ROI\")\n",
    "\n",
    "            # Save merged df\n",
    "            # fix column names\n",
    "            df_merged_n.columns = [' '.join(col).strip() for col in df_merged_n.columns.values]\n",
    "            df_merged_n.to_excel(writer, sheet_name=\"merged_normal\")\n",
    "\n",
    "            # Save merged ROI\n",
    "            # fix column names\n",
    "            df_merged_n_ROI.columns = [' '.join(col).strip() for col in df_merged_n_ROI.columns.values]\n",
    "            df_merged_n_ROI.to_excel(writer, sheet_name=\"merged_normal_ROI\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9fc52725",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: DEB\n",
      "Save path: plots/TC/DEB_merge.avg.pdf\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>(Model Avg., ZS+Manual)</th>\n",
       "      <th>(Model Avg., FS+Manual)</th>\n",
       "      <th>(Model Avg., ZS+PPL)</th>\n",
       "      <th>(Model Avg., FS+PPL)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>History Signal</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>None_0</th>\n",
       "      <td>0.876117</td>\n",
       "      <td>0.949751</td>\n",
       "      <td>0.876117</td>\n",
       "      <td>0.949751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No DH, no BI</th>\n",
       "      <td>0.876117</td>\n",
       "      <td>0.949751</td>\n",
       "      <td>0.804145</td>\n",
       "      <td>0.891926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BART-D</th>\n",
       "      <td>0.836667</td>\n",
       "      <td>0.750229</td>\n",
       "      <td>0.813121</td>\n",
       "      <td>0.663412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-CD</th>\n",
       "      <td>0.890246</td>\n",
       "      <td>0.758013</td>\n",
       "      <td>0.861943</td>\n",
       "      <td>0.595830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-DS</th>\n",
       "      <td>0.838892</td>\n",
       "      <td>0.750983</td>\n",
       "      <td>0.814472</td>\n",
       "      <td>0.667482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-DS + BI</th>\n",
       "      <td>0.857621</td>\n",
       "      <td>0.730233</td>\n",
       "      <td>0.850182</td>\n",
       "      <td>0.683912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-1</th>\n",
       "      <td>0.865879</td>\n",
       "      <td>0.812345</td>\n",
       "      <td>0.811634</td>\n",
       "      <td>0.851828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-2</th>\n",
       "      <td>0.878496</td>\n",
       "      <td>0.811646</td>\n",
       "      <td>0.849782</td>\n",
       "      <td>0.867925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-4</th>\n",
       "      <td>0.885858</td>\n",
       "      <td>0.805802</td>\n",
       "      <td>0.868605</td>\n",
       "      <td>0.878070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-8</th>\n",
       "      <td>0.892653</td>\n",
       "      <td>0.810486</td>\n",
       "      <td>0.878807</td>\n",
       "      <td>0.879559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-10</th>\n",
       "      <td>0.895740</td>\n",
       "      <td>0.814740</td>\n",
       "      <td>0.888595</td>\n",
       "      <td>0.880904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-1</th>\n",
       "      <td>0.863829</td>\n",
       "      <td>0.850983</td>\n",
       "      <td>0.858830</td>\n",
       "      <td>0.880555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-2</th>\n",
       "      <td>0.880362</td>\n",
       "      <td>0.859005</td>\n",
       "      <td>0.874379</td>\n",
       "      <td>0.894131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-4</th>\n",
       "      <td>0.913023</td>\n",
       "      <td>0.854586</td>\n",
       "      <td>0.899634</td>\n",
       "      <td>0.894472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-8</th>\n",
       "      <td>0.897860</td>\n",
       "      <td>0.855310</td>\n",
       "      <td>0.894338</td>\n",
       "      <td>0.903837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-10</th>\n",
       "      <td>0.900634</td>\n",
       "      <td>0.855169</td>\n",
       "      <td>0.902249</td>\n",
       "      <td>0.903280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Full</th>\n",
       "      <td>0.958955</td>\n",
       "      <td>0.923682</td>\n",
       "      <td>0.967303</td>\n",
       "      <td>0.961021</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 (Model Avg., ZS+Manual)  (Model Avg., FS+Manual)  \\\n",
       "History Signal                                                      \n",
       "None_0                          0.876117                 0.949751   \n",
       "No DH, no BI                    0.876117                 0.949751   \n",
       "BART-D                          0.836667                 0.750229   \n",
       "Pegasus-CD                      0.890246                 0.758013   \n",
       "Pegasus-DS                      0.838892                 0.750983   \n",
       "Pegasus-DS + BI                 0.857621                 0.730233   \n",
       "Recent-1                        0.865879                 0.812345   \n",
       "Recent-2                        0.878496                 0.811646   \n",
       "Recent-4                        0.885858                 0.805802   \n",
       "Recent-8                        0.892653                 0.810486   \n",
       "Recent-10                       0.895740                 0.814740   \n",
       "Semantic-1                      0.863829                 0.850983   \n",
       "Semantic-2                      0.880362                 0.859005   \n",
       "Semantic-4                      0.913023                 0.854586   \n",
       "Semantic-8                      0.897860                 0.855310   \n",
       "Semantic-10                     0.900634                 0.855169   \n",
       "Full                            0.958955                 0.923682   \n",
       "\n",
       "                 (Model Avg., ZS+PPL)  (Model Avg., FS+PPL)  \n",
       "History Signal                                               \n",
       "None_0                       0.876117              0.949751  \n",
       "No DH, no BI                 0.804145              0.891926  \n",
       "BART-D                       0.813121              0.663412  \n",
       "Pegasus-CD                   0.861943              0.595830  \n",
       "Pegasus-DS                   0.814472              0.667482  \n",
       "Pegasus-DS + BI              0.850182              0.683912  \n",
       "Recent-1                     0.811634              0.851828  \n",
       "Recent-2                     0.849782              0.867925  \n",
       "Recent-4                     0.868605              0.878070  \n",
       "Recent-8                     0.878807              0.879559  \n",
       "Recent-10                    0.888595              0.880904  \n",
       "Semantic-1                   0.858830              0.880555  \n",
       "Semantic-2                   0.874379              0.894131  \n",
       "Semantic-4                   0.899634              0.894472  \n",
       "Semantic-8                   0.894338              0.903837  \n",
       "Semantic-10                  0.902249              0.903280  \n",
       "Full                         0.967303              0.961021  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_runs_pro(df_merged_n, df_merged_p, metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7b88d679",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>flanT5-XL FS-ICL</th>\n",
       "      <th>flanT5-XL ZS-ICL</th>\n",
       "      <th>T0 FS-ICL</th>\n",
       "      <th>T0 ZS-ICL</th>\n",
       "      <th>Tk-Instruct FS-ICL</th>\n",
       "      <th>Tk-Instruct ZS-ICL</th>\n",
       "      <th>GPT-3 FS-ICL</th>\n",
       "      <th>GPT-3 ZS-ICL</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>History Signal</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>None_0</th>\n",
       "      <td>0.958204</td>\n",
       "      <td>0.871732</td>\n",
       "      <td>0.968648</td>\n",
       "      <td>0.935264</td>\n",
       "      <td>0.950834</td>\n",
       "      <td>0.837493</td>\n",
       "      <td>0.921320</td>\n",
       "      <td>0.859980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No DH, no BI</th>\n",
       "      <td>0.958204</td>\n",
       "      <td>0.871732</td>\n",
       "      <td>0.968648</td>\n",
       "      <td>0.935264</td>\n",
       "      <td>0.950834</td>\n",
       "      <td>0.837493</td>\n",
       "      <td>0.921320</td>\n",
       "      <td>0.859980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BART-D</th>\n",
       "      <td>0.748232</td>\n",
       "      <td>0.770405</td>\n",
       "      <td>0.708383</td>\n",
       "      <td>0.922652</td>\n",
       "      <td>0.721551</td>\n",
       "      <td>0.786473</td>\n",
       "      <td>0.822749</td>\n",
       "      <td>0.867140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-CD</th>\n",
       "      <td>0.823246</td>\n",
       "      <td>0.835588</td>\n",
       "      <td>0.597069</td>\n",
       "      <td>0.924177</td>\n",
       "      <td>0.700447</td>\n",
       "      <td>0.894951</td>\n",
       "      <td>0.911289</td>\n",
       "      <td>0.906269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-DS</th>\n",
       "      <td>0.751057</td>\n",
       "      <td>0.775066</td>\n",
       "      <td>0.705942</td>\n",
       "      <td>0.923344</td>\n",
       "      <td>0.722671</td>\n",
       "      <td>0.790558</td>\n",
       "      <td>0.824261</td>\n",
       "      <td>0.866598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-DS + BI</th>\n",
       "      <td>0.771571</td>\n",
       "      <td>0.830483</td>\n",
       "      <td>0.640035</td>\n",
       "      <td>0.931706</td>\n",
       "      <td>0.701979</td>\n",
       "      <td>0.799220</td>\n",
       "      <td>0.807347</td>\n",
       "      <td>0.869076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-1</th>\n",
       "      <td>0.567848</td>\n",
       "      <td>0.822037</td>\n",
       "      <td>0.920688</td>\n",
       "      <td>0.918384</td>\n",
       "      <td>0.876031</td>\n",
       "      <td>0.863552</td>\n",
       "      <td>0.884813</td>\n",
       "      <td>0.859543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-2</th>\n",
       "      <td>0.568873</td>\n",
       "      <td>0.826602</td>\n",
       "      <td>0.919616</td>\n",
       "      <td>0.920718</td>\n",
       "      <td>0.862600</td>\n",
       "      <td>0.888921</td>\n",
       "      <td>0.895496</td>\n",
       "      <td>0.877742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-4</th>\n",
       "      <td>0.567367</td>\n",
       "      <td>0.826899</td>\n",
       "      <td>0.919493</td>\n",
       "      <td>0.921281</td>\n",
       "      <td>0.832034</td>\n",
       "      <td>0.905339</td>\n",
       "      <td>0.904315</td>\n",
       "      <td>0.889914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-8</th>\n",
       "      <td>0.608545</td>\n",
       "      <td>0.831874</td>\n",
       "      <td>0.920155</td>\n",
       "      <td>0.926721</td>\n",
       "      <td>0.804754</td>\n",
       "      <td>0.916964</td>\n",
       "      <td>0.908491</td>\n",
       "      <td>0.895052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-10</th>\n",
       "      <td>0.624815</td>\n",
       "      <td>0.835678</td>\n",
       "      <td>0.924369</td>\n",
       "      <td>0.931744</td>\n",
       "      <td>0.801327</td>\n",
       "      <td>0.918507</td>\n",
       "      <td>0.908448</td>\n",
       "      <td>0.897032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-1</th>\n",
       "      <td>0.704281</td>\n",
       "      <td>0.821068</td>\n",
       "      <td>0.921225</td>\n",
       "      <td>0.914770</td>\n",
       "      <td>0.886418</td>\n",
       "      <td>0.857489</td>\n",
       "      <td>0.892009</td>\n",
       "      <td>0.861988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-2</th>\n",
       "      <td>0.704742</td>\n",
       "      <td>0.835119</td>\n",
       "      <td>0.922341</td>\n",
       "      <td>0.920385</td>\n",
       "      <td>0.907260</td>\n",
       "      <td>0.881778</td>\n",
       "      <td>0.901677</td>\n",
       "      <td>0.884165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-4</th>\n",
       "      <td>0.724145</td>\n",
       "      <td>0.902566</td>\n",
       "      <td>0.911312</td>\n",
       "      <td>0.930783</td>\n",
       "      <td>0.881724</td>\n",
       "      <td>0.925379</td>\n",
       "      <td>0.901166</td>\n",
       "      <td>0.893365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-8</th>\n",
       "      <td>0.677077</td>\n",
       "      <td>0.848952</td>\n",
       "      <td>0.923641</td>\n",
       "      <td>0.932063</td>\n",
       "      <td>0.911246</td>\n",
       "      <td>0.911140</td>\n",
       "      <td>0.909277</td>\n",
       "      <td>0.899285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-10</th>\n",
       "      <td>0.675763</td>\n",
       "      <td>0.852709</td>\n",
       "      <td>0.925407</td>\n",
       "      <td>0.936571</td>\n",
       "      <td>0.911231</td>\n",
       "      <td>0.913387</td>\n",
       "      <td>0.908276</td>\n",
       "      <td>0.899866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Full</th>\n",
       "      <td>0.824886</td>\n",
       "      <td>0.926513</td>\n",
       "      <td>0.979484</td>\n",
       "      <td>0.985691</td>\n",
       "      <td>0.938309</td>\n",
       "      <td>0.977289</td>\n",
       "      <td>0.952048</td>\n",
       "      <td>0.946325</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 flanT5-XL FS-ICL  flanT5-XL ZS-ICL  T0 FS-ICL  T0 ZS-ICL  \\\n",
       "History Signal                                                              \n",
       "None_0                   0.958204          0.871732   0.968648   0.935264   \n",
       "No DH, no BI             0.958204          0.871732   0.968648   0.935264   \n",
       "BART-D                   0.748232          0.770405   0.708383   0.922652   \n",
       "Pegasus-CD               0.823246          0.835588   0.597069   0.924177   \n",
       "Pegasus-DS               0.751057          0.775066   0.705942   0.923344   \n",
       "Pegasus-DS + BI          0.771571          0.830483   0.640035   0.931706   \n",
       "Recent-1                 0.567848          0.822037   0.920688   0.918384   \n",
       "Recent-2                 0.568873          0.826602   0.919616   0.920718   \n",
       "Recent-4                 0.567367          0.826899   0.919493   0.921281   \n",
       "Recent-8                 0.608545          0.831874   0.920155   0.926721   \n",
       "Recent-10                0.624815          0.835678   0.924369   0.931744   \n",
       "Semantic-1               0.704281          0.821068   0.921225   0.914770   \n",
       "Semantic-2               0.704742          0.835119   0.922341   0.920385   \n",
       "Semantic-4               0.724145          0.902566   0.911312   0.930783   \n",
       "Semantic-8               0.677077          0.848952   0.923641   0.932063   \n",
       "Semantic-10              0.675763          0.852709   0.925407   0.936571   \n",
       "Full                     0.824886          0.926513   0.979484   0.985691   \n",
       "\n",
       "                 Tk-Instruct FS-ICL  Tk-Instruct ZS-ICL  GPT-3 FS-ICL  \\\n",
       "History Signal                                                          \n",
       "None_0                     0.950834            0.837493      0.921320   \n",
       "No DH, no BI               0.950834            0.837493      0.921320   \n",
       "BART-D                     0.721551            0.786473      0.822749   \n",
       "Pegasus-CD                 0.700447            0.894951      0.911289   \n",
       "Pegasus-DS                 0.722671            0.790558      0.824261   \n",
       "Pegasus-DS + BI            0.701979            0.799220      0.807347   \n",
       "Recent-1                   0.876031            0.863552      0.884813   \n",
       "Recent-2                   0.862600            0.888921      0.895496   \n",
       "Recent-4                   0.832034            0.905339      0.904315   \n",
       "Recent-8                   0.804754            0.916964      0.908491   \n",
       "Recent-10                  0.801327            0.918507      0.908448   \n",
       "Semantic-1                 0.886418            0.857489      0.892009   \n",
       "Semantic-2                 0.907260            0.881778      0.901677   \n",
       "Semantic-4                 0.881724            0.925379      0.901166   \n",
       "Semantic-8                 0.911246            0.911140      0.909277   \n",
       "Semantic-10                0.911231            0.913387      0.908276   \n",
       "Full                       0.938309            0.977289      0.952048   \n",
       "\n",
       "                 GPT-3 ZS-ICL  \n",
       "History Signal                 \n",
       "None_0               0.859980  \n",
       "No DH, no BI         0.859980  \n",
       "BART-D               0.867140  \n",
       "Pegasus-CD           0.906269  \n",
       "Pegasus-DS           0.866598  \n",
       "Pegasus-DS + BI      0.869076  \n",
       "Recent-1             0.859543  \n",
       "Recent-2             0.877742  \n",
       "Recent-4             0.889914  \n",
       "Recent-8             0.895052  \n",
       "Recent-10            0.897032  \n",
       "Semantic-1           0.861988  \n",
       "Semantic-2           0.884165  \n",
       "Semantic-4           0.893365  \n",
       "Semantic-8           0.899285  \n",
       "Semantic-10          0.899866  \n",
       "Full                 0.946325  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged_n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "daf1632e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>flanT5-XL FS-ICL</th>\n",
       "      <th>flanT5-XL ZS-ICL</th>\n",
       "      <th>T0 FS-ICL</th>\n",
       "      <th>T0 ZS-ICL</th>\n",
       "      <th>Tk-Instruct FS-ICL</th>\n",
       "      <th>Tk-Instruct ZS-ICL</th>\n",
       "      <th>GPT-3 FS-ICL</th>\n",
       "      <th>GPT-3 ZS-ICL</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>History Signal</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>None_0</th>\n",
       "      <td>0.958204</td>\n",
       "      <td>0.871732</td>\n",
       "      <td>0.968648</td>\n",
       "      <td>0.935264</td>\n",
       "      <td>0.950834</td>\n",
       "      <td>0.837493</td>\n",
       "      <td>0.92132</td>\n",
       "      <td>0.85998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No DH, no BI</th>\n",
       "      <td>0.835805</td>\n",
       "      <td>0.745327</td>\n",
       "      <td>0.947230</td>\n",
       "      <td>0.844509</td>\n",
       "      <td>0.892743</td>\n",
       "      <td>0.822598</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BART-D</th>\n",
       "      <td>0.733770</td>\n",
       "      <td>0.768278</td>\n",
       "      <td>0.485161</td>\n",
       "      <td>0.870441</td>\n",
       "      <td>0.771304</td>\n",
       "      <td>0.800645</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-CD</th>\n",
       "      <td>0.802168</td>\n",
       "      <td>0.841282</td>\n",
       "      <td>0.333552</td>\n",
       "      <td>0.889625</td>\n",
       "      <td>0.651769</td>\n",
       "      <td>0.854922</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-DS</th>\n",
       "      <td>0.739330</td>\n",
       "      <td>0.770427</td>\n",
       "      <td>0.494839</td>\n",
       "      <td>0.871279</td>\n",
       "      <td>0.768276</td>\n",
       "      <td>0.801710</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pegasus-DS + BI</th>\n",
       "      <td>0.759812</td>\n",
       "      <td>0.792150</td>\n",
       "      <td>0.628158</td>\n",
       "      <td>0.914411</td>\n",
       "      <td>0.663767</td>\n",
       "      <td>0.843986</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-1</th>\n",
       "      <td>0.815727</td>\n",
       "      <td>0.783195</td>\n",
       "      <td>0.889686</td>\n",
       "      <td>0.874325</td>\n",
       "      <td>0.850071</td>\n",
       "      <td>0.777381</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-2</th>\n",
       "      <td>0.834819</td>\n",
       "      <td>0.821352</td>\n",
       "      <td>0.893178</td>\n",
       "      <td>0.889057</td>\n",
       "      <td>0.875777</td>\n",
       "      <td>0.838937</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-4</th>\n",
       "      <td>0.847949</td>\n",
       "      <td>0.834340</td>\n",
       "      <td>0.895526</td>\n",
       "      <td>0.898493</td>\n",
       "      <td>0.890736</td>\n",
       "      <td>0.872981</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-8</th>\n",
       "      <td>0.835151</td>\n",
       "      <td>0.842755</td>\n",
       "      <td>0.900926</td>\n",
       "      <td>0.904858</td>\n",
       "      <td>0.902600</td>\n",
       "      <td>0.888808</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recent-10</th>\n",
       "      <td>0.834241</td>\n",
       "      <td>0.856619</td>\n",
       "      <td>0.905429</td>\n",
       "      <td>0.913987</td>\n",
       "      <td>0.903043</td>\n",
       "      <td>0.895179</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-1</th>\n",
       "      <td>0.863107</td>\n",
       "      <td>0.833928</td>\n",
       "      <td>0.897214</td>\n",
       "      <td>0.885895</td>\n",
       "      <td>0.881344</td>\n",
       "      <td>0.856667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-2</th>\n",
       "      <td>0.880653</td>\n",
       "      <td>0.849786</td>\n",
       "      <td>0.902268</td>\n",
       "      <td>0.903402</td>\n",
       "      <td>0.899472</td>\n",
       "      <td>0.869948</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-4</th>\n",
       "      <td>0.897276</td>\n",
       "      <td>0.871512</td>\n",
       "      <td>0.885003</td>\n",
       "      <td>0.917501</td>\n",
       "      <td>0.901137</td>\n",
       "      <td>0.909890</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-8</th>\n",
       "      <td>0.885293</td>\n",
       "      <td>0.864899</td>\n",
       "      <td>0.908675</td>\n",
       "      <td>0.920425</td>\n",
       "      <td>0.917543</td>\n",
       "      <td>0.897690</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic-10</th>\n",
       "      <td>0.879943</td>\n",
       "      <td>0.875884</td>\n",
       "      <td>0.912709</td>\n",
       "      <td>0.927113</td>\n",
       "      <td>0.917188</td>\n",
       "      <td>0.903751</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Full</th>\n",
       "      <td>0.968759</td>\n",
       "      <td>0.940215</td>\n",
       "      <td>0.943559</td>\n",
       "      <td>0.985073</td>\n",
       "      <td>0.970744</td>\n",
       "      <td>0.976622</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 flanT5-XL FS-ICL  flanT5-XL ZS-ICL  T0 FS-ICL  T0 ZS-ICL  \\\n",
       "History Signal                                                              \n",
       "None_0                   0.958204          0.871732   0.968648   0.935264   \n",
       "No DH, no BI             0.835805          0.745327   0.947230   0.844509   \n",
       "BART-D                   0.733770          0.768278   0.485161   0.870441   \n",
       "Pegasus-CD               0.802168          0.841282   0.333552   0.889625   \n",
       "Pegasus-DS               0.739330          0.770427   0.494839   0.871279   \n",
       "Pegasus-DS + BI          0.759812          0.792150   0.628158   0.914411   \n",
       "Recent-1                 0.815727          0.783195   0.889686   0.874325   \n",
       "Recent-2                 0.834819          0.821352   0.893178   0.889057   \n",
       "Recent-4                 0.847949          0.834340   0.895526   0.898493   \n",
       "Recent-8                 0.835151          0.842755   0.900926   0.904858   \n",
       "Recent-10                0.834241          0.856619   0.905429   0.913987   \n",
       "Semantic-1               0.863107          0.833928   0.897214   0.885895   \n",
       "Semantic-2               0.880653          0.849786   0.902268   0.903402   \n",
       "Semantic-4               0.897276          0.871512   0.885003   0.917501   \n",
       "Semantic-8               0.885293          0.864899   0.908675   0.920425   \n",
       "Semantic-10              0.879943          0.875884   0.912709   0.927113   \n",
       "Full                     0.968759          0.940215   0.943559   0.985073   \n",
       "\n",
       "                 Tk-Instruct FS-ICL  Tk-Instruct ZS-ICL  GPT-3 FS-ICL  \\\n",
       "History Signal                                                          \n",
       "None_0                     0.950834            0.837493       0.92132   \n",
       "No DH, no BI               0.892743            0.822598           NaN   \n",
       "BART-D                     0.771304            0.800645           NaN   \n",
       "Pegasus-CD                 0.651769            0.854922           NaN   \n",
       "Pegasus-DS                 0.768276            0.801710           NaN   \n",
       "Pegasus-DS + BI            0.663767            0.843986           NaN   \n",
       "Recent-1                   0.850071            0.777381           NaN   \n",
       "Recent-2                   0.875777            0.838937           NaN   \n",
       "Recent-4                   0.890736            0.872981           NaN   \n",
       "Recent-8                   0.902600            0.888808           NaN   \n",
       "Recent-10                  0.903043            0.895179           NaN   \n",
       "Semantic-1                 0.881344            0.856667           NaN   \n",
       "Semantic-2                 0.899472            0.869948           NaN   \n",
       "Semantic-4                 0.901137            0.909890           NaN   \n",
       "Semantic-8                 0.917543            0.897690           NaN   \n",
       "Semantic-10                0.917188            0.903751           NaN   \n",
       "Full                       0.970744            0.976622           NaN   \n",
       "\n",
       "                 GPT-3 ZS-ICL  \n",
       "History Signal                 \n",
       "None_0                0.85998  \n",
       "No DH, no BI              NaN  \n",
       "BART-D                    NaN  \n",
       "Pegasus-CD                NaN  \n",
       "Pegasus-DS                NaN  \n",
       "Pegasus-DS + BI           NaN  \n",
       "Recent-1                  NaN  \n",
       "Recent-2                  NaN  \n",
       "Recent-4                  NaN  \n",
       "Recent-8                  NaN  \n",
       "Recent-10                 NaN  \n",
       "Semantic-1                NaN  \n",
       "Semantic-2                NaN  \n",
       "Semantic-4                NaN  \n",
       "Semantic-8                NaN  \n",
       "Semantic-10               NaN  \n",
       "Full                      NaN  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5396cba4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>flanT5-XL FS-ICL (ppl)</th>\n",
       "      <th>flanT5-XL ZS-ICL (ppl)</th>\n",
       "      <th>T0 FS-ICL (ppl)</th>\n",
       "      <th>T0 ZS-ICL (ppl)</th>\n",
       "      <th>Tk-Instruct FS-ICL (ppl)</th>\n",
       "      <th>Tk-Instruct ZS-ICL (ppl)</th>\n",
       "      <th>GPT-3 FS-ICL (ppl)</th>\n",
       "      <th>GPT-3 ZS-ICL (ppl)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>History Signal</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>None_0</th>\n",
       "      <td>0.958204</td>\n",
       "      <td>0.871732</td>\n",
       "      <td>0.968648</td>\n",
       "      <td>0.935264</td>\n",
       "      <td>0.950834</td>\n",
       "      <td>0.837493</td>\n",
       "      <td>0.92132</td>\n",
       "      <td>0.85998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BART</th>\n",
       "      <td>0.759812</td>\n",
       "      <td>0.792150</td>\n",
       "      <td>0.628158</td>\n",
       "      <td>0.914411</td>\n",
       "      <td>0.663767</td>\n",
       "      <td>0.843986</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PegasusFT</th>\n",
       "      <td>0.759812</td>\n",
       "      <td>0.792150</td>\n",
       "      <td>0.628158</td>\n",
       "      <td>0.914411</td>\n",
       "      <td>0.663767</td>\n",
       "      <td>0.843986</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                flanT5-XL FS-ICL (ppl)  flanT5-XL ZS-ICL (ppl)  \\\n",
       "History Signal                                                   \n",
       "None_0                        0.958204                0.871732   \n",
       "BART                          0.759812                0.792150   \n",
       "PegasusFT                     0.759812                0.792150   \n",
       "\n",
       "                T0 FS-ICL (ppl)  T0 ZS-ICL (ppl)  Tk-Instruct FS-ICL (ppl)  \\\n",
       "History Signal                                                               \n",
       "None_0                 0.968648         0.935264                  0.950834   \n",
       "BART                   0.628158         0.914411                  0.663767   \n",
       "PegasusFT              0.628158         0.914411                  0.663767   \n",
       "\n",
       "                Tk-Instruct ZS-ICL (ppl)  GPT-3 FS-ICL (ppl)  \\\n",
       "History Signal                                                 \n",
       "None_0                          0.837493             0.92132   \n",
       "BART                            0.843986                 NaN   \n",
       "PegasusFT                       0.843986                 NaN   \n",
       "\n",
       "                GPT-3 ZS-ICL (ppl)  \n",
       "History Signal                      \n",
       "None_0                     0.85998  \n",
       "BART                           NaN  \n",
       "PegasusFT                      NaN  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_subframes[\"Pegasus-CD_ppl\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "afa25b28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>flanT5-XL FS-ICL</th>\n",
       "      <th>flanT5-XL ZS-ICL</th>\n",
       "      <th>T0 FS-ICL</th>\n",
       "      <th>T0 ZS-ICL</th>\n",
       "      <th>Tk-Instruct FS-ICL</th>\n",
       "      <th>Tk-Instruct ZS-ICL</th>\n",
       "      <th>GPT-3 FS-ICL</th>\n",
       "      <th>GPT-3 ZS-ICL</th>\n",
       "      <th>flanT5-XL FS-ICL (ppl)</th>\n",
       "      <th>flanT5-XL ZS-ICL (ppl)</th>\n",
       "      <th>T0 FS-ICL (ppl)</th>\n",
       "      <th>T0 ZS-ICL (ppl)</th>\n",
       "      <th>Tk-Instruct FS-ICL (ppl)</th>\n",
       "      <th>Tk-Instruct ZS-ICL (ppl)</th>\n",
       "      <th>GPT-3 FS-ICL (ppl)</th>\n",
       "      <th>GPT-3 ZS-ICL (ppl)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>History Signal</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>None_1</th>\n",
       "      <td>0.958204</td>\n",
       "      <td>0.871732</td>\n",
       "      <td>0.968648</td>\n",
       "      <td>0.935264</td>\n",
       "      <td>0.950834</td>\n",
       "      <td>0.837493</td>\n",
       "      <td>0.92132</td>\n",
       "      <td>0.85998</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>None_1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.835805</td>\n",
       "      <td>0.745327</td>\n",
       "      <td>0.94723</td>\n",
       "      <td>0.844509</td>\n",
       "      <td>0.892743</td>\n",
       "      <td>0.822598</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                flanT5-XL FS-ICL  flanT5-XL ZS-ICL  T0 FS-ICL  T0 ZS-ICL  \\\n",
       "History Signal                                                             \n",
       "None_1                  0.958204          0.871732   0.968648   0.935264   \n",
       "None_1                       NaN               NaN        NaN        NaN   \n",
       "\n",
       "                Tk-Instruct FS-ICL  Tk-Instruct ZS-ICL  GPT-3 FS-ICL  \\\n",
       "History Signal                                                         \n",
       "None_1                    0.950834            0.837493       0.92132   \n",
       "None_1                         NaN                 NaN           NaN   \n",
       "\n",
       "                GPT-3 ZS-ICL  flanT5-XL FS-ICL (ppl)  flanT5-XL ZS-ICL (ppl)  \\\n",
       "History Signal                                                                 \n",
       "None_1               0.85998                     NaN                     NaN   \n",
       "None_1                   NaN                0.835805                0.745327   \n",
       "\n",
       "                T0 FS-ICL (ppl)  T0 ZS-ICL (ppl)  Tk-Instruct FS-ICL (ppl)  \\\n",
       "History Signal                                                               \n",
       "None_1                      NaN              NaN                       NaN   \n",
       "None_1                  0.94723         0.844509                  0.892743   \n",
       "\n",
       "                Tk-Instruct ZS-ICL (ppl)  GPT-3 FS-ICL (ppl)  \\\n",
       "History Signal                                                 \n",
       "None_1                               NaN                 NaN   \n",
       "None_1                          0.822598                 NaN   \n",
       "\n",
       "                GPT-3 ZS-ICL (ppl)  \n",
       "History Signal                      \n",
       "None_1                         NaN  \n",
       "None_1                         NaN  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# assuming all_subframes is a dictionary containing the subframes\n",
    "none_normal = all_subframes['None_normal'].loc[['None_1']]\n",
    "none_ppl = all_subframes['None_ppl'].loc[['None_1']]\n",
    "\n",
    "# display the subframes side by side\n",
    "pd.concat([none_normal, none_ppl], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ed9eb0eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "History Signal  Model        Prompt Type \n",
       "BART            GPT-3        FS-ICL          1\n",
       "                             ZS-ICL          1\n",
       "PegasusFT       flanT5-XL    ZS-ICL          1\n",
       "                             FS-ICL (ppl)    1\n",
       "                             FS-ICL          1\n",
       "                Tk-Instruct  ZS-ICL (ppl)    1\n",
       "                             ZS-ICL          1\n",
       "                             FS-ICL (ppl)    1\n",
       "                             FS-ICL          1\n",
       "                T0           ZS-ICL (ppl)    1\n",
       "                             ZS-ICL          1\n",
       "                             FS-ICL (ppl)    1\n",
       "                             FS-ICL          1\n",
       "                GPT-3        ZS-ICL          1\n",
       "                             FS-ICL          1\n",
       "BART            flanT5-XL    ZS-ICL (ppl)    1\n",
       "                             ZS-ICL          1\n",
       "                             FS-ICL (ppl)    1\n",
       "                             FS-ICL          1\n",
       "                Tk-Instruct  ZS-ICL (ppl)    1\n",
       "                             ZS-ICL          1\n",
       "                             FS-ICL (ppl)    1\n",
       "                             FS-ICL          1\n",
       "                T0           ZS-ICL (ppl)    1\n",
       "                             ZS-ICL          1\n",
       "                             FS-ICL (ppl)    1\n",
       "                             FS-ICL          1\n",
       "PegasusFT       flanT5-XL    ZS-ICL (ppl)    1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df[[\"History Signal\", \"Model\", \"Prompt Type\"]].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "de9eb7ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Pegasus-CD'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "signal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c6d971af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    _       _ _           _ \n",
      "   / \\   __| (_) ___  ___| |\n",
      "  / _ \\ / _` | |/ _ \\/ __| |\n",
      " / ___ \\ (_| | | (_) \\__ \\_|\n",
      "/_/   \\_\\__,_|_|\\___/|___(_)\n",
      "                            \n"
     ]
    }
   ],
   "source": [
    "%%bash \n",
    "    #!/bin/bash\n",
    "    # if figlet is installed\n",
    "    if [ -x \"$(command -v figlet)\" ]; then\n",
    "        figlet \"Adios!\"\n",
    "    fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caa83d2a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "6d3b0ed3f3b8d2534ba0b286c748703f0af5e95709f70acd7080da9e41254233"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}