SED EVAL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-27T05:20:37.801253Z",
     "start_time": "2021-03-27T05:20:37.718605Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from ipywidgets import interact, interactive, fixed, interact_manual,widgets\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import datatool.seddata\n",
    "import general.utils\n",
    "import result_analyse.kfold_analyse as an\n",
    "import metric.Metrics\n",
    "import result_analyse.visualisation as vs\n",
    "from SED.evaluation_measures import psds_score, compute_psds_from_operating_points, compute_metrics\n",
    "import SED.evaluation_measures \n",
    "import psds_eval\n",
    "\n",
    "rootFolder='/workspace/sed2020/'   \n",
    "    \n",
    "@interact\n",
    "def result_selector(gtf=os.listdir(f'{rootFolder}/metadata/')):\n",
    "    typ=gtf.split('.')[0]\n",
    "    gtf=f'{rootFolder}/metadata/{gtf}'\n",
    "    groundtruth=pd.read_csv(gtf,delimiter='\\t')\n",
    "    groundtruth_dataset=datatool.seddata.SED(gtf,typ,None)\n",
    "    \n",
    "    groundtruth_dataset.load()\n",
    "    meta_dur_df=pd.DataFrame(columns=['filename','duration'])\n",
    "    meta_dur_df['filename']=groundtruth['filename']\n",
    "    meta_dur_df['duration']=10\n",
    "    @interact\n",
    "    def result_selector(team=sorted(os.listdir(f'{rootFolder}/submissions/'))):\n",
    "        @interact\n",
    "        def result_selector(code=sorted([x for x in os.listdir(f'{rootFolder}/submissions/{team}') if os.path.isfile(f'{rootFolder}/submissions/{team}/{x}/{typ}/{x}.output.tsv')]) ):\n",
    "            title=code.replace('_task4','')\n",
    "            base_prediction_path=f'{rootFolder}/submissions/{team}/{code}/{typ}/'\n",
    "            pef = f'{base_prediction_path}/{code}.output.tsv'\n",
    "            psdsf = f'{base_prediction_path}/{code}.output_PSDS'            \n",
    "            print(gtf,pef,psdsf)\n",
    "#             get_single_result(pef,groundtruth,meta_dur_df,psdsf, groundtruth_dataset)\n",
    "            return\n",
    "            computeGem(pef, groundtruth_dataset)\n",
    "            \n",
    "            \n",
    "            # Evaluate a single prediction\n",
    "            single_predictions = pd.read_csv(pef, sep=\"\\t\")\n",
    "            compute_metrics(single_predictions, groundtruth,meta_dur_df)\n",
    "            \n",
    "            \n",
    "\n",
    "            # Evaluate predictions with multiple thresholds (better). Need a list of predictions.\n",
    "            prediction_list_path = glob.glob(os.path.join(psdsf, \"*.tsv\"))\n",
    "            list_predictions = []\n",
    "            for fname in prediction_list_path:\n",
    "                pred_df = pd.read_csv(fname, sep=\"\\t\")\n",
    "                list_predictions.append(pred_df)\n",
    "            psds = compute_psds_from_operating_points(list_predictions, groundtruth, meta_dur_df)\n",
    "            psds_score(psds, filename_roc_curves=os.path.join(base_prediction_path, \"figures/psds_roc.png\"))\n",
    "\n",
    "    \n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-07T13:14:06.734895Z",
     "start_time": "2021-03-07T13:14:06.726353Z"
    }
   },
   "outputs": [],
   "source": [
    "from ipywidgets import interact, interactive, fixed, interact_manual,widgets\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import datatool.seddata\n",
    "import general.utils\n",
    "import result_analyse.kfold_analyse as an\n",
    "import metric.Metrics\n",
    "import result_analyse.visualisation as vs\n",
    "from SED.evaluation_measures import psds_score, compute_psds_from_operating_points, compute_metrics\n",
    "import SED.evaluation_measures \n",
    "import psds_eval\n",
    "rootFolder='/workspace/sed2020/'\n",
    "\n",
    "\n",
    "# res1=get_single_result('/workspace/sed2020//metadata/public.tsv','/workspace/sed2020//submissions/CTK_NU/CTK_NU_task4_SED_1/public//CTK_NU_task4_SED_1.output.tsv','/workspace/sed2020//submissions/CTK_NU/CTK_NU_task4_SED_1/public//CTK_NU_task4_SED_1.output_PSDS')\n",
    "# res2=get_single_result('/workspace/sed2020//metadata/public.tsv','/workspace/sed2020//submissions/CTK_NU/CTK_NU_task4_SED_3/public//CTK_NU_task4_SED_3.output.tsv','/workspace/sed2020//submissions/CTK_NU/CTK_NU_task4_SED_1/public//CTK_NU_task4_SED_3.output_PSDS')\n",
    "\n",
    "# total=pd.DataFrame({'res1':{c:res1[c].loc['macro-avg']['f1'] for c in res1.keys()},\n",
    "#                    'res2':{c:res2[c].loc['macro-avg']['f1'] for c in res2.keys()} }).T\n",
    "\n",
    "# print(total)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T09:17:23.316167Z",
     "start_time": "2021-03-26T09:17:23.311444Z"
    }
   },
   "source": [
    "# Run All submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-30T21:26:14.747865Z",
     "start_time": "2021-03-30T21:25:09.445756Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "#@interact\n",
    "#def result_selector(gtf=os.listdir(f'{rootFolder}/metadata/')):\n",
    "import SED.my_eval\n",
    "gtf='public.tsv'\n",
    "rootFolder='/workspace/sed2020/'\n",
    "typ=gtf.split('.')[0]\n",
    "gtf=f'{rootFolder}/metadata/{gtf}'\n",
    "# meta_dur_df=pd.DataFrame(columns=['filename','duration'])\n",
    "# meta_dur_df['filename']=groundtruth['filename']\n",
    "# meta_dur_df['duration']=10\n",
    "\n",
    "l=['Miyazaki_NU_task4_SED_1','Miyazaki_NU_task4_SED_3','Hao_CQU_task4_SED_4','Koh_NTHU_task4_SED_3','Liu_thinkit_task4_SED_3','Miyazaki_NU_task4_SED_2','Ebbers_UPB_task4_SED_1','Hao_CQU_task4_SED_2','CTK_NU_task4_SED_4','Yao_UESTC_task4_SED_3']\n",
    "# l=['Ebbers_UPB_task4_SED_1']\n",
    "total_dic={}\n",
    "for team in sorted(os.listdir(f'{rootFolder}/submissions/')):\n",
    "    print(f'analysing team {team}')\n",
    "    for code in sorted(os.listdir(f'{rootFolder}/submissions/{team}')):\n",
    "        if not(code in l):continue\n",
    "        print(f'    {code}')\n",
    "        \n",
    "        base_prediction_path=f'{rootFolder}/submissions/{team}/{code}/{typ}/'\n",
    "        pef = f'{base_prediction_path}/{code}.output.tsv'\n",
    "        if not(os.path.isfile(pef)):\n",
    "            all=[x for x in os.listdir(base_prediction_path) if '.output.tsv' in x]\n",
    "            if len(all)>0:\n",
    "                pef=f'{base_prediction_path}/{all[0]}'\n",
    "            else:\n",
    "                print(pef)\n",
    "                continue\n",
    "        title=code.replace('_task4','')\n",
    "        try:\n",
    "            res1=SED.my_eval.get_single_result(gtf,pef,debug=0)\n",
    "            total_dic[title]=res1\n",
    "#             print(res1['detection'])\n",
    "#             break\n",
    "        except Exception as e:\n",
    "            print('Error! submission is ignored',e)\n",
    "#         if('gem' in res1):\n",
    "#             for k in res1['gem'].keys():\n",
    "#                 total_dic[title][f'gem-{k}']=res1['gem'].loc['avg'][k]\n",
    "#     break\n",
    "\n",
    "fs={s:{c:total_dic[s][c].loc['macro-avg']['f1'] for c in total_dic[s]} for s in total_dic  }\n",
    "total=pd.DataFrame(fs).T\n",
    "total"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chart compare all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-31T11:53:31.772937Z",
     "start_time": "2021-03-31T11:53:31.345121Z"
    }
   },
   "outputs": [],
   "source": [
    "def aliplot(total):\n",
    "    from matplotlib.pylab import plt\n",
    "    w={'detection':1.,\n",
    "       'uniformity':1.,\n",
    "       'total duration':1.,\n",
    "       'relative duration':1.,\n",
    "#        'boundary onset':1.\n",
    "      }\n",
    "    wsum=sum(w.values())\n",
    "    for wi in w:\n",
    "        if('our' in total): total['our']+=w[wi]/wsum*total[wi]\n",
    "        else: total['our']=w[wi]/wsum*total[wi]\n",
    "#     total['tmp']=total['total duration']/total['detection']\n",
    "#     print(total[['tmp','relative duration']])\n",
    "#     print(total['gem'])\n",
    "    total=total.sort_values(['collar'])\n",
    "#     total=total[-10:]\n",
    "#     total=total[7:]\n",
    "    fig, axs = plt.subplots(1, 2, figsize=(8, len(total)/3), sharey=True)\n",
    "#     total=total.sort_values(['gem'])\n",
    "    import pandas\n",
    "    import matplotlib.pyplot as plt\n",
    "    import numpy as np\n",
    "    import itertools\n",
    "\n",
    "    # colorcycle = cycler(color=['blue', 'orange', 'green', 'magenta'])\n",
    "    # Or use the default color cycle:\n",
    "    # colorcycle = cycler(color=plt.rcParams['axes.prop_cycle'].by_key()['color'])\n",
    "\n",
    "\n",
    "\n",
    "    ind = np.arange(len(total))\n",
    "    def myplot(ax,colis):\n",
    "        cols=total.columns[colis]\n",
    "        width = .9/len(cols)\n",
    "        marker = itertools.cycle(( '^', 'o', 'D', 's','X', '*'))\n",
    "        i=0\n",
    "        ax.grid(True)\n",
    "        # fig, ax = plt.subplots(figsize=(5,10))\n",
    "        for i,x in enumerate(cols):\n",
    "            if x=='y':continue\n",
    "#             ax.plot(total[x],ind+.2 ,label=x,marker=next(marker),alpha=.75)\n",
    "            ax.scatter(total[x],ind+.2 ,label=x,marker=next(marker),alpha=.75)\n",
    "    #         ax.barh(ind+.4-i*width, total[x], width, label=x)#, color='red'\n",
    "\n",
    "        ax.set(yticks=ind + width, yticklabels=[f'S{len(total)-i} : {y.replace(\"_SED\",\"\")}' for i,y in enumerate(total.index)], ylim=[-.5, len(total)-.5])\n",
    "        \n",
    "        ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.25),\n",
    "                  fancybox=True, shadow=True, ncol=2)\n",
    "        ax.set_xlabel('F1')\n",
    "\n",
    "#     myplot(axs[0],np.append([0,1,len(total.columns)-1],range(2,len(total.columns)-5)))\n",
    "    myplot(axs[0],[0,1,len(total.columns)-1,2,4,5])\n",
    "    # myplot(axs[1],[2])\n",
    "#     myplot(axs[1],range(len(total.columns)-9,len(total.columns)-1))\n",
    "#     myplot(axs[1],[8,11,12,13])\n",
    "    myplot(axs[1],[8,11,12,13,14])\n",
    "    \n",
    "    # print(total)\n",
    "# \n",
    "    fig.tight_layout()\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "fs={s:{c:total_dic[s][c].loc['macro-avg']['f1'] for c in total_dic[s]} for s in total_dic  }\n",
    "total=pd.DataFrame(fs).T\n",
    "total\n",
    "aliplot(total)\n",
    "# total.sort_values(['events'],ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-29T09:03:24.613134Z",
     "start_time": "2021-03-29T09:03:24.468482Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "fs={s:{c:total_dic[s][c].loc['macro-avg']['f1'] for c in total_dic[s]} for s in total_dic  }\n",
    "total=pd.DataFrame(fs).T\n",
    "total\n",
    "# print([c for s in total_dic for c in total_dic[s]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-29T09:03:18.709794Z",
     "start_time": "2021-03-29T09:03:18.334886Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-29T09:11:06.715788Z",
     "start_time": "2021-03-29T09:11:06.700845Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print('a')\n",
    "# s='Miyazaki_NU_SED_1'\n",
    "s='CTK_NU_SED_1'\n",
    "# print( total_dic[s]['collar'])\n",
    "# print(total_dic[s]['collar'][['Ntp']])\n",
    "team={c:total_dic[s][c] for c in total_dic[s] }\n",
    "# print(team)\n",
    "# aliplot2(team)\n",
    "team['detection']\n",
    "team['detection'].T/team['detection'].sum(axis=1)\n",
    "(team['segment'].T/team['segment'].sum(axis=1)).T.sort_index()[['Ntp','Ntn','Nfp','Nfn']]\n",
    "team['segment'].sort_index()[['Ntp','Ntn','Nfp','Nfn']]\n",
    "# pd.concat(team, axis=1)\n",
    "# print(pd.DataFrame(team).T.round(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-28T23:45:09.590700Z",
     "start_time": "2021-03-28T23:45:09.584492Z"
    }
   },
   "outputs": [],
   "source": [
    "(team['total duration'].T/team['total duration'].sum(axis=1)).T.sort_index()[['Ntp','Ntn','Nfp','Nfn']]\n",
    "team['total duration'].sort_index()[['Ntp','Ntn','Nfp','Nfn']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2021-03-29T07:03:00.963Z"
    }
   },
   "outputs": [],
   "source": [
    "def aliplot2(team):\n",
    "    from matplotlib.pylab import plt\n",
    "    \n",
    "    allmetrics=list(team.keys())\n",
    "    metrics=[allmetrics[m] for m in [0,6,1,2,4,8,9,10,11,12,13,14]]\n",
    "#     total=total[-10:]\n",
    "    fig, axs = plt.subplots(1, len(metrics), figsize=(12, len(team[metrics[0]])/5), sharey=True)\n",
    "#     total=total.sort_values(['gem'])\n",
    "    import pandas\n",
    "    import matplotlib.pyplot as plt\n",
    "    import numpy as np\n",
    "    import itertools\n",
    "\n",
    "    # colorcycle = cycler(color=['blue', 'orange', 'green', 'magenta'])\n",
    "    # Or use the default color cycle:\n",
    "    # colorcycle = cycler(color=plt.rcParams['axes.prop_cycle'].by_key()['color'])\n",
    "    \n",
    "    removerows=['micro-avg','macro-avg']#,'Alarm_bell_ringing','Blender']\n",
    "\n",
    "\n",
    "    ind = np.arange(len(team['total duration'].drop(removerows)))\n",
    "    def myplot(ax,num,m):\n",
    "#         print(team[m])\n",
    "        \n",
    "        team_m=team[m][['Ntp','Nfn','Nfp']].drop(removerows,errors='ignore')\n",
    "        team_m=team_m.sort_index()\n",
    "        ind = np.arange(len(team_m))\n",
    "        cols=team_m\n",
    "        team_m=(team_m.T/team_m.sum(axis=1)).T\n",
    "        width = .8\n",
    "        marker = itertools.cycle(( '^', 'o', 'D', 's','X', '*'))\n",
    "        i=0\n",
    "#         ax.grid(True)\n",
    "        \n",
    "        # fig, ax = plt.subplots(figsize=(5,10))\n",
    "#         for i,x in enumerate(cols):\n",
    "#             if x=='y':continue\n",
    "#             print(team[m][x])\n",
    "#             ax.plot(team[m][x].T,ind+.2 ,label=x,marker=next(marker),alpha=.75)\n",
    "#             ax.scatter(total[x],ind+.2 ,label=x,marker=next(marker),alpha=.75)\n",
    "#             ax.barh(ind+.4-i*width, team[m][x], width, label=x)#, color='red'\n",
    "            \n",
    "        ax.barh(ind+.8, team_m['Ntp'], width, label='TP',color='g')#, color='red'\n",
    "        ax.barh(ind+.8, team_m['Nfn'], width,left=team_m['Ntp'], label='FN',color='#f6ff00',alpha=.25)#, color='red'\n",
    "        ax.barh(ind+.8, team_m['Nfp'], width,left=team_m[['Ntp','Nfn']].sum(axis=1), label='FP',color='r',alpha=1)#, color='red'\n",
    "        \n",
    "\n",
    "        ax.set(yticks=ind + width, yticklabels=[f'C{len(team_m)-i} : {y}' for i,y in enumerate(team_m.index)], ylim=[0.4, len(team_m)+.2],xlim=[0,1])\n",
    "        if num==4:\n",
    "            ax.legend(loc='upper center', bbox_to_anchor=(0, -0.35),fancybox=True, shadow=True, ncol=5)\n",
    "        ax.set_xlabel(m.replace(' ','\\n'))\n",
    "\n",
    "#     myplot(axs[0],np.append([0,1,len(total.columns)-1],range(2,len(total.columns)-5)))\n",
    "#     myplot(axs[0],[0,1,len(total.columns)-1,2,4,6])\n",
    "    # myplot(axs[1],[2])\n",
    "    ddd=pd.DataFrame(columns=['duration','events'])\n",
    "    gdurations=team['total duration'][['Ntp','Nfn']].drop(removerows).sum(axis=1).sort_index()\n",
    "    ddd['duration']=gdurations\n",
    "    print(gdurations)\n",
    "    gdp=gdurations/gdurations.sum()\n",
    "    pdurations=team['total duration'][['Ntp','Nfp']].drop(removerows).sum(axis=1).sort_index()\n",
    "    gevents=team['detection2'][['Ntp','Nfn']].drop(removerows).sum(axis=1).sort_index()\n",
    "    gep=gevents/gevents.sum()\n",
    "    ddd['events']=gevents\n",
    "    pevents=team['detection2'][['Ntp','Nfp']].drop(removerows).sum(axis=1).sort_index()\n",
    "#     ddd=ddd.astype(float)\n",
    "    ddd=ddd.applymap('{:,.0f}'.format)\n",
    "#     print(ddd)\n",
    "#     a=a.round()\n",
    "#     axs[0].barh(ind+1.1, gdp, .2, label='reference duration',color='g',alpha=.8)#, color='red'\n",
    "#     axs[0].barh(ind+.9, pdurations/pdurations.sum(), .2, color='b', label='prediction duration',alpha=1)#, color='red'\n",
    "#     axs[0].barh(ind+.7, gep, .2, label='reference count',color='y',alpha=.7)#, color='red'\n",
    "#     axs[0].barh(ind+.5, pevents/pevents.sum(), .2, label='prediction count',color='c',alpha=1)#, color='red'\n",
    "#     axs[0].legend(loc='upper center', bbox_to_anchor=(2, -0.25),fancybox=True, shadow=True, ncol=5)\n",
    "#     axs[0].set_xscale('log')\n",
    "#     the_table = axs[-1].table(\n",
    "#         cellText=ddd.values,\n",
    "#         loc='center',\n",
    "# #         colLabels=ddd.columns,\n",
    "#     )\n",
    "#     the_table.auto_set_font_size(False)\n",
    "#     the_table.set_fontsize(7)\n",
    "    \n",
    "\n",
    "    for i,m in enumerate(metrics):\n",
    "        myplot(axs[i],i,m)\n",
    "#     myplot(axs[1],range(len(total.columns)-8,len(total.columns)-1))\n",
    "    \n",
    "    # print(total)\n",
    "    plt.show()\n",
    "\n",
    "aliplot2(team)\n",
    "# total.sort_values(['events'],ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# New Chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-30T16:26:06.531219Z",
     "start_time": "2021-03-30T16:26:04.786571Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def aliplot3(team):\n",
    "    from matplotlib.pylab import plt\n",
    "    \n",
    "    allmetrics=list(team.keys())\n",
    "    removerows=['micro-avg','macro-avg']#,'Alarm_bell_ringing','Blender']\n",
    "    metrics=allmetrics\n",
    "#     total=total[-10:]\n",
    "    fig, axs = plt.subplots(1, len(metrics), figsize=(len(metrics), len(team[metrics[0]])/5), sharey=True)\n",
    "#     total=total.sort_values(['gem'])\n",
    "    import pandas\n",
    "    import matplotlib.pyplot as plt\n",
    "    import numpy as np\n",
    "    import itertools\n",
    "\n",
    "    # colorcycle = cycler(color=['blue', 'orange', 'green', 'magenta'])\n",
    "    # Or use the default color cycle:\n",
    "    # colorcycle = cycler(color=plt.rcParams['axes.prop_cycle'].by_key()['color'])\n",
    "    \n",
    "    \n",
    "\n",
    "    gtf='/workspace/sed2020/metadata/public.tsv'\n",
    "    gt = pd.read_csv(gtf, sep=\"\\t\")\n",
    "\n",
    "    gt['duration']=gt.offset-gt.onset\n",
    "    print(gt['duration'].mean())\n",
    "    info=gt.groupby('event_label')['duration'].agg(['mean','std','count'])\n",
    "    info=info.loc[metrics]\n",
    "    print(info)\n",
    "    maxd=(info['mean']+info['std']).max().max();\n",
    "    print(maxd)\n",
    "    gt['duration']=maxd\n",
    "    info_n=info/maxd\n",
    "\n",
    "    avg_e_dur=pd.Series({m:team[m].loc['total duration'][['Ntp','Nfn']].sum()/team[m].loc['detection'][['Ntp','Nfn']].sum() for m in metrics})\n",
    "    avg_e_dur=pd.Series({m:team[m].loc['total duration'][['Ntp','Nfn']].sum()/team[m].loc['detection'][['Ntp','Nfn']].sum() for m in metrics})\n",
    "    e_count=pd.Series({m:team[m].loc['detection'][['Ntp','Nfn']].sum() for m in metrics})\n",
    "    \n",
    "#   \n",
    "    avg_e_dur=avg_e_dur/avg_e_dur.max()\n",
    "    print(e_count.max())\n",
    "    e_count=e_count/e_count.max()\n",
    "    \n",
    "    ind = np.arange(len(team[list(team.keys())[0]]))\n",
    "    def myplot(ax,num,m):\n",
    "#         print(team[m])\n",
    "        \n",
    "        team_m=team[m][['Ntp','Nfn','Nfp']]#.iloc[[0,6,8,2,1,12,10,11,9,13,14,15]]\n",
    "        \n",
    "#         team_m=team_m.sort_index()\n",
    "        ind = np.arange(len(team_m))\n",
    "        cols=team_m\n",
    "        team_m=(team_m.T/team_m.sum(axis=1)).T\n",
    "        width = .8\n",
    "        marker = itertools.cycle(( '^', 'o', 'D', 's','X', '*'))\n",
    "        i=0\n",
    "\n",
    "        ax.barh(ind+.8, team_m['Ntp'], width, label='TP',color='g')#, color='red'\n",
    "        ax.barh(ind+.8, team_m['Nfn'], width,left=team_m['Ntp'], label='FN',color='#f6ff00',alpha=.25)#, color='red'\n",
    "        ax.barh(ind+.8, team_m['Nfp'], width,left=team_m[['Ntp','Nfn']].sum(axis=1), label='FP',color='r',alpha=1)#, color='red'\n",
    "        \n",
    "        \n",
    "        ax.barh(-.2,e_count[m],width,label='Count')\n",
    "#         ax.barh(-1,avg_e_dur[m],width,label='Duration(avg)')\n",
    "        ax.barh(-1,info_n.loc[m]['mean'],width,label='Duration (std)')\n",
    "        ax.errorbar( info_n.loc[m]['mean'],-1.1, xerr=info_n.loc[m]['std'],color='black',elinewidth=1, fmt='-.')\n",
    "        all_data=gt['duration'][gt['event_label']==m]\n",
    "        \n",
    "        ax.set(yticks=ind + width, yticklabels=[f'{y}' for i,y in enumerate(team_m.index)], ylim=[0.4-2, len(team_m)+.2],xlim=[0,1])\n",
    "        if num==2:\n",
    "            ax.legend(loc='upper center', bbox_to_anchor=(0, -0.45),fancybox=True, shadow=True, ncol=5)\n",
    "        ax.set_xlabel(m.replace(' ','\\n').replace('bell_','').replace('_','\\n'))\n",
    "        \n",
    "\n",
    "#         ax.set_xticks(np.arange(0, 1, .5))\n",
    "        ax.set_xticks(np.arange(0, 1, .25), minor=True)\n",
    "\n",
    "        ax.grid(True,axis='x',which='both')\n",
    "\n",
    "    for i,m in enumerate(metrics):\n",
    "        myplot(axs[i],i,m)\n",
    "#     myplot(axs[1],range(len(total.columns)-8,len(total.columns)-1))\n",
    "    \n",
    "    # print(total)\n",
    "    plt.show()\n",
    "\n",
    "for s in ['Miyazaki_NU_SED_1','CTK_NU_SED_4','Hao_CQU_SED_4']:#total_dic:\n",
    "    print(s)\n",
    "    metrics=[m for m in total_dic[s]]\n",
    "    metrics=['collar','psd d/gtc=0.8','psd d/gtc=0.5','psd d/gtc=0.1','detection','segment','total duration','uniformity','relative duration']\n",
    "    team={m:total_dic[s][m] for m in metrics }\n",
    "\n",
    "#     metrics=list(team.keys())\n",
    "    clas=[c for c in team[metrics[0]].index[:-2]]\n",
    "#     clas=[c for i,c in enumerate(clas) if i in [0,2,3,4,9]]\n",
    "    clas=['Dishes',  'Cat', 'Speech', 'Blender','Alarm_bell_ringing']\n",
    "    print(clas)\n",
    "    team3={c:pd.DataFrame({m: team[m].loc[c] for m in metrics}).T for c in clas}\n",
    "#     print(team3)\n",
    "    aliplot3(team3)\n",
    "# total.sort_values(['events'],ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-30T13:34:49.909570Z",
     "start_time": "2021-03-30T13:34:49.653710Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# example data\n",
    "x = np.arange(0.1, 4, 0.5)\n",
    "y = np.exp(-x)\n",
    "# example error bar values that vary with x-position\n",
    "error = 0.1 + 0.2 * x\n",
    "# error bar values w/ different -/+ errors\n",
    "lower_error = 0.4 * error\n",
    "upper_error = error\n",
    "asymmetric_error = [lower_error, upper_error]\n",
    "\n",
    "fig, (ax0, ax1) = plt.subplots(nrows=2, sharex=True)\n",
    "ax0.errorbar(x[0],-1, xerr=error[0], fmt='-o')\n",
    "ax0.set_title('variable, symmetric error')\n",
    "\n",
    "ax1.errorbar(x, y, xerr=asymmetric_error, fmt='o')\n",
    "ax1.set_title('variable, asymmetric error')\n",
    "ax1.set_yscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-29T07:30:37.553732Z",
     "start_time": "2021-03-29T07:30:37.544665Z"
    }
   },
   "outputs": [],
   "source": [
    "m='Blender'\n",
    "# print(team)\n",
    "# pd.DataFrame({m:team3[m].loc['total duration'][['Ntp','Nfn']].sum()})\n",
    "a={m:team3[m].loc['total duration'][['Ntp','Nfn']].sum()  for m in team3}\n",
    "a=pd.Series(a)/pd.Series(a).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-29T07:31:48.747477Z",
     "start_time": "2021-03-29T07:31:48.742881Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.Series(a)/a.drop(['macro-avg','micro-avg']).max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Maker\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-22T12:46:52.678198Z",
     "start_time": "2021-09-22T12:46:52.673354Z"
    }
   },
   "outputs": [],
   "source": [
    "def Convert2SED(g,p,duration,path=None):\n",
    "    \n",
    "    gdf=pd.DataFrame(g,columns=['onset','offset'])\n",
    "#     gdf=gdf/5\n",
    "    gdf['event_label']='Test'\n",
    "    gdf['filename']='Test'\n",
    "    gdf=gdf[['filename','onset','offset','event_label']]\n",
    "\n",
    "    pdf=pd.DataFrame(p,columns=['onset','offset'])\n",
    "#     pdf=pdf/6\n",
    "    pdf['event_label']='Test'\n",
    "    pdf['filename']='Test'\n",
    "    pdf=pdf[['filename','onset','offset','event_label']]\n",
    "    \n",
    "    meta_df=pd.DataFrame(columns=['filename','duration'])\n",
    "    meta_df['filename']=gdf['filename'].unique()\n",
    "#     meta_df['duration']=max(gdf['offset'].max(),pdf['offset'].max())\n",
    "    meta_df['duration']=duration\n",
    "    if path != None:\n",
    "        import os\n",
    "        from pathlib import Path\n",
    "        Path(path).mkdir(parents=True, exist_ok=True)\n",
    "        meta_df.to_csv(f'{path}/meta.tsv',sep='\\t',index=False)\n",
    "        pdf.to_csv(f'{path}/p.tsv',sep='\\t',index=False)\n",
    "        gdf.to_csv(f'{path}/g.tsv',sep='\\t',index=False)\n",
    "    return gdf,pdf,meta_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-30T22:19:25.410778Z",
     "start_time": "2021-03-30T22:19:25.405612Z"
    }
   },
   "outputs": [],
   "source": [
    "r[11:,:]+1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## all possible situation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-22T12:48:23.535066Z",
     "start_time": "2021-09-22T12:48:22.824103Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============== class=Test=========file=Test\n",
      "real= [[28.  30.5]\n",
      " [31.  31.5]\n",
      " [32.  33.5]]\n",
      "pred= [[27.5 28.5]\n",
      " [29.  29.5]\n",
      " [30.  32.5]\n",
      " [33.  34. ]]\n",
      "real_= [(0, 28.0), (30.5, 31.0), (31.5, 32.0), (33.5, 34.5)]\n",
      "pred_= [(0, 27.5), (28.5, 29.0), (29.5, 30.0), (32.5, 33.0), (34.0, 34.5)]\n",
      "r+: {0: {'p+': {0: [28.0, 28.5], 1: [29.0, 29.5], 2: [30.0, 30.5]}, 'p-': {1: [28.5, 29.0], 2: [29.5, 30.0]}}, 1: {'p+': {2: [31.0, 31.5]}, 'p-': {}}, 2: {'p+': {2: [32.0, 32.5], 3: [33.0, 33.5]}, 'p-': {3: [32.5, 33.0]}}, 3: {'p+': {}, 'p-': {}}}\n",
      "r-: {0: {'p+': {0: [27.5, 28.0]}, 'p-': {0: [0, 27.5]}}, 1: {'p+': {2: [30.5, 31.0]}, 'p-': {}}, 2: {'p+': {2: [31.5, 32.0]}, 'p-': {}}, 3: {'p+': {3: [33.5, 34.0]}, 'p-': {4: [34.0, 34.5]}}}\n",
      "p+: {0: {'r+': {0: [28.0, 28.5]}, 'r-': {0: [27.5, 28.0]}}, 1: {'r+': {0: [29.0, 29.5]}, 'r-': {}}, 2: {'r+': {0: [30.0, 30.5], 1: [31.0, 31.5], 2: [32.0, 32.5]}, 'r-': {1: [30.5, 31.0], 2: [31.5, 32.0]}}, 3: {'r+': {2: [33.0, 33.5]}, 'r-': {3: [33.5, 34.0]}}, 4: {'r+': {}, 'r-': {}}}\n",
      "p-: {0: {'r+': {}, 'r-': {0: [0, 27.5]}}, 1: {'r+': {0: [28.5, 29.0]}, 'r-': {}}, 2: {'r+': {0: [29.5, 30.0]}, 'r-': {}}, 3: {'r+': {2: [32.5, 33.0]}, 'r-': {}}, 4: {'r+': {}, 'r-': {3: [34.0, 34.5]}}}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2EAAABKCAYAAAAyj1xXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAR0UlEQVR4nO3deXhV1bnH8e+bhEGSMIcwCYShUAKE+blpQSggzuBQlTjgBFiLU6GXXoei5Xa4ep1q9bYPVR7tLYpay0WQItCLFVuqRAZlqCKIAoYEiIEwJjln3T9ypDxeEs9Ozjmbs/P7PE8ezj5n7Xe/Z61nnXMWe6+1zTmHiIiIiIiIJEaK3wmIiIiIiIg0JBqEiYiIiIiIJJAGYSIiIiIiIgmkQZiIiIiIiEgCaRAmIiIiIiKSQGnxCNq6bXN3dtd28QgtIiIiIuJJydFjfqeQcO2aneWpvNc6inf8RPD6Hrx6f932/c65rNO9FpdB2Nld27F0zSPxCC0iIiIi4slT6zb5nULC3T64n6fyXuso3vETwet78Kpzk8s+rem1mF6OaGaXmNncQwePxDKsiIiIiIhIYMR0EOacW+ycm9a8RXosw4qIiIiIiASGFuYQERERERFJoLjMCRMRkfpZ9cY6Hpj5LKFQmIKbx3H7v17hd0oiIuLBjjXbWPnE64RDjrwJQ8iffI6v+cyc9itWLi2kbVYL/rz+ya8t//pPF7L9bx/SrFU6U+bf8bXlP9+1n7tu+SX7i8swM6655Vym3HFJLFIH4FDxQZbMeZUjpYcxg7yJwxh2dX7M4tfm+PEKrhh7HxUnqghVhbjw8nx+OLugXjF1JkxE5AwTCoW4/665/PdrP2bVxidZ9NLbfLR1l99piYhIlMKhMMsfXcxVj01m6ot3sGXF++z/pMTXnK68fgy/Xzw76vL9LxrEVY9Pjrp8aloKsx+6kVUbf8Vrqx/i+d/8KabfXSmpKYy583ymvngn1//2Vta9+k7C6rRJk0a8/MYcVhQ+zhtrH+PN5et5750P6xVTC3OIiJxhNqzdRrceHejavT2NGzdi4lUjWL74Xb/TEhGRKBVt2U2rzm1o2ak1qY3S6DuuP9ve2uprTv8yMpeWrTKjLt9lUDeaNo9+CffsDq3pP6gHABmZZ9GrT2f27jngOc+aZLTNpH3vjgA0SW9Cm25ZlO87FLP4tTEz0jOq66KqMkRVZQgzq1fMrx2EmVnIzDaY2SYzW2xmLWsqq4U5RETqr+jzUjqc3fbkdvtObSiK4ReZiIjEV/m+Q2S2a3FyO7NdC8r3lfuYUWLt2lnCpo2fMGj4N+ISv6zoC0o+KqJjbue4xD+dUCjE+GE/IK/zjYwcm8fger63aM6EHXPODXTO9QNKgen1OqKIiIiIiATSkcPHmDbpIR585GYymzeLefyKoydYeM8Cxt59AU3Sm8Y8fk1SU1NZvvZx1u54hg2F2/jH5hpvARYVr5cjrgE61euIIiJSqw4dW1O0a//J7b17DtChUxsfMxIRES8ys5pTXnLw5HZ5yUEys6K/FDBZVVZWMe3qh7ls0jlceGnsF80IVYVYeO8Ccs8bQO/RuTGPH40WLdP51qh+vPnG+nrFiXoQZmapwFjgtRpen2ZmhWZWeGB/Yq7PFBEJoryhvfjk4yI++6SYiopKFr38NudePMzvtEREJEodvtmJ0l0HKPv8C0KVVWxZ+QE9R/bxO624cs7xw1ufpmefzky7e2Jc4i/92ULadM1ieMG3Yx6/Ngf2HeRgWfWaF8eOnWD1nzfSs3f9zktFs0T9WWa2geozYFuBFacr5JybC8wFyBvS09UrKxGRBiwtLZV/f2Iq1178E8KhMFffOJbefbv4nZaIiEQpJS2V8TMv5qW7n8eFwwy4eDBZ3bN9zWn69Y+y5q3NlO4/xNDuU5j540kU3DSuxvKLZr/MZ+s+4VjZUZ6e8J+MmDKGvAlDaiy/9m9beXX+m/Tp15Xxw34AwI/mXMfYC2rex4vd73/G5mUbyeqRzbzJTwMw6nvn0uNb8Zl3dqrivV/wg1ueJBQK48JhLv7utxl3Uf3+c9Scq328ZGaHnXMZZtYMeAN4xTlX680F8ob0dEvXPFKvxEREREREYuGpdZv8TiHhbh/cz1N5r3UU7/iJ4PU9eNW5yWXvOeeGnu61qC9HdM4dBe4EZpqZbvIsIiIiIiJSB54W5nDOrQfeB+p3i2gREREREZEG6mvPaDnnMr6yfUlNZc3sEuCSbj3axyA1ERERERGR4PG6RH2tdLNmERERERGR2sVlbperqqLyQGk8QouIiIhIA1f68ceeypes974oROuePT2V95pTvOMXVx71VN5rHcU7Ppx5deQ1n9rE9EyYiIiIiIiI1C6mgzAzu8TM5h46dCyWYUVERERERAIjPnPCmp8Vy7AiIiIiIiKBocsRRUREREREEkiDMBERERERkQTSnDAREREREZEE0pwwERERERGRBIrLfcLSqtLJ2j8sHqFFREREpIHbu3Sbp/JTyfV8jP7fucZT+Q9Wv3BGxa8ocp7Ke62jeMeHM6+Osq6J3fhGc8JEREREREQSSIMwERERERGRBIrLwhwHy8tjGVZERERERCQwohqEmVnIzDaY2SYze8XMmp2u3JcLc7TIzIxtliIiIiIiIgER7cIcx5xzAwHMbD7wPeCxeCUlItJQrHj7bWY99BChcJgbLr+cmbfc4ndKIiKBsmbHDp5YuZJQOMyEvDwm5+fXKc5ts2fzp7/8hazWrVm7cOHXlv/p66/zt+3badWsGfOnTKm17O69e5l6332UHDiAmXHTFVcw/brr6pRnrBQfOsScJUsoPXIEM2NiXh5XD/Nv4b3jJ05w3k03caKigqpQiEvHjeP+6dM9xzlRVcVt8+dTWVVFyDm+07s3U0eOrHNeoVCIkQUFdGzXjj889VTU+9VldcTVwIA67CciIqcIhULM+PnPeW3uXDplZ3NOQQEXjh7NN3v08Ds1EZFACIXDPLp8Ob+cNIl2mZnc/NxzjOzVi5y2bT3HunbCBG6dNImp990XVfmL+vfnyiFDmLNkydeWTUtN5RczZzKwb1/Kjxxh5KRJjMnP9/X7IDUlhTvHjKF3+/YcOXGCm557juE5OXWqu1ho0rgxrz/zDBnNmlFZWcm5N9zA+BEjGJ6X5ylO49RUnioooFnjxlSFQtz6+9+T3707/Tp1qlNe/zV/Pr1zcig/csTTfp7mhJlZGnAB8MFpXptmZoVmVrj/iy88JSEi0hAVbtpE9y5dyOncmcaNGvHd88/n9VWr/E5LRCQwthQV0blVKzq1bEmj1FTG9e3LW9u8LW//pRFDh9KqRYuoyw/q0oXmTZtGVbZ9VhYD+/YFIDM9nd45ORSVlNQpz1hpm5FB7/btAUhv0oRubdqwz8d1H8yMjGbVM6Iqq6qorKrCzOoUp1njxgBUhcNUhcN1igOwZ+9elr31FjdcfrnnfaMdhJ1lZhuAQuAz4NmvFnDOzXXODXXODW3bqpXnREREGprPi4vpnJ19crtTdjaf+/ylKyISJPvKy2l3yloF7TIzfR1IROPTPXvY+I9/MLR/f79TOamorIyPSkrI7djR1zxCoRD5V15JzujRjMnPZ9iAul2cFwqHmTxvHhc++STDu3Wr8/ua9fDD/HTGDFJSvK91GO0ex5xzAyN/dzjnKjwfSUREREREanT46FGunTGDh2bNonlGht/pAHC0ooJ7Fi7k7rFjSW/SxNdcUlNTWfPKK3y4YgWFmzaxuY5nNVNTUvjdzTezaPp0thQVsX3fPs8xvpwfOChyBtMr3SdMRMQnHbOz2V1cfHJ7T3ExHdu18zEjEZFgycrMpOSUM18l5eVknaGreFdWVnLtjBlcfdFFTBw3zu90AKgKhbh34ULOy81ldO/efqdzUsvmzTln2DBW/vWv9YqT2bQpg7t04e87dnje9+8bNrD0zTfpe/753DhrFn95911uueeeqPfXfcJERHwyJDeX7Z9+ys7du6morOQPy5Zx4ejRfqclIhIY3+zQgV2lpXxeVkZlKMTKLVsY2bOn32n9P845vv/AA/TOyeGOyZP9TgeozulnS5fStU0bCoYP9zsd9pWWUnboEADHjh/nf9es4Rs5OZ7jfHH0KOXHjwNwvLKStTt30rVNG89xfnLXXXy0ciVbli3juYcfZtTw4Tz7i19EvX9UqyM656I6H+qcWwwsHpybOzXqDEREGqi0tDQevfdeLr3tNkKhENdfeil9z8AfByIiySotJYWZ48dz90svEXaOiwcMoHtWVp1i3ThrFqsLCzlQVsY3xo3jvu9/v9YFGWYvWsS6zz6j7NgxJjz9NFNGjGBCDSv5rVm/nheXLCG3Vy/yr7wSgAfvvJPz6rF0en29v3s3yzZvpkdWFpPnzQPge6NG8S2fVmws3r+fafffTygUIhwOc/l553HBqFGe4xw4fJg5S5YQdg7nHGP69GGED9+95pyLedDBublu9YIFMY8rIiIiIvLBCy/E/Rj9r7nGU3mvOcU7fhCcaXXkNZ+MAQPec84NPd1rmhMmIiIiIiKSQBqEiYiIiIiIJJAW5hAREREREUmgmA7CnHOLnXPTWpyhS3+KiIiIiIj4LS4Lc5hZOfBhzAP/UwvgYBzjJ+IYyR6/LbA/jvEh+eso2eND/Ns52esoCJ9Fyd7GiThGssfX53Xw40Py9+Vkj5+IY6iN/T+G1/hdnXOnX47TRZZnjOUfUBiPuKfEnxvP+Ik4RgDix7WNA1JHSR0/Ee2c7HUUkM+ipG7jILRzsrdxQOooqeMnop2TvY4C8lmkNg7Qe0jWhTkWB+AYyR4/EZK9jpI9fiIkex0F4bMo3oJQR8kePxGSvY6SPX4iJHsdBeGzKN7UBgmMH6/LEQtdDWviSzCojRsGtXPwqY2DT23cMKidg09tHCzxOhM2N05x5cyhNm4Y1M7BpzYOPrVxw6B2Dj61cYDE5UyYiIiIiIiInF6yzgkTERERERFJShqEiYiIiIiIJFDMB2Fmdr6ZfWhmH5vZv8U6vvjPzHaa2QdmtsHMCv3OR+rPzOaZWYmZbTrludZmtsLMtkX+beVnjlJ/NbTzg2a2J9KfN5jZhX7mKPVjZmeb2Soz22Jmm83srsjz6s8BUUsbqy8HhJk1NbN3zWxjpI1/Enk+x8zeifzGfsnMGvudq9RdTOeEmVkq8BFwLrAbWAsUOOe2xOwg4jsz2wkMdc7F++afkiBmdg5wGPidc65f5LmHgVLn3H9E/kOllXPuR37mKfVTQzs/CBx2zj3iZ24SG2bWAejgnFtnZpnAe8ClwI2oPwdCLW18FerLgWBmBqQ75w6bWSPgbeAuYAbwR+fcAjP7DbDROfdrP3OVuov1mbDhwMfOuR3OuQpgATAxxscQkRhzzr0FlH7l6YnA85HHz1P9JS9JrIZ2lgBxzhU559ZFHpcDW4FOqD8HRi1tLAHhqh2ObDaK/DlgDPCHyPPqx0ku1oOwTsCuU7Z3ow+GIHLAcjN7z8ym+Z2MxE22c64o8ngvkO1nMhJXt5vZ+5HLFXWZWkCYWTdgEPAO6s+B9JU2BvXlwDCzVDPbAJQAK4DtQJlzripSRL+xk5wW5pC6GOGcGwxcAEyPXOIkAeaqr1vW/SyC6ddAD2AgUAQ86ms2EhNmlgG8CtztnDt06mvqz8FwmjZWXw4Q51zIOTcQ6Ez1lWZ9/M1IYi3Wg7A9wNmnbHeOPCcB4pzbE/m3BFhI9YeDBE9xZO7Bl3MQSnzOR+LAOVcc+bIPA79F/TnpReaQvArMd879MfK0+nOAnK6N1ZeDyTlXBqwC8oGWZpYWeUm/sZNcrAdha4FekdVbGgOTgNdifAzxkZmlRyYCY2bpwHhgU+17SZJ6Dbgh8vgGYJGPuUicfPnDPOIy1J+TWmRC/7PAVufcY6e8pP4cEDW1sfpycJhZlpm1jDw+i+oF77ZSPRj7bqSY+nGSi+nqiACRJVGfAFKBec65n8X0AOIrM+tO9dkvgDTgBbVx8jOzF4HRQFugGHgA+B/gZaAL8ClwlXNOizoksRraeTTVly85YCdw6ylzhyTJmNkIYDXwARCOPH0v1XOG1J8DoJY2LkB9ORDMbADVC2+kUn3C5GXn3JzIb7AFQGtgPXCdc+6Ef5lKfcR8ECYiIiIiIiI108IcIiIiIiIiCaRBmIiIiIiISAJpECYiIiIiIpJAGoSJiIiIiIgkkAZhIiIiIiIiCaRBmIiIiIiISAJpECYiIiIiIpJA/wd/7eFdxG8MRwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x57.6 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "detection {'tp': 3, 'fp': 0, 'fn': 0, 'tn': 2}\n",
      "detect-mono {'tp': 3, 'fp': 2, 'fn': 3, 'tn': 2}\n",
      "monotony {'tp': 0, 'fp': 4, 'fn': 3, 'tn': 2}\n",
      "uniformity {'tp': 1.0, 'fp': 2.5833333333333335, 'fn': 2.0, 'tn': 0}\n",
      "total duration {'tp': 3.0, 'fp': 2.0, 'fn': 1.5, 'tn': 28.0}\n",
      "relative duration {'tp': 2.2666666666666666, 'fp': 0.5178571428571429, 'fn': 0.7333333333333334, 'tn': 1.4821428571428572}\n",
      "boundary onset {'tp': 0.8, 'fp': 2.2, 'fn': 0.0, 'tn': 0}\n",
      "boundary offset {'tp': 0.8666666666666667, 'fp': 2.1333333333333333, 'fn': 0.0, 'tn': 0}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ntp</th>\n",
       "      <th>Nfn</th>\n",
       "      <th>Nfp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>collar</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>segment</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>psd d/gtc=0.1</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>psd d/gtc=0.3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>psd d/gtc=0.5</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>psd d/gtc=0.8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>psd d/gtc=0.85</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>psd d/gtc=0.9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>detection</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>detect-mono</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>monotony</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>uniformity</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total duration</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relative duration</th>\n",
       "      <td>2.3</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boundary onset</th>\n",
       "      <td>0.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boundary offset</th>\n",
       "      <td>0.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Ntp  Nfn  Nfp\n",
       "collar             0.0  3.0  4.0\n",
       "segment            6.0  0.0  1.0\n",
       "psd d/gtc=0.1      3.0  0.0  0.0\n",
       "psd d/gtc=0.3      3.0  0.0  0.0\n",
       "psd d/gtc=0.5      3.0  0.0  0.0\n",
       "psd d/gtc=0.8      0.0  0.0  3.0\n",
       "psd d/gtc=0.85     0.0  0.0  3.0\n",
       "psd d/gtc=0.9      0.0  0.0  3.0\n",
       "detection          3.0  0.0  0.0\n",
       "detect-mono        3.0  3.0  2.0\n",
       "monotony           0.0  3.0  4.0\n",
       "uniformity         1.0  2.0  2.6\n",
       "total duration     3.0  1.5  2.0\n",
       "relative duration  2.3  0.7  0.5\n",
       "boundary onset     0.8  0.0  2.2\n",
       "boundary offset    0.9  0.0  2.1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import SED.my_eval\n",
    "scale=2\n",
    "r=np.array([[2,3],[5,6],[9,10],[11,13],[14,17],[18,20],[22,24],[25,27],[30,31],[33,34],[37,38],[40., 45.],[47., 49.],[50., 51.],[52., 54.],[56., 61.],[62., 63.],[64., 67.]])/scale\n",
    "p=np.array([[1,2],[3,4],[7,8],[9,10],[11,12],[15,16],[19,20],[21,23],[26,28],[29,32],[33,35],[36,38],[39., 41.], [42., 43.], [44., 46.], [48., 53.], [55., 57.], [58., 59.], [60., 65.], [66., 68.]])/scale\n",
    "r=np.array([[56., 61.],[62., 63.],[64., 67.]])/scale\n",
    "p=np.array([[55., 57.], [58., 59.], [60., 65.], [66., 68.]])/scale\n",
    "\n",
    "\n",
    "rdf,pdf,meta=Convert2SED(r,p,duration=69/scale)\n",
    "    \n",
    "\n",
    "res=SED.my_eval.get_single_result_df(rdf,pdf,meta,debug=1)\n",
    "res={m:res[m].loc['Test'][['Ntp','Nfn','Nfp']] for m in res}\n",
    "pd.DataFrame(res).T.round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-22T12:46:39.191064Z",
     "start_time": "2021-09-22T12:46:37.260289Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sox in /opt/conda/lib/python3.8/site-packages (1.4.0)\n",
      "Requirement already satisfied: numpy>=1.9.0 in /opt/conda/lib/python3.8/site-packages (from sox) (1.18.5)\n"
     ]
    }
   ],
   "source": [
    "!pip install sox\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-31T10:31:56.220659Z",
     "start_time": "2021-03-31T10:31:54.785679Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "parts={ '1':{'r':range(0,11),'p':range(0,12)},\n",
    "        '2':{'r':range(11,12),'p':range(12,15)},\n",
    "        '3':{'r':range(12,15),'p':range(15,16)},\n",
    "        '4':{'r':range(15,18),'p':range(16,20)}  }\n",
    "all={}\n",
    "for part in parts:\n",
    "    r2=r[parts[part]['r'],:]\n",
    "    p2=p[parts[part]['p'],:]\n",
    "    mi=min(r2.min(),p2.min())-1\n",
    "    r2=r2-mi\n",
    "    p2=p2-mi\n",
    "    ma=max(r2.max(),p2.max())+1\n",
    "    rdf,pdf,meta=Convert2SED(r2,p2,duration=ma)\n",
    "\n",
    "#     print()\n",
    "    import SED.my_eval2\n",
    "    res=SED.my_eval.get_single_result_df(rdf,pdf,meta,debug=0)\n",
    "    res={m:res[m].loc['Test'][['Ntp','Nfn','Nfp']] for m in res}\n",
    "    all[f'Part {part}']=pd.DataFrame(res)\n",
    "\n",
    "# print(pd.DataFrame(res).T.round(1))\n",
    "\n",
    "# print(pd.concat(all).T.round(1))\n",
    "pd.concat(all).T.round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat(all).T.round(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-28T18:30:49.486941Z",
     "start_time": "2021-03-28T18:30:49.166075Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import SED.my_eval\n",
    "path=\"/tmp/test/\"\n",
    "\n",
    "g=[[2,3],[5,6]]\n",
    "p=[[1,2],[3,7]]\n",
    "saveTestSED(path,g,p,10)\n",
    "res=SED.my_eval.get_single_result(f'{path}/g.tsv',f'{path}/p.tsv',f'{path}/meta.tsv',debug=0)\n",
    "# for m in res:\n",
    "for m in ['segment','total duration']:\n",
    "    print(m)\n",
    "    print(pd.DataFrame(res[m][['Ntp','Nfp','Nfn','Ntn']]))\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-30T19:36:29.865846Z",
     "start_time": "2021-03-30T19:36:29.860065Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.DataFrame([[2,3],[5,6]],columns=['onset','offset'])\n",
    "# g=[[2,3],[5,6]]\n",
    "# p=[[1,2],[3,7]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-30T19:35:30.384838Z",
     "start_time": "2021-03-30T19:35:30.373218Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "gtf='/workspace/sed2020/metadata/public.tsv'\n",
    "gt = pd.read_csv(gtf, sep=\"\\t\")\n",
    "gt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## real test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-28T20:35:35.564799Z",
     "start_time": "2021-03-28T20:35:12.953052Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import SED.my_eval\n",
    "path=\"/tmp/test/\"\n",
    "\n",
    "gtf='/workspace/sed2020/metadata/public.tsv'\n",
    "pef='/workspace/sed2020/submissions/CTK_NU/CTK_NU_task4_SED_1/public/CTK_NU_task4_SED_1.output.tsv'\n",
    "\n",
    "gt = pd.read_csv(gtf, sep=\"\\t\")\n",
    "    # Evaluate a single prediction\n",
    "pt = pd.read_csv(pef, sep=\"\\t\")\n",
    "\n",
    "clas=gt.event_label.append(pt.event_label).unique()\n",
    "    \n",
    "result={}\n",
    "for c in ['Blender']:\n",
    "        gtc=gt.loc[gt.event_label==c]\n",
    "        ptc=pt.loc[pt.event_label==c]\n",
    "        all=None\n",
    "        for f in gtc.filename.unique():\n",
    "            # if(c=='Blender'):debug=1\n",
    "#             print(f'============== class={c}=========file={f}')\n",
    "            g=gtc.loc[gtc.filename==f][['onset','offset']].values\n",
    "            p=ptc.loc[ptc.filename==f][['onset','offset']].values\n",
    "            saveTestSED(path,g,p,10)\n",
    "            res=SED.my_eval.get_single_result(f'{path}/g.tsv',f'{path}/p.tsv',f'{path}/meta.tsv',debug=0)\n",
    "            # for m in res:\n",
    "            out={}\n",
    "            out['segment']=res['segment'][['Ntp','Nfp','Nfn']].loc['Test']\n",
    "            out['total duration']=res['total duration'][['Ntp','Nfp','Nfn']].loc['Test']\n",
    "            out['diff']=out['segment']-out['total duration']\n",
    "            out=pd.DataFrame(out).T\n",
    "            if(all is None):\n",
    "                all=out\n",
    "            else:\n",
    "                all+=out\n",
    "            if (out.loc['diff']['Nfp'])>.1:\n",
    "                print(f'============== class={c}=========file={f}')\n",
    "                print(out)\n",
    "#             res=SED.my_eval.get_single_result(f'{path}/g.tsv',f'{path}/p.tsv',f'{path}/meta.tsv',debug=1)\n",
    "#             break\n",
    "#         break\n",
    "\n",
    "all\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Single file result direct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-28T21:18:53.879761Z",
     "start_time": "2021-03-28T21:18:52.356282Z"
    }
   },
   "outputs": [],
   "source": [
    "print('ali')\n",
    "import pandas as pd\n",
    "import SED.my_eval\n",
    "path=\"/tmp/test/\"\n",
    "\n",
    "gtf='/workspace/sed2020/metadata/public.tsv'\n",
    "pef='/workspace/sed2020/submissions/CTK_NU/CTK_NU_task4_SED_1/public/CTK_NU_task4_SED_1.output.tsv'\n",
    "\n",
    "gt = pd.read_csv(gtf, sep=\"\\t\")\n",
    "    # Evaluate a single prediction\n",
    "pt = pd.read_csv(pef, sep=\"\\t\")\n",
    "\n",
    "clas=gt.event_label.append(pt.event_label).unique()\n",
    "    \n",
    "result={}\n",
    "for f in gt.filename.unique():\n",
    "\n",
    "        gtc=gt.loc[gt.filename==f]\n",
    "        ptc=pt.loc[pt.filename==f]\n",
    "        all=None\n",
    "        res=SED.my_eval.get_single_result_df(gtc,ptc,debug=0)\n",
    "        # for m in res:\n",
    "        out={}\n",
    "        out['segment']=res['segment'][['Ntp','Nfp','Nfn']].loc['Test']\n",
    "        out['total duration']=res['total duration'][['Ntp','Nfp','Nfn']].loc['Test']\n",
    "        out['diff']=out['segment']-out['total duration']\n",
    "        out=pd.DataFrame(out).T\n",
    "        if(all is None):\n",
    "            all=out\n",
    "        else:\n",
    "            all+=out\n",
    "        if (out.loc['diff']['Nfp'])>.1:\n",
    "            print(f'============== class={c}=========file={f}')\n",
    "            print(out)\n",
    "#             res=SED.my_eval.get_single_result(f'{path}/g.tsv',f'{path}/p.tsv',f'{path}/meta.tsv',debug=1)\n",
    "#             break\n",
    "#         break\n",
    "\n",
    "all\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2021-03-28T20:36:38.627Z"
    }
   },
   "source": [
    "## single team test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-28T20:49:14.618640Z",
     "start_time": "2021-03-28T20:49:11.023327Z"
    }
   },
   "outputs": [],
   "source": [
    "print('ali')\n",
    "import pandas as pd\n",
    "import SED.my_eval\n",
    "gtf='/workspace/sed2020/metadata/public.tsv'\n",
    "pef='/workspace/sed2020/submissions/CTK_NU/CTK_NU_task4_SED_1/public/CTK_NU_task4_SED_1.output.tsv'\n",
    "res=SED.my_eval.get_single_result(gtf,pef,debug=0)\n",
    "out={}\n",
    "out['segment']=res['segment'][['Ntp','Nfp','Nfn']].loc['Blender']\n",
    "out['total duration']=res['total duration'][['Ntp','Nfp','Nfn']].loc['Blender']\n",
    "out['diff']=out['segment']-out['total duration']\n",
    "pd.DataFrame(out).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-30T12:14:38.424981Z",
     "start_time": "2021-03-30T12:14:38.410679Z"
    }
   },
   "outputs": [],
   "source": [
    "res2={c:res[c].loc['Test'] for c in res.keys() }\n",
    "print(pd.DataFrame(res2).T[ ['Nfn','Nfp','Ntp','precision','recall','f1' ]].round(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T17:10:59.313360Z",
     "start_time": "2021-03-25T17:10:58.085384Z"
    }
   },
   "outputs": [],
   "source": [
    "# total=pd.DataFrame(total).T\n",
    "# total['y']=total.index\n",
    "# print(total)\n",
    "# total2=total2[-20:-1]\n",
    "# total2=total\n",
    "import seaborn as sns\n",
    "sns.set_theme(style=\"whitegrid\")\n",
    "# Make the PairGrid\n",
    "g = sns.PairGrid(total2.sort_values(\"psds\", ascending=False),\n",
    "                 x_vars=total2.columns[0:-1], y_vars=['y'],\n",
    "                 height=len(total2.index)/2, aspect=4/len(total2.index))\n",
    "\n",
    "def qqplot(x, y, **kwargs):\n",
    "#     sns.stripplot\n",
    "    print(x.__dict__)\n",
    "    sns.stripplot(x=x, y=y, data=total,**kwargs)\n",
    "# Draw a dot plot using the stripplot function\n",
    "g.map(sns.stripplot, size=15, orient=\"h\", jitter=False,\n",
    "      palette=\"flare_r\", linewidth=1, edgecolor=\"w\")\n",
    "\n",
    "# Use the same x axis limits on all columns anmidmx add better lab+(mx-mi)/2els\n",
    "mx=total.max(axis=1).max()\n",
    "mi=total.min(axis=1).min()\n",
    "g.set(xlim=((mx+mi)/2,mx ), xlabel=\"F1\", ylabel=\"\")\n",
    "\n",
    "# Use semantically meaningful titles for the columns\n",
    "titles = total.columns[0:-1]\n",
    "\n",
    "for ax, title in zip(g.axes.flat, titles):\n",
    "\n",
    "    # Set a different title for each axes\n",
    "    ax.set(title=title)\n",
    "\n",
    "    # Make the grid horizontal instead of vertical\n",
    "    ax.xaxis.grid(True)\n",
    "    ax.yaxis.grid(True)\n",
    "\n",
    "sns.despine(left=True, bottom=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T17:22:48.797122Z",
     "start_time": "2021-03-25T17:22:48.564838Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(1, 3, figsize=(9, 3), sharey=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T18:17:40.496476Z",
     "start_time": "2021-03-25T18:17:40.493296Z"
    }
   },
   "outputs": [],
   "source": [
    "a=[1,2,3,4]\n",
    "a[-2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-12T12:43:23.241959Z",
     "start_time": "2021-03-12T12:43:22.131854Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# total=pd.DataFrame(total).T\n",
    "# total['y']=total.index\n",
    "# print(total)\n",
    "import seaborn as sns\n",
    "sns.set_theme(style=\"whitegrid\")\n",
    "# Make the PairGrid\n",
    "g = sns.PairGrid(total.sort_values(\"gem-avg\", ascending=False),\n",
    "                 x_vars=total.columns[0:-1], y_vars=['y'],\n",
    "                 height=len(total.index)/2, aspect=4/len(total.index))\n",
    "\n",
    "f=lambda x,y: sns.barplot if(y==2) else sns.stripplot \n",
    "# Draw a dot plot using the stripplot function\n",
    "g.map(f, size=15, orient=\"h\", jitter=False,\n",
    "      palette=\"flare_r\", linewidth=1, edgecolor=\"w\")\n",
    "\n",
    "# Use the same x axis limits on all columns anmidmx add better lab+(mx-mi)/2els\n",
    "mx=total.max(axis=1).max()\n",
    "mi=total.min(axis=1).min()\n",
    "g.set(xlim=((mx+mi)/2,mx ), xlabel=\"F1\", ylabel=\"\")\n",
    "\n",
    "# Use semantically meaningful titles for the columns\n",
    "titles = total.columns[0:-1]\n",
    "\n",
    "for ax, title in zip(g.axes.flat, titles):\n",
    "\n",
    "    # Set a different title for each axes\n",
    "    ax.set(title=title)\n",
    "\n",
    "    # Make the grid horizontal instead of vertical\n",
    "    ax.xaxis.grid(True)\n",
    "    ax.yaxis.grid(True)\n",
    "\n",
    "sns.despine(left=True, bottom=True)  \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T15:00:22.629450Z",
     "start_time": "2021-03-25T15:00:18.700146Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "#@interact\n",
    "#def result_selector(gtf=os.listdir(f'{rootFolder}/metadata/')):\n",
    "# import SED.my_eval\n",
    "gtf='public.tsv'\n",
    "rootFolder='/workspace/sed2020/'\n",
    "typ=gtf.split('.')[0]\n",
    "gtf=f'{rootFolder}/metadata/{gtf}'\n",
    "# meta_dur_df=pd.DataFrame(columns=['filename','duration'])\n",
    "# meta_dur_df['filename']=groundtruth['filename']\n",
    "# meta_dur_df['duration']=10\n",
    "total_dic={}\n",
    "for team in sorted(os.listdir(f'{rootFolder}/submissions/')):\n",
    "    print(f'analysing team {team}')\n",
    "    for code in sorted(os.listdir(f'{rootFolder}/submissions/{team}')):\n",
    "        print(f'    {code}')\n",
    "        base_prediction_path=f'{rootFolder}/submissions/{team}/{code}/{typ}/'\n",
    "        pef = f'{base_prediction_path}/{code}.output.tsv'\n",
    "        if not(os.path.isfile(pef)):\n",
    "            all=[x for x in os.listdir(base_prediction_path) if '.output.tsv' in x]\n",
    "            if len(all)>0:\n",
    "                pef=f'{base_prediction_path}/{all[0]}'\n",
    "            else:\n",
    "                print(pef)\n",
    "                continue\n",
    "        title=code.replace('_task4','')\n",
    "        groundtruth = pd.read_csv(gtf, sep=\"\\t\")\n",
    "        # Evaluate a single prediction\n",
    "        predictions = pd.read_csv(pef, sep=\"\\t\")\n",
    "        res1=SED.my_eval.get_single_result(gtf,pef)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T15:00:39.000499Z",
     "start_time": "2021-03-25T15:00:38.992084Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T15:05:47.948920Z",
     "start_time": "2021-03-25T15:05:47.941093Z"
    }
   },
   "outputs": [],
   "source": [
    "#%matplotlib inline\n",
    "\n",
    "clas=groundtruth.event_label.append(predictions.event_label).unique()\n",
    "clas\n",
    "gt=groundtruth\n",
    "pt=predictions\n",
    "import metric\n",
    "# m=metric.GEM_NEW\n",
    "for c in clas:\n",
    "    gtc=gt.loc[gt.event_label==c]\n",
    "    ptc=pt.loc[pt.event_label==c]\n",
    "    \n",
    "    for f in gtc.filename.unique():\n",
    "        print(f'============== class={c}=========file={f}')\n",
    "        g=gtc.loc[gtc.filename==f][['onset','offset']].values\n",
    "        p=ptc.loc[ptc.filename==f][['onset','offset']].values\n",
    "#         print(eval_my_metric(g,p,(0,10)))\n",
    "        print(g)\n",
    "        break\n",
    "#         print('gtc',\n",
    "#         print('ptc',ptc.loc[ptc.filename==f]).apply(p=>(p.onset,p.offset))\n",
    "#         p=pt.loc[pt.event_label==c and pt.filename=g.filename]\n",
    "#         print(p)\n",
    "    break    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T15:11:53.122112Z",
     "start_time": "2021-03-25T15:11:53.118653Z"
    }
   },
   "outputs": [],
   "source": [
    "# g2=np.array(g)\n",
    "# g2=g2[g2[:,0].argsort(),:]\n",
    "# g2[0,:]=g[-1]*2\n",
    "# g2\n",
    "np.vstack((g,[1,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T13:57:38.627537Z",
     "start_time": "2021-03-25T13:57:38.605840Z"
    }
   },
   "outputs": [],
   "source": [
    "def intersection(e1,e2):    \n",
    "    inter=(max(e1[0],e2[0]),min(e1[1],e2[1]))\n",
    "    if(inter[1]<=inter[0]): \n",
    "        inter=None\n",
    "#     print(e1,e2,inter)\n",
    "    return inter\n",
    "\n",
    "def dur(e):\n",
    "    d= e[1]-e[0]\n",
    "    if(d<0):\n",
    "        print('erorr duration is less than zero' )\n",
    "    return d\n",
    "\n",
    "\n",
    "def eval_my_metric(real,pred,duration=(0,10),alpha=2,debug=0,calcne=1):\n",
    "        debug={'D':1,'T':1, 'M':1,'R':1,'V':1}#V:verbose\n",
    "        # real=merge_events_if_necessary(real)\n",
    "        # pred=merge_events_if_necessary(pred)\n",
    "        # real_tree=_makeIntervalTree(real,'r')\n",
    "        # pred_tree=_makeIntervalTree(pred,'p')\n",
    "        duration=(min(duration[0],real[0][0]),max(duration[1],real[-1][1]))\n",
    "        real=np.append(real,duration[1])# add a zero duration event in the end for ease comparision the last event\n",
    "        real[-1]=(duration[1],duration[1])\n",
    "        pred=np.append(pred,duration[1])# add a zero duration event in the end for ease comparision the last event\n",
    "        pred[-1]=(duration[1],duration[1])\n",
    "        #_ means negative\n",
    "        rel={'r+':{},'r-':{},'p+':{},'p-':{}}\n",
    "        print(real)\n",
    "        r_0=(duration[0],real[0][0])\n",
    "        r_n=(real[-1][1],duration[1])\n",
    "        metric={}\n",
    "        pi=0\n",
    "        rcalc=[]\n",
    "        real_=[]\n",
    "        pred_=[]\n",
    "        ri_=0\n",
    "        for ri in range(len(real)):\n",
    "            r=real[ri]\n",
    "            rp=real[ri-1] if ri>0 else (duration[0],duration[0])\n",
    "            r_=(rp[1],r[0])\n",
    "            \n",
    "            tmpr={'p+':{},'p-':{}}\n",
    "            tmpr_={'p+':{},'p-':{}}\n",
    "            rel['r+'][ri]=tmpr\n",
    "            \n",
    "            if(dur(r_)>0):\n",
    "                real_.append(r_)\n",
    "                ri_=len(real_)-1\n",
    "                rel['r-'][ri_]=tmpr_\n",
    "            \n",
    "            cond=pi<len(pred) \n",
    "            pi_=-1\n",
    "            while  cond:\n",
    "                pp=pred[pi-1] if pi>0 else (duration[0],duration[0])\n",
    "                p=pred[pi]\n",
    "                p_=(pp[1],p[0])\n",
    "                \n",
    "                if(dur(p_)>0 and (len(pred_)==0 or pred_[-1]!=p_)):\n",
    "                    pred_.append(p_)\n",
    "                pi_=len(pred_)-1\n",
    "\n",
    "                if not(pi in rel['p+']):\n",
    "                    rel['p+'][pi]={'r+':{},'r-':{}}\n",
    "                if not(pi_ in rel['p-']):\n",
    "                    rel['p-'][pi_]={'r+':{},'r-':{}}\n",
    "                tmpp=rel['p+'][pi]\n",
    "                tmpp_=rel['p-'][pi_]\n",
    "\n",
    "                rinter  =intersection(r,p)\n",
    "                rinter_ =intersection(r,p_)\n",
    "                r_inter =intersection(r_,p)                \n",
    "                r_inter_=intersection(r_,p_)\n",
    "                if(rinter is not None):\n",
    "                    # tmpr['p+'].append((pi,rinter))\n",
    "                    # tmpp['r+'].append((ri,rinter))\n",
    "                    tmpr['p+'][pi]=rinter\n",
    "                    tmpp['r+'][ri]=rinter\n",
    "                if(rinter_ is not None):\n",
    "                    # tmpr['p-'].append((pi,rinter_))\n",
    "                    # tmpp_['r+'].append((ri,rinter_))\n",
    "                    tmpr['p-'][pi_]=rinter_\n",
    "                    tmpp_['r+'][ri]=rinter_\n",
    "                if(r_inter is not None):\n",
    "                    # tmpr_['p+'].append((pi,r_inter))\n",
    "                    # tmpp['r-'].append((ri,r_inter))\n",
    "                    tmpr_['p+'][pi]=r_inter\n",
    "                    tmpp['r-'][ri_]=r_inter\n",
    "                if(r_inter_ is not None):\n",
    "                    # tmpr_['p-'].append((pi,r_inter_))\n",
    "                    # tmpp_['r-'].append((ri,r_inter_))\n",
    "                    tmpr_['p-'][pi_]=r_inter_\n",
    "                    tmpp_['r-'][ri_]=r_inter_\n",
    "                \n",
    "                \n",
    "                if pred[pi][1] < r[1]:\n",
    "                    pi+=1\n",
    "                else: cond=False\n",
    "                \n",
    "            \n",
    "            # for k in list(rel.keys()):\n",
    "            #     if len(rel[k])>0: continue\n",
    "            #     del rel[k]\n",
    "        \n",
    "        real=np.delete(real,-1,0)#real.pop()\n",
    "        pred=np.delete(pred,-1,0)#pred.pop()\n",
    "#         if(dur(pred_[-1])==0):pred_=np.delete(pred_,-1,0)\n",
    "#         if(dur(real_[-1])==0):real_=np.delete(real_,-1,0)\n",
    "\n",
    "        \n",
    "        out={\n",
    "            'detection':        {'tp':0,'fp':0,'fn':0,'tn':0},\n",
    "            'monotony':         {'tp':0,'fp':0,'fn':0,'tn':0},\n",
    "            'total duration':   {'tp':0,'fp':0,'fn':0,'tn':0},\n",
    "            'relative duration':{'tp':0,'fp':0,'fn':0,'tn':0}\n",
    "        }\n",
    "        \n",
    "        if debug['V']:\n",
    "            print(\"real=\",real)\n",
    "            print(\"pred=\",pred)\n",
    "            print(\"real_=\",real_)\n",
    "            print(\"pred_=\",pred_)\n",
    "            #for x in rel:\n",
    "            [print(f'{x}: {rel[x]}') for x in rel]\n",
    "        \n",
    "        for ri in range(len(real)):\n",
    "            tpd=int(len(rel['r+'][ri]['p+'])>0)\n",
    "            out['detection']['tp']+=tpd\n",
    "            if debug['D']: print(f\"D TP+{tpd}      ri={ri}, p+={rel['r+'][ri]['p+']}>0\")\n",
    "            #monotony {\n",
    "            if (len(rel['r+'][ri]['p+'])==1):\n",
    "                for rpi in rel['r+'][ri]['p+']:\n",
    "                    if len(rel['p+'][rpi]['r+'])==1:\n",
    "                        out['monotony']['tp']+=1\n",
    "                        if debug['M']:print(f\"M TP+1     rel[r+][{ri}][p+]={rel['r+'][ri]['p+']}==1 rel[p+][{rpi}][r+]={rel['p+'][rpi]['r+']}==1\")\n",
    "                    elif(len(rel['p+'][rpi]['r+'])==0):\n",
    "                        print('error it can not be zero')\n",
    "                    elif debug['M']:print(f\"M--tp rel[r+][{ri}][p+]={rel['r+'][ri]['p+']}==1 rel[p+][{rpi}][r+]={rel['p+'][rpi]['r+']}>1\")\n",
    "            #}\n",
    "            \n",
    "            for pi in rel['r+'][ri]['p+']:\n",
    "                tpt=dur(rel['r+'][ri]['p+'][pi])\n",
    "                tpr=tpt/dur(real[ri])\n",
    "                out['total duration']['tp']+=tpt\n",
    "                out['relative duration']['tp']+=tpr\n",
    "                if debug['T']:print(f\"T tp+={tpt}             rel[r+][{ri}][p+][{pi}]=dur({rel['r+'][ri]['p+'][pi]})\")\n",
    "                if debug['R']:print(f\"R tp+={tpr}             rel[r+][{ri}][p+][{pi}]==dur({rel['r+'][ri]['p+'][pi]}) / real[{ri}]=dur({real[ri]})\")\n",
    "                          \n",
    "            for pi in rel['r+'][ri]['p-']:\n",
    "                fnt=dur(rel['r+'][ri]['p-'][pi])\n",
    "                fnr=fnt/dur(real[ri])\n",
    "                out['total duration']['fn']+=fnt\n",
    "                out['relative duration']['fn']+=fnr\n",
    "                if debug['T']:print(f\"T fn+={fnt}             rel[r+][{ri}][p-][{pi}]=dur({rel['r+'][ri]['p-'][pi]})\")\n",
    "                if debug['R']:print(f\"R fn+={fnr}             rel[r+][{ri}][p-][{pi}]==dur({rel['r+'][ri]['p-'][pi]}) / real[{ri}]=dur({real[ri]})\")\n",
    "                \n",
    "\n",
    "        for ri in range(len(real_)):\n",
    "            tnd=int(len(rel['r-'][ri]['p-'])>0)\n",
    "            out['detection']['tn']+=tnd\n",
    "            if debug['D']: print(f\"D TN+{tnd}      ri-={ri}, p-={rel['r-'][ri]['p-']}>0\")\n",
    "            #monotony {\n",
    "            \n",
    "            if (len(rel['r-'][ri]['p-'])==1):\n",
    "                for rpi in rel['r-'][ri]['p-']:\n",
    "                    if len(rel['p-'][rpi]['r-'])==1:\n",
    "                        out['monotony']['tn']+=1\n",
    "                        if debug['M']:print(f\"M TN+1     rel[r-][{ri}][p-]={rel['r-'][ri]['p-']}==1 rel[p-][{rpi}][r-]={rel['p-'][rpi]['r-']}==1\")\n",
    "                    elif(len(rel['p-'][rpi]['r-'])==0):\n",
    "                        print('error it can not be zero')\n",
    "                    elif debug['M']:print(f\"M--tn rel[r-][{ri}][p-]={rel['r-'][ri]['p-']}==1 rel[p-][{rpi}][r-]={rel['p-'][rpi]['r-']}>1\")\n",
    "            #}\n",
    "\n",
    "            for pi in rel['r-'][ri]['p-']:\n",
    "                tnt=dur(rel['r-'][ri]['p-'][pi])\n",
    "                tnr=tnt/dur(real_[ri])\n",
    "                out['total duration']['tn']+=tnt\n",
    "                out['relative duration']['tn']+=tnr\n",
    "                if debug['T']:print(f\"T tn+={tnt}             rel[r-][{ri}][p-][{pi}]=dur({rel['r-'][ri]['p-'][pi]})\")\n",
    "                if debug['R']:print(f\"R tn+={tnr}             rel[r-][{ri}][p-][{pi}]==dur({rel['r-'][ri]['p-'][pi]}) / real_[{ri}]=dur({real_[ri]})\")\n",
    "            for pi in rel['r-'][ri]['p+']:\n",
    "                fpt=dur(rel['r-'][ri]['p+'][pi])\n",
    "                fpr=fpt/dur(real_[ri])\n",
    "                out['total duration']['fp']+=fpt\n",
    "                out['relative duration']['fp']+=fpr\n",
    "                if debug['T']:print(f\"T fp+={fpt}             rel[r-][{ri}][p+][{pi}]=dur({rel['r-'][ri]['p+'][pi]})\")\n",
    "                if debug['R']:print(f\"R fp+={fpr}             rel[r-][{ri}][p+][{pi}]==dur({rel['r-'][ri]['p+'][pi]}) / real_[{ri}]=dur({real_[ri]})\")\n",
    "\n",
    "        out['detection']['fp']=len(real_)-out['detection']['tn']\n",
    "        if debug['D']: print(f\"D fp={out['detection']['fp']} #r-={len(real_)} - tn={out['detection']['tn']}\"  )\n",
    "        out['detection']['fn']=len(real)-out['detection']['tp']\n",
    "        if debug['D']: print(f\"D fn={out['detection']['fn']} #r+={len(real)} - tp={out['detection']['tp']}\"  )\n",
    "                        \n",
    "        out['monotony']['fn']=len(real)-out['monotony']['tp']+len(pred_)-out['monotony']['tn']\n",
    "        if debug['M']: print(f\"M fn={out['monotony']['fn']}     #r+={len(real)} - tp={out['monotony']['tp']} + #p-={len(pred_)} - tn={out['monotony']['tn']}\")\n",
    "        out['monotony']['fp']=len(pred)-out['monotony']['tp']+len(real_)-out['monotony']['tn']\n",
    "        if debug['M']: print(f\"M fp={out['monotony']['fp']}     #p+={len(pred)} - tp={out['monotony']['tp']} + #r-={len(real_)} - tn={out['monotony']['tn']}\")\n",
    "        \n",
    "                        \n",
    "        for pi in range(len(pred)):\n",
    "            fpd=int(len(rel['p+'][pi]['r+'])==0)\n",
    "            out['detection']['fp']+=fpd\n",
    "            if debug['D']: print(f\"D FP+{fpd}      pi={pi}, r={rel['p+'][pi]['r+']}==0\")\n",
    "#             for ri in rel['p+'][pi]['r-']:\n",
    "#                 out['total duration']['fp']+=dur(rel['p+'][pi]['r-'][ri])\n",
    "#                 out['relative duration']['fp']+=dur(rel['p+'][pi]['r-'][ri])/dur(pred[pi])\n",
    "\n",
    "        for pi in range(len(pred_)):\n",
    "            fnd=int(len(rel['p-'][pi]['r-'])==0)\n",
    "            out['detection']['fn']+=fnd\n",
    "            if debug['D']: print(f\"D FN+{fnd}      pi-={pi}, r-={rel['p-'][pi]['r-']}==0\")\n",
    "#             for ri in rel['p-'][pi]['r+']:\n",
    "#                 out['total duration']['fn']+=dur(rel['p-'][pi]['r+'][ri])\n",
    "#                 out['relative duration']['fn']+=dur(rel['p-'][pi]['r+'][ri])/dur(pred_[pi])\n",
    "\n",
    "#         plot_events_with_event_scores(range(len(real)),range(len(pred)),real,pred)\n",
    "#         plot_events_with_event_scores(range(len(real_)),range(len(pred_)),real_,pred_)\n",
    "        if debug['V']:plot_events(real,pred,real_,pred_)\n",
    "        return  out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-25T14:00:55.110533Z",
     "start_time": "2021-03-25T14:00:55.103507Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_events(real,pred,real_,pred_, label=None):\n",
    "    from matplotlib.pylab import plt\n",
    "    import random\n",
    "    fig,ax = plt.subplots(figsize=(10, 2))\n",
    "    ax.set_title(label)\n",
    "    plt.xlim(0,max(real[-1][1],10))\n",
    "    ax.set_xticks(np.arange(0,max(real[-1][1],10),.1),minor=True)\n",
    "    maxsize=20\n",
    "    for i in range(min(maxsize,len(pred_))):\n",
    "        d = pred_[i]\n",
    "        plt.axvspan(d[0], d[1], 0, 0.4,linewidth=1,edgecolor='k',facecolor='m', alpha=.6)\n",
    "        plt.text((d[1] + d[0]) / 2, 0.1+random.random()/4,f'{i}' , horizontalalignment='center', verticalalignment='center')\n",
    "    for i in range(min(maxsize,len(pred))):\n",
    "        d = pred[i]\n",
    "        plt.axvspan(d[0], d[1], 0.1, 0.5,linewidth=1,edgecolor='k',facecolor='r', alpha=.6)\n",
    "        plt.text((d[1] + d[0]) / 2, 0.2+random.random()/4,f'{i}' , horizontalalignment='center', verticalalignment='center')\n",
    "#     maxsize=len(real)\n",
    "    for i in range(min(maxsize,len(real_))):\n",
    "        gt = real_[i]\n",
    "        plt.axvspan(gt[0], gt[1], 0.6, 1,linewidth=1,edgecolor='k',facecolor='y', alpha=.6)\n",
    "        plt.text((gt[1] + gt[0]) / 2, 0.6+random.random()/4,f'{i}' , horizontalalignment='center', verticalalignment='center')\n",
    "        \n",
    "    for i in range(min(maxsize,len(real))):\n",
    "        gt = real[i]\n",
    "        plt.axvspan(gt[0], gt[1], 0.5, .9,linewidth=1,edgecolor='k',facecolor='g', alpha=.6)\n",
    "        plt.text((gt[1] + gt[0]) / 2, 0.5+random.random()/4,f'{i}' , horizontalalignment='center', verticalalignment='center')\n",
    "    plt.grid(True)\n",
    "    plt.minorticks_on()\n",
    "    plt.grid(b=True, which='minor', color='#999999', linestyle='-', alpha=0.2)\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}