MCAN-RoBERTa_pair-cat_box_tfidf-neg_focal_all_predict-all_cls.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### imports\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch, math\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils import data\n",
    "from torch import optim\n",
    "from torch.optim.lr_scheduler import LambdaLR\n",
    "\n",
    "from transformers import get_linear_schedule_with_warmup,get_cosine_with_hard_restarts_schedule_with_warmup\n",
    "from transformers import AutoTokenizer, AutoModel\n",
    "\n",
    "from gensim import corpora, similarities, models\n",
    "from gensim.matutils import corpus2csc\n",
    "from sklearn.model_selection import KFold\n",
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import base64\n",
    "from tqdm.auto import tqdm\n",
    "import pickle, random\n",
    "import matplotlib.pyplot as plt\n",
    "from multiprocessing import Pool\n",
    "import sys, csv, json, os, gc, time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### parameters\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fb10e6513d0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no = '3'\n",
    "device = torch.device('cuda:'+no) if torch.cuda.is_available() else torch.device('cpu')\n",
    "print(device)\n",
    "\n",
    "k = 10\n",
    "lr = 1e-5\n",
    "# true batch size = batch_size * grad_step\n",
    "batch_size = 64\n",
    "margin = 8\n",
    "grad_step = 1\n",
    "max_img_len = 30\n",
    "epochs = 1 # only read data once\n",
    "MOD = 20000\n",
    "shuffle_fold = True\n",
    "workers = 48\n",
    "seed = 9115\n",
    "random.seed(seed)\n",
    "torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class params:\n",
    "    LABEL_SIZE = 32\n",
    "    IMG_FEAT_SIZE = 2048+6+LABEL_SIZE\n",
    "    WORD_EMBED_SIZE = 1024\n",
    "    LAYER = 12\n",
    "    HIDDEN_SIZE = 1024\n",
    "    MULTI_HEAD = 16\n",
    "    DROPOUT_R = 0.1\n",
    "    FLAT_MLP_SIZE = 512\n",
    "    FLAT_GLIMPSES = 1\n",
    "    FLAT_OUT_SIZE = 2048\n",
    "    FF_SIZE = HIDDEN_SIZE*4\n",
    "    HIDDEN_SIZE_HEAD = HIDDEN_SIZE // MULTI_HEAD\n",
    "    OPT_BETAS = (0.9, 0.98)\n",
    "    OPT_EPS = 1e-9\n",
    "    TRAIN_SIZE = 3000000\n",
    "\n",
    "__C = params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load data\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "trash = {'!', '$', \"'ll\", \"'s\", ',', '&', ':', 'and', 'cut', 'is', 'are', 'was'}\n",
    "trash_replace = ['\"hey siri, play some', 'however, ', 'yin and yang, ',\n",
    "                 'shopping mall/']\n",
    "\n",
    "def process(x):\n",
    "    tmp = x.split()\n",
    "    if tmp[0] in trash: x = ' '.join(tmp[1:])\n",
    "    if tmp[0][0] == '-': x = x[1:]\n",
    "    for tr in trash_replace:\n",
    "        x = x.replace(tr, '')\n",
    "    return x\n",
    "\n",
    "def normalize(x):\n",
    "    ret = x['boxes'].copy()\n",
    "    ret[:,0] /= x['image_h']\n",
    "    ret[:,1] /= x['image_w']\n",
    "    ret[:,2] /= x['image_h']\n",
    "    ret[:,3] /= x['image_w']\n",
    "    wh = (ret[:,2]-ret[:,0]) * (ret[:,3]-ret[:,1])\n",
    "    wh2 = (ret[:,2]-ret[:,0]) / (ret[:,3]-ret[:,1]+1e-6)\n",
    "    ret = np.hstack((ret, wh.reshape(-1,1), wh2.reshape(-1,1)))\n",
    "    return ret\n",
    "\n",
    "def load_data(file_name, reset=False, decode=True):\n",
    "    ret = pd.read_csv(file_name, sep='\\t')\n",
    "    if decode:\n",
    "        ret['boxes'] = ret['boxes'].apply(lambda x: np.frombuffer(base64.b64decode(x), dtype=np.float32).reshape(-1, 4))\n",
    "        ret['features'] = ret['features'].apply(lambda x: np.frombuffer(base64.b64decode(x), dtype=np.float32).reshape(-1, 2048))\n",
    "        ret['class_labels'] = ret['class_labels'].apply(lambda x: np.frombuffer(base64.b64decode(x), dtype=np.int64).reshape(-1, 1))\n",
    "        ret['boxes'] = ret.apply(lambda x: normalize(x), axis=1)\n",
    "        ret['features'] = ret.apply(lambda x: np.concatenate((x['class_labels'], x['features'], x['boxes']), axis=1)[:max_img_len], axis=1)\n",
    "    ret['query'] = ret['query'].apply(lambda x: process(x))\n",
    "    # reset query_id\n",
    "    if reset:\n",
    "        query2qid = {query: qid for qid, (query, _) in enumerate(tqdm(ret.groupby('query')))}\n",
    "        ret['query_id'] = ret.apply(lambda x: query2qid[x['query']], axis=1)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './'\n",
    "test = load_data(path+'valid.tsv')\n",
    "testA = load_data(path+'testB.tsv')\n",
    "answers = json.loads(open(path+'valid_answer.json', 'r').read())\n",
    "test['target'] = test.apply(lambda x: 1 if x['product_id'] in answers[str(x['query_id'])] else 0, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### preprocess\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "395dabc6dd974974a37746f2cff0fde6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07918f9934ac474abf288cf020e98b35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# load pre-trained model\n",
    "take = 'roberta-large'\n",
    "emb_size = __C.WORD_EMBED_SIZE\n",
    "tokenizer = AutoTokenizer.from_pretrained(take)\n",
    "pretrained_emb = AutoModel.from_pretrained(take)\n",
    "pad_id = tokenizer.pad_token_id\n",
    "\n",
    "qid2token = {qid: tokenizer.encode(group['query'].values[0]) for qid, group in tqdm(test.groupby('query_id'))}\n",
    "test['token'] = test['query_id'].apply(lambda x: qid2token[x])\n",
    "qid2token = {qid: tokenizer.encode(group['query'].values[0]) for qid, group in tqdm(testA.groupby('query_id'))}\n",
    "testA['token'] = testA['query_id'].apply(lambda x: qid2token[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### model\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FC(nn.Module):\n",
    "    def __init__(self, in_size, out_size, dropout_r=0., use_relu=True):\n",
    "        super(FC, self).__init__()\n",
    "        self.dropout_r = dropout_r\n",
    "        self.use_relu = use_relu\n",
    "\n",
    "        self.linear = nn.Linear(in_size, out_size)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.dropout = nn.Dropout(dropout_r)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.linear(x)\n",
    "        x = self.relu(x)\n",
    "        x = self.dropout(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class MLP(nn.Module):\n",
    "    def __init__(self, in_size, mid_size, out_size, dropout_r=0., use_relu=True):\n",
    "        super(MLP, self).__init__()\n",
    "\n",
    "        self.fc = FC(in_size, mid_size, dropout_r=dropout_r, use_relu=use_relu)\n",
    "        self.linear = nn.Linear(mid_size, out_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.linear(self.fc(x))\n",
    "\n",
    "\n",
    "class LayerNorm(nn.Module):\n",
    "    def __init__(self, size, eps=1e-6):\n",
    "        super(LayerNorm, self).__init__()\n",
    "        self.eps = eps\n",
    "\n",
    "        self.a_2 = nn.Parameter(torch.ones(size))\n",
    "        self.b_2 = nn.Parameter(torch.zeros(size))\n",
    "\n",
    "    def forward(self, x):\n",
    "        mean = x.mean(-1, keepdim=True)\n",
    "        std = x.std(-1, keepdim=True)\n",
    "        return self.a_2 * (x-mean) / (std+self.eps) + self.b_2\n",
    "    \n",
    "class MHAtt(nn.Module):\n",
    "    def __init__(self, __C):\n",
    "        super(MHAtt, self).__init__()\n",
    "        self.__C = __C\n",
    "\n",
    "        self.linear_v = nn.Linear(__C.HIDDEN_SIZE, __C.HIDDEN_SIZE)\n",
    "        self.linear_k = nn.Linear(__C.HIDDEN_SIZE, __C.HIDDEN_SIZE)\n",
    "        self.linear_q = nn.Linear(__C.HIDDEN_SIZE, __C.HIDDEN_SIZE)\n",
    "        self.linear_merge = nn.Linear(__C.HIDDEN_SIZE, __C.HIDDEN_SIZE)\n",
    "        self.dropout = nn.Dropout(__C.DROPOUT_R)\n",
    "\n",
    "    def forward(self, v, k, q, mask):\n",
    "        n_batches = q.size(0)\n",
    "        v = self.linear_v(v).view(n_batches,\n",
    "                                  -1,\n",
    "                                  self.__C.MULTI_HEAD,\n",
    "                                  self.__C.HIDDEN_SIZE_HEAD).transpose(1, 2)\n",
    "        k = self.linear_k(k).view(n_batches,\n",
    "                                  -1,\n",
    "                                  self.__C.MULTI_HEAD,\n",
    "                                  self.__C.HIDDEN_SIZE_HEAD).transpose(1, 2)\n",
    "        q = self.linear_q(q).view(n_batches,\n",
    "                                  -1,\n",
    "                                  self.__C.MULTI_HEAD,\n",
    "                                  self.__C.HIDDEN_SIZE_HEAD).transpose(1, 2)\n",
    "\n",
    "        atted = self.att(v, k, q, mask)\n",
    "        atted = atted.transpose(1, 2).contiguous().view(n_batches, -1, self.__C.HIDDEN_SIZE)\n",
    "        atted = self.linear_merge(atted)\n",
    "\n",
    "        return atted\n",
    "\n",
    "    def att(self, value, key, query, mask):\n",
    "        d_k = query.size(-1)\n",
    "        scores = torch.matmul(query, key.transpose(-2, -1))/math.sqrt(d_k)\n",
    "        scores = scores.masked_fill(mask, -1e9)\n",
    "        att_map = F.softmax(scores, dim=-1)\n",
    "        att_map = self.dropout(att_map)\n",
    "        return torch.matmul(att_map, value)\n",
    "\n",
    "# ---------------------------\n",
    "# ---- Feed Forward Nets ----\n",
    "# ---------------------------\n",
    "\n",
    "class FFN(nn.Module):\n",
    "    def __init__(self, __C):\n",
    "        super(FFN, self).__init__()\n",
    "\n",
    "        self.mlp = MLP(in_size=__C.HIDDEN_SIZE,\n",
    "                       mid_size=__C.FF_SIZE,\n",
    "                       out_size=__C.HIDDEN_SIZE,\n",
    "                       dropout_r=__C.DROPOUT_R,\n",
    "                       use_relu=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.mlp(x)\n",
    "\n",
    "# ------------------------\n",
    "# ---- Self Attention ----\n",
    "# ------------------------\n",
    "\n",
    "class SA(nn.Module):\n",
    "    def __init__(self, __C):\n",
    "        super(SA, self).__init__()\n",
    "\n",
    "        self.mhatt = MHAtt(__C)\n",
    "        self.ffn = FFN(__C)\n",
    "\n",
    "        self.dropout1 = nn.Dropout(__C.DROPOUT_R)\n",
    "        self.norm1 = LayerNorm(__C.HIDDEN_SIZE)\n",
    "\n",
    "        self.dropout2 = nn.Dropout(__C.DROPOUT_R)\n",
    "        self.norm2 = LayerNorm(__C.HIDDEN_SIZE)\n",
    "\n",
    "    def forward(self, x, x_mask):\n",
    "        x = self.norm1(x + self.dropout1(self.mhatt(x, x, x, x_mask)))\n",
    "        x = self.norm2(x + self.dropout2(self.ffn(x)))\n",
    "        return x\n",
    "\n",
    "# -------------------------------\n",
    "# ---- Self Guided Attention ----\n",
    "# -------------------------------\n",
    "\n",
    "class SGA(nn.Module):\n",
    "    def __init__(self, __C):\n",
    "        super(SGA, self).__init__()\n",
    "\n",
    "        self.mhatt1 = MHAtt(__C)\n",
    "        self.mhatt2 = MHAtt(__C)\n",
    "        self.ffn = FFN(__C)\n",
    "\n",
    "        self.dropout1 = nn.Dropout(__C.DROPOUT_R)\n",
    "        self.norm1 = LayerNorm(__C.HIDDEN_SIZE)\n",
    "\n",
    "        self.dropout2 = nn.Dropout(__C.DROPOUT_R)\n",
    "        self.norm2 = LayerNorm(__C.HIDDEN_SIZE)\n",
    "\n",
    "        self.dropout3 = nn.Dropout(__C.DROPOUT_R)\n",
    "        self.norm3 = LayerNorm(__C.HIDDEN_SIZE)\n",
    "\n",
    "    def forward(self, x, y, x_mask, y_mask):\n",
    "        x = self.norm1(x + self.dropout1(self.mhatt1(x, x, x, x_mask)))\n",
    "        x = self.norm2(x + self.dropout2(self.mhatt2(y, y, x, y_mask)))\n",
    "        x = self.norm3(x + self.dropout3(self.ffn(x)))\n",
    "        return x\n",
    "    \n",
    "class GA(nn.Module):\n",
    "    def __init__(self, __C):\n",
    "        super(GA, self).__init__()\n",
    "\n",
    "        self.mhatt2 = MHAtt(__C)\n",
    "        self.ffn = FFN(__C)\n",
    "\n",
    "        self.dropout2 = nn.Dropout(__C.DROPOUT_R)\n",
    "        self.norm2 = LayerNorm(__C.HIDDEN_SIZE)\n",
    "\n",
    "        self.dropout3 = nn.Dropout(__C.DROPOUT_R)\n",
    "        self.norm3 = LayerNorm(__C.HIDDEN_SIZE)\n",
    "\n",
    "    def forward(self, x, y, x_mask, y_mask):\n",
    "        x = self.norm2(x + self.dropout2(self.mhatt2(y, y, x, y_mask)))\n",
    "        x = self.norm3(x + self.dropout3(self.ffn(x)))\n",
    "        return x\n",
    "        \n",
    "# ------------------------------------------------\n",
    "# ---- MAC Layers Cascaded by Encoder-Decoder ----\n",
    "# ------------------------------------------------\n",
    "\n",
    "class MCA_ED(nn.Module):\n",
    "    def __init__(self, __C):\n",
    "        super(MCA_ED, self).__init__()\n",
    "\n",
    "        self.enc_list = nn.ModuleList([SA(__C) for _ in range(__C.LAYER)])\n",
    "        self.dec_list = nn.ModuleList([SGA(__C) for _ in range(__C.LAYER)])\n",
    "\n",
    "    def forward(self, x, y, x_mask, y_mask):\n",
    "        # Get hidden vector\n",
    "        for enc in self.enc_list:\n",
    "            x = enc(x, x_mask)\n",
    "        for dec in self.dec_list:\n",
    "            y = dec(y, x, y_mask, x_mask)\n",
    "        return x, y\n",
    "      \n",
    "    \n",
    "class AttFlat(nn.Module):\n",
    "    def __init__(self, __C):\n",
    "        super(AttFlat, self).__init__()\n",
    "        self.__C = __C\n",
    "\n",
    "        self.mlp = MLP(in_size=__C.HIDDEN_SIZE,\n",
    "                       mid_size=__C.FLAT_MLP_SIZE,\n",
    "                       out_size=__C.FLAT_GLIMPSES,\n",
    "                       dropout_r=__C.DROPOUT_R,\n",
    "                       use_relu=True)\n",
    "        self.linear_merge = nn.Linear(__C.HIDDEN_SIZE*__C.FLAT_GLIMPSES, __C.FLAT_OUT_SIZE)\n",
    "\n",
    "    def forward(self, x, x_mask):\n",
    "        att = self.mlp(x)\n",
    "        att = att.masked_fill(x_mask.squeeze(1).squeeze(1).unsqueeze(2), -1e9)\n",
    "        att = F.softmax(att, dim=1)\n",
    "\n",
    "        att_list = []\n",
    "        for i in range(self.__C.FLAT_GLIMPSES):\n",
    "            att_list.append(torch.sum(att[:,:,i:i+1]*x, dim=1))\n",
    "\n",
    "        x_atted = torch.cat(att_list, dim=1)\n",
    "        x_atted = self.linear_merge(x_atted)\n",
    "        return x_atted\n",
    "\n",
    "# -------------------------\n",
    "# ---- Main MCAN Model ----\n",
    "# -------------------------\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, __C, pretrained_emb, answer_size):\n",
    "        super(Net, self).__init__()\n",
    "\n",
    "        self.embedding = pretrained_emb.embeddings\n",
    "        self.label_emb = nn.Embedding(33, __C.LABEL_SIZE)\n",
    "        self.img_feat_linear = MLP(__C.IMG_FEAT_SIZE, __C.IMG_FEAT_SIZE, __C.HIDDEN_SIZE)\n",
    "        self.lang_feat_linear = nn.Linear(__C.WORD_EMBED_SIZE, __C.HIDDEN_SIZE)\n",
    "        self.backbone = MCA_ED(__C)\n",
    "\n",
    "        self.attflat_img = AttFlat(__C)\n",
    "        self.attflat_lang = AttFlat(__C)\n",
    "        \n",
    "        self.proj_norm_lang = LayerNorm(__C.FLAT_OUT_SIZE)\n",
    "        self.proj_norm_img = LayerNorm(__C.FLAT_OUT_SIZE)\n",
    "        self.proj_norm_mul = LayerNorm(__C.FLAT_OUT_SIZE)\n",
    "        self.proj_norm_dis = LayerNorm(__C.FLAT_OUT_SIZE)\n",
    "        self.proj = MLP(__C.FLAT_OUT_SIZE*4, __C.FLAT_OUT_SIZE*2, answer_size)\n",
    "\n",
    "    def forward(self, ques_ix, img_feats):\n",
    "        proj_feats = []\n",
    "        for img_feat in img_feats:\n",
    "            # Make mask\n",
    "            lang_feat_mask = self.make_mask(ques_ix.unsqueeze(2), pad_id)\n",
    "            img_feat_mask = self.make_mask(img_feat, 0)\n",
    "\n",
    "            # Pre-process Language Feature\n",
    "            lang_feat = self.embedding(ques_ix)\n",
    "            lang_feat = self.lang_feat_linear(lang_feat)\n",
    "\n",
    "            # Pre-process Image Feature\n",
    "            label_feat = self.label_emb(img_feat[:,:,0].long())\n",
    "            img_feat = torch.cat((img_feat[:,:,1:], label_feat), dim=2)\n",
    "            img_feat = self.img_feat_linear(img_feat)\n",
    "\n",
    "            # Backbone Framework\n",
    "            lang_feat, img_feat = self.backbone(lang_feat, img_feat, lang_feat_mask, img_feat_mask)\n",
    "            lang_feat = self.attflat_lang(lang_feat, lang_feat_mask)\n",
    "            img_feat = self.attflat_img(img_feat, img_feat_mask)\n",
    "            distance = torch.abs(lang_feat-img_feat)\n",
    "            \n",
    "            proj_feat = torch.cat((self.proj_norm_lang(lang_feat),\n",
    "                                   self.proj_norm_img(img_feat),\n",
    "                                   self.proj_norm_mul(lang_feat*img_feat),\n",
    "                                   self.proj_norm_dis(distance)\n",
    "                                  ), dim=1)\n",
    "            proj_feat = self.proj(proj_feat)\n",
    "            proj_feats.append(proj_feat)\n",
    "        return proj_feats\n",
    "\n",
    "    # Masking\n",
    "    def make_mask(self, feature, target):\n",
    "        return (torch.sum(torch.abs(feature), dim=-1) == target).unsqueeze(1).unsqueeze(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(model):\n",
    "    model = model.eval()\n",
    "    preds = {}\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for qid, group in tqdm(test.groupby('query_id')):\n",
    "            # prepare batch\n",
    "            tokens, features = group['token'].values.tolist(), group['features'].values.tolist()\n",
    "            max_len_f = len(max(features, key=lambda x: len(x)))\n",
    "            features = [np.concatenate((feature, np.zeros((max_len_f-feature.shape[0], feature.shape[1]))), axis=0) for feature in features]\n",
    "            # # to tensor\n",
    "            tokens = torch.LongTensor(tokens).to(device)\n",
    "            features = torch.FloatTensor(features).to(device)\n",
    "            # predict\n",
    "            out = model(tokens, (features,))[0].view(-1)\n",
    "            pred = [(pid, val) for pid, val in zip(group['product_id'].values.tolist(), out.tolist())]\n",
    "            pred.sort(key=lambda x: x[1], reverse=True)\n",
    "            preds[qid] = [pid for pid, _ in pred[:5]]\n",
    "            \n",
    "    model = model.train()\n",
    "    return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(data.Dataset):\n",
    "    def __init__(self, train_x):\n",
    "        self.train_x = train_x\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        tokens, pos_features, neg_features = self.train_x[index][0], self.train_x[index][1], self.train_x[index][2]\n",
    "        return [tokens, pos_features, neg_features]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.train_x)\n",
    "    \n",
    "def collate_fn(batch):\n",
    "    tokens, pos_features, neg_features = zip(*batch)\n",
    "    max_len_t, max_len_pf, max_len_nf = len(max(tokens, key=lambda x: len(x))), len(max(pos_features, key=lambda x: len(x))), len(max(neg_features, key=lambda x: len(x)))\n",
    "    tokens, pos_features, neg_features = [token+[pad_id]*(max_len_t-len(token)) for token in tokens], [np.concatenate((feature, np.zeros((max_len_pf-feature.shape[0], feature.shape[1]))), axis=0) for feature in pos_features], [np.concatenate((feature, np.zeros((max_len_nf-feature.shape[0], feature.shape[1]))), axis=0) for feature in neg_features]\n",
    "    return torch.LongTensor(tokens), torch.FloatTensor(pos_features), torch.FloatTensor(neg_features)\n",
    "\n",
    "def custom_schedule(optimizer, num_warmup_steps, num_training_steps, num_cycles=0.5, amplitude=0.1, last_epoch=-1):\n",
    "    \n",
    "    def lr_lambda(current_step):\n",
    "        if current_step < num_warmup_steps:\n",
    "            return float(current_step) / float(max(1, num_warmup_steps))\n",
    "        progress = 2.0 * math.pi * float(num_cycles) * float(current_step-num_warmup_steps) / float(max(1, num_training_steps-num_warmup_steps))\n",
    "        linear = float(num_training_steps-current_step) / float(max(1, num_training_steps-num_warmup_steps))\n",
    "        return abs(linear + math.sin(progress)*linear*amplitude)\n",
    "\n",
    "    return LambdaLR(optimizer, lr_lambda, last_epoch)\n",
    "\n",
    "def shuffle(x):\n",
    "    idxs = [i for i in range(x.shape[0])]\n",
    "    random.shuffle(idxs)\n",
    "    return x[idxs]\n",
    "\n",
    "def nDCG_score(preds, answers):\n",
    "    iDCG = sum([sum([np.log(2)/np.log(i+2) for i in range(min(len(answer), 5))]) \\\n",
    "                for answer in list(answers.values())])\n",
    "    DCG = sum([sum([np.log(2)/np.log(i+2) if preds[qid][i] in answers[str(qid)] else 0 \\\n",
    "                    for i in range(len(preds[qid]))]) for qid in list(preds.keys())])\n",
    "    return DCG/iDCG\n",
    "\n",
    "class FocalLoss(nn.Module):\n",
    "    def __init__(self, alpha=1, gamma=2, logits=False, reduce=True):\n",
    "        super(FocalLoss, self).__init__()\n",
    "        self.alpha = alpha\n",
    "        self.gamma = gamma\n",
    "        self.logits = logits\n",
    "        self.reduce = reduce\n",
    "\n",
    "    def forward(self, inputs, targets):\n",
    "        if self.logits:\n",
    "            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)\n",
    "        else:\n",
    "            BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)\n",
    "        pt = torch.exp(-BCE_loss)\n",
    "        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n",
    "\n",
    "        if self.reduce:\n",
    "            return torch.mean(F_loss)\n",
    "        else:\n",
    "            return F_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initializing model...\n"
     ]
    }
   ],
   "source": [
    "print('initializing model...')\n",
    "nDCGs = []\n",
    "best_nDCG = 0.0\n",
    "model = Net(__C, pretrained_emb, 1).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### prediction\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_embedding(model, ques_ix, img_feat):\n",
    "    # Make mask\n",
    "    lang_feat_mask = model.make_mask(ques_ix.unsqueeze(2), pad_id)\n",
    "    img_feat_mask = model.make_mask(img_feat, 0)\n",
    "\n",
    "    # Pre-process Language Feature\n",
    "    lang_feat = model.embedding(ques_ix)\n",
    "    lang_feat = model.lang_feat_linear(lang_feat)\n",
    "\n",
    "    # Pre-process Image Feature\n",
    "    label_feat = model.label_emb(img_feat[:,:,0].long())\n",
    "    img_feat = torch.cat((img_feat[:,:,1:], label_feat), dim=2)\n",
    "    img_feat = model.img_feat_linear(img_feat)\n",
    "\n",
    "    # Backbone Framework\n",
    "    lang_feat, img_feat = model.backbone(lang_feat, img_feat, lang_feat_mask, img_feat_mask)\n",
    "    lang_feat = model.attflat_lang(lang_feat, lang_feat_mask)\n",
    "    img_feat = model.attflat_img(img_feat, img_feat_mask)\n",
    "    distance = torch.abs(lang_feat-img_feat)\n",
    "\n",
    "    proj_feat = torch.cat((model.proj_norm_lang(lang_feat),\n",
    "                           model.proj_norm_img(img_feat),\n",
    "                           model.proj_norm_mul(lang_feat*img_feat),\n",
    "                           model.proj_norm_dis(distance)\n",
    "                          ), dim=1)\n",
    "    return proj_feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cls(model, n_splits):\n",
    "    model.eval()\n",
    "    qids = [[qid] for qid, _ in test.groupby('query_id')]\n",
    "    kf = KFold(n_splits=n_splits, shuffle=True)\n",
    "    train_x, train_y = [], []\n",
    "    test_x, test_y = [], []\n",
    "    qid2fold = {qids[idx][0]: i \\\n",
    "                for i, (train_index, test_index) in enumerate(kf.split(qids)) \\\n",
    "                for idx in test_index}\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for qid, group in tqdm(test.groupby('query_id')):\n",
    "            # prepare batch\n",
    "            tokens, features = group['token'].values.tolist(), group['features'].values.tolist()\n",
    "            max_len_f = len(max(features, key=lambda x: len(x)))\n",
    "            features = [np.concatenate((feature, np.zeros((max_len_f-feature.shape[0], feature.shape[1]))), axis=0) for feature in features]\n",
    "            # # to tensor\n",
    "            tokens = torch.LongTensor(tokens).to(device)\n",
    "            features = torch.FloatTensor(features).to(device)\n",
    "            # predict\n",
    "            tmp_x = extract_embedding(model, tokens, features).tolist()\n",
    "            tmp_y = group['target'].values.tolist()\n",
    "            # only use first fold\n",
    "            if qid2fold[qid]:\n",
    "                train_x += tmp_x\n",
    "                train_y += tmp_y\n",
    "            else:\n",
    "                test_x += tmp_x\n",
    "                test_y += tmp_y\n",
    "    \n",
    "    train_x, train_y = np.array(train_x), np.array(train_y)\n",
    "    test_x, test_y = np.array(test_x), np.array(test_y)\n",
    "    print('train:test = {}:{}'.format(train_x.shape[0], test_x.shape[0]))\n",
    "    cls = LGBMClassifier(random_state=0, n_jobs=24)\n",
    "    cls.fit(train_x, train_y,\n",
    "            eval_set=[(test_x, test_y)],\n",
    "            early_stopping_rounds=100,\n",
    "            verbose=100)\n",
    "        \n",
    "    return cls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(model, test, thd, cls):\n",
    "    model.eval()\n",
    "    counts = Counter(test['product_id'].values.tolist())\n",
    "    preds = {}\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for qid, group in tqdm(test.groupby('query_id')):\n",
    "            # prepare batch\n",
    "            tokens, features = group['token'].values.tolist(), group['features'].values.tolist()\n",
    "            max_len_f = len(max(features, key=lambda x: len(x)))\n",
    "            features = [np.concatenate((feature, np.zeros((max_len_f-feature.shape[0], feature.shape[1]))), axis=0) for feature in features]\n",
    "            # # to tensor\n",
    "            tokens = torch.LongTensor(tokens).to(device)\n",
    "            features = torch.FloatTensor(features).to(device)\n",
    "            # predict\n",
    "            embeddings = np.array(extract_embedding(model, tokens, features).tolist())\n",
    "            out = cls.predict_proba(embeddings)[:,1]\n",
    "            pred = [(pid, val) for pid, val in zip(group['product_id'].values.tolist(), out.tolist())]\n",
    "            thd_tmp = thd\n",
    "            pred2 = [x for x in pred if counts[x[0]] <= thd_tmp]\n",
    "            while len(pred2) < 5:\n",
    "                thd_tmp += 1\n",
    "                pred2 = [x for x in pred if counts[x[0]] <= thd_tmp]\n",
    "            pred2.sort(key=lambda x: x[1], reverse=True)\n",
    "            preds[qid] = [pid for pid, _ in pred2[:5]]\n",
    "            \n",
    "    return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_all(model, test, pad_len, cls):\n",
    "    model.eval()\n",
    "    preds = {}\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for qid, group in tqdm(test.groupby('query_id')):\n",
    "            # prepare batch\n",
    "            tokens, features = group['token'].values.tolist(), group['features'].values.tolist()\n",
    "            max_len_f = len(max(features, key=lambda x: len(x)))\n",
    "            features = [np.concatenate((feature, np.zeros((max_len_f-feature.shape[0], feature.shape[1]))), axis=0) for feature in features]\n",
    "            # # to tensor\n",
    "            tokens = torch.LongTensor(tokens).to(device)\n",
    "            features = torch.FloatTensor(features).to(device)\n",
    "            # predict\n",
    "            embeddings = np.array(extract_embedding(model, tokens, features).tolist())\n",
    "            out = cls.predict_proba(embeddings)[:,1]\n",
    "            pred = [(pid, val) for pid, val in zip(group['product_id'].values.tolist(), out.tolist())]\n",
    "            pred.sort(key=lambda x: x[1], reverse=True)\n",
    "            assert len(pred) <= pad_len\n",
    "            pid, score = [p for p, s in pred], [s for p, s in pred]\n",
    "            pid, score = pid+[np.nan]*(pad_len-len(pred)), score+[np.nan]*(pad_len-len(pred))\n",
    "            preds[qid] = pid+score\n",
    "            \n",
    "    return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed: 5119; fold: 30\n",
      "seed: 5119; fold: 31\n",
      "seed: 5119; fold: 32\n",
      "seed: 5119; fold: 33\n",
      "seed: 5119; fold: 34\n",
      "seed: 5119; fold: 35\n",
      "seed: 5119; fold: 36\n",
      "seed: 5119; fold: 37\n",
      "seed: 5119; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b678b874989845a29fe9bbc98363ce60",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13233:1487\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.358975\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[98]\tvalid_0's binary_logloss: 0.358509\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35c99aec1bc746f2a9467b527fa06d16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9668546708950472\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29fc3bc27b70487d972186caa20d130d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 5119; fold: 39\n",
      "time consumed: 5 min 47 sec\n",
      "seed: 519; fold: 30\n",
      "seed: 519; fold: 31\n",
      "seed: 519; fold: 32\n",
      "seed: 519; fold: 33\n",
      "seed: 519; fold: 34\n",
      "seed: 519; fold: 35\n",
      "seed: 519; fold: 36\n",
      "seed: 519; fold: 37\n",
      "seed: 519; fold: 38\n",
      "seed: 519; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66a7e2dd70804c4f99e291f08ab74bcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13234:1486\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.298493\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.298493\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4bdbe2a8a88483f926b68e3ebc90fe0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9772368862445103\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4988cfdc9cd84d9eb2ffc32910187569",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 5 min 39 sec\n",
      "seed: 123; fold: 30\n",
      "seed: 123; fold: 31\n",
      "seed: 123; fold: 32\n",
      "seed: 123; fold: 33\n",
      "seed: 123; fold: 34\n",
      "seed: 123; fold: 35\n",
      "seed: 123; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f618c735c34a4cc5a8e21aea20fb6188",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13256:1464\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.260583\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.260583\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "291d3e393f394a5d968e0fb7a55cf21b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9850638706466852\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a70ea12a13b4bbab5cf5e91ff74bfba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 123; fold: 37\n",
      "seed: 123; fold: 38\n",
      "seed: 123; fold: 39\n",
      "time consumed: 5 min 51 sec\n",
      "seed: 1234; fold: 30\n",
      "seed: 1234; fold: 31\n",
      "seed: 1234; fold: 32\n",
      "seed: 1234; fold: 33\n",
      "seed: 1234; fold: 34\n",
      "seed: 1234; fold: 35\n",
      "seed: 1234; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59d2e5bae8e04ac2a55b2405083b667e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13255:1465\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.380677\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[51]\tvalid_0's binary_logloss: 0.377051\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b88f76ed177e47659010a9136f8a59ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.957552483105929\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df3191bf13644c80a493758fa77ae864",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 1234; fold: 37\n",
      "seed: 1234; fold: 38\n",
      "seed: 1234; fold: 39\n",
      "time consumed: 5 min 55 sec\n",
      "seed: 12345; fold: 30\n",
      "seed: 12345; fold: 31\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7629c71fc81447419433414ff199d92d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13264:1456\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.301551\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.301551\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11cbfe62a53c4f6382bfdbe87303167a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9767271827333706\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e70de936ba1d492a8ba9b997be785084",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 12345; fold: 32\n",
      "seed: 12345; fold: 33\n",
      "seed: 12345; fold: 34\n",
      "seed: 12345; fold: 35\n",
      "seed: 12345; fold: 36\n",
      "seed: 12345; fold: 37\n",
      "seed: 12345; fold: 38\n",
      "seed: 12345; fold: 39\n",
      "time consumed: 5 min 58 sec\n",
      "seed: 1213; fold: 30\n",
      "seed: 1213; fold: 31\n",
      "seed: 1213; fold: 32\n",
      "seed: 1213; fold: 33\n",
      "seed: 1213; fold: 34\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27e8e98f58a445c8898430809f6751b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13246:1474\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.273055\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.273055\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01b68a7d7a4c4eb4adfe9000197cf278",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9785844254372659\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a14309246714819b0b557ae90a3be34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 1213; fold: 35\n",
      "seed: 1213; fold: 36\n",
      "seed: 1213; fold: 37\n",
      "seed: 1213; fold: 38\n",
      "seed: 1213; fold: 39\n",
      "time consumed: 5 min 57 sec\n",
      "seed: 207; fold: 30\n",
      "seed: 207; fold: 31\n",
      "seed: 207; fold: 32\n",
      "seed: 207; fold: 33\n",
      "seed: 207; fold: 34\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4aa9ef5c21d441d8b99822e911c62d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13245:1475\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.342255\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[56]\tvalid_0's binary_logloss: 0.34192\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "822f3b2e2bb74503b9241fcb21757b48",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9646527575844526\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54611200a2e84f1c86cb57badbffe94c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 207; fold: 35\n",
      "seed: 207; fold: 36\n",
      "seed: 207; fold: 37\n",
      "seed: 207; fold: 38\n",
      "seed: 207; fold: 39\n",
      "time consumed: 5 min 52 sec\n",
      "seed: 1333; fold: 30\n",
      "seed: 1333; fold: 31\n",
      "seed: 1333; fold: 32\n",
      "seed: 1333; fold: 33\n",
      "seed: 1333; fold: 34\n",
      "seed: 1333; fold: 35\n",
      "seed: 1333; fold: 36\n",
      "seed: 1333; fold: 37\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ada89dba7b9940359559d7b5735d0465",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13235:1485\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.276593\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[96]\tvalid_0's binary_logloss: 0.276581\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74723e6756a1469e8bb6ff0e1f9bae4c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9813497558658116\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c2694441f0640b48e6f611afd1df280",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 1333; fold: 38\n",
      "seed: 1333; fold: 39\n",
      "time consumed: 6 min 2 sec\n",
      "seed: 2020; fold: 30\n",
      "seed: 2020; fold: 31\n",
      "seed: 2020; fold: 32\n",
      "seed: 2020; fold: 33\n",
      "seed: 2020; fold: 34\n",
      "seed: 2020; fold: 35\n",
      "seed: 2020; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97e56dc53aae4fefb7f164a59e988971",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.259725\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[86]\tvalid_0's binary_logloss: 0.259545\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d49bbf8925c64f8ea67bb79245a777d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9831765574957253\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95ef2030afc34602b81670db6af6355c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 2020; fold: 37\n",
      "seed: 2020; fold: 38\n",
      "seed: 2020; fold: 39\n",
      "time consumed: 5 min 55 sec\n",
      "seed: 1115; fold: 30\n",
      "seed: 1115; fold: 31\n",
      "seed: 1115; fold: 32\n",
      "seed: 1115; fold: 33\n",
      "seed: 1115; fold: 34\n",
      "seed: 1115; fold: 35\n",
      "seed: 1115; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0631cb2b32f94e1db62c53326c6e8815",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13233:1487\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.316497\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's binary_logloss: 0.316381\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bcf399912df544f3a81320e70972c724",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9782751190526735\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6741c3d1bea74eb8bf58322cde7797c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 1115; fold: 37\n",
      "seed: 1115; fold: 38\n",
      "seed: 1115; fold: 39\n",
      "time consumed: 5 min 48 sec\n",
      "seed: 666; fold: 30\n",
      "seed: 666; fold: 31\n",
      "seed: 666; fold: 32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed22363c64ff4cea98ef8384b506a311",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13241:1479\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.362539\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[84]\tvalid_0's binary_logloss: 0.359684\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69c90a281110421388dc9e897aea1d74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9670164798033215\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b0c52e312bd41ecaf81de4ea821760f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 666; fold: 33\n",
      "seed: 666; fold: 34\n",
      "seed: 666; fold: 35\n",
      "seed: 666; fold: 36\n",
      "seed: 666; fold: 37\n",
      "seed: 666; fold: 38\n",
      "seed: 666; fold: 39\n",
      "time consumed: 5 min 51 sec\n",
      "seed: 2574; fold: 30\n",
      "seed: 2574; fold: 31\n",
      "seed: 2574; fold: 32\n",
      "seed: 2574; fold: 33\n",
      "seed: 2574; fold: 34\n",
      "seed: 2574; fold: 35\n",
      "seed: 2574; fold: 36\n",
      "seed: 2574; fold: 37\n",
      "seed: 2574; fold: 38\n",
      "seed: 2574; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0722a12abe8443c876cd780a08798c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13257:1463\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.297542\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[92]\tvalid_0's binary_logloss: 0.296735\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc02e721a0274c2c8f36ba34722ac52c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9743219301517267\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab4a4c0b4756470b875fdf73d5b645ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 5 min 47 sec\n",
      "seed: 89983; fold: 30\n",
      "seed: 89983; fold: 31\n",
      "seed: 89983; fold: 32\n",
      "seed: 89983; fold: 33\n",
      "seed: 89983; fold: 34\n",
      "seed: 89983; fold: 35\n",
      "seed: 89983; fold: 36\n",
      "seed: 89983; fold: 37\n",
      "seed: 89983; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bff09900b5db46c1af57d83f34c955be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13231:1489\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.312401\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[93]\tvalid_0's binary_logloss: 0.312102\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb14bdfad1ba4f649be45e50c81f3681",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9732788997482158\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a818e138a73b4235944d49aeba676390",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 89983; fold: 39\n",
      "time consumed: 5 min 57 sec\n",
      "seed: 46555; fold: 30\n",
      "seed: 46555; fold: 31\n",
      "seed: 46555; fold: 32\n",
      "seed: 46555; fold: 33\n",
      "seed: 46555; fold: 34\n",
      "seed: 46555; fold: 35\n",
      "seed: 46555; fold: 36\n",
      "seed: 46555; fold: 37\n",
      "seed: 46555; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ccd1b150d904e7aaa656d76ec45b881",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13235:1485\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.286967\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's binary_logloss: 0.286919\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a902a107012541259d62ee49a01ddaf0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9774844592837807\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aaf99362899e405c8568b1229730344f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 46555; fold: 39\n",
      "time consumed: 5 min 45 sec\n",
      "seed: 86031; fold: 30\n",
      "seed: 86031; fold: 31\n",
      "seed: 86031; fold: 32\n",
      "seed: 86031; fold: 33\n",
      "seed: 86031; fold: 34\n",
      "seed: 86031; fold: 35\n",
      "seed: 86031; fold: 36\n",
      "seed: 86031; fold: 37\n",
      "seed: 86031; fold: 38\n",
      "seed: 86031; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63d43e375f6b472b959cc1febf49def6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13221:1499\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.254539\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.254539\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44d5f66b41dc42f98686201dc19e76fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9797896363024714\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6735b71b33154e3ba9e034d6e3bba27b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 5 min 52 sec\n",
      "seed: 7414; fold: 30\n",
      "seed: 7414; fold: 31\n",
      "seed: 7414; fold: 32\n",
      "seed: 7414; fold: 33\n",
      "seed: 7414; fold: 34\n",
      "seed: 7414; fold: 35\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "144dd15822a84dfea2220d3b746fdab7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13236:1484\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.371815\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[51]\tvalid_0's binary_logloss: 0.36854\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8f2aa9e78e94d60bdb39a9919602677",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9557871872094567\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d2c5ec980bb476885fb01f2c5ee194b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 7414; fold: 36\n",
      "seed: 7414; fold: 37\n",
      "seed: 7414; fold: 38\n",
      "seed: 7414; fold: 39\n",
      "time consumed: 5 min 50 sec\n",
      "seed: 71438; fold: 30\n",
      "seed: 71438; fold: 31\n",
      "seed: 71438; fold: 32\n",
      "seed: 71438; fold: 33\n",
      "seed: 71438; fold: 34\n",
      "seed: 71438; fold: 35\n",
      "seed: 71438; fold: 36\n",
      "seed: 71438; fold: 37\n",
      "seed: 71438; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2c9430db97c45a290581b7b6ca1764f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13237:1483\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.293864\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.293864\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "815fb7811dee462d81eab36e37a07718",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9758548321740006\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4170900758345149bf70ac14153f742",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 71438; fold: 39\n",
      "time consumed: 5 min 49 sec\n",
      "seed: 777; fold: 30\n",
      "seed: 777; fold: 31\n",
      "seed: 777; fold: 32\n",
      "seed: 777; fold: 33\n",
      "seed: 777; fold: 34\n",
      "seed: 777; fold: 35\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71fa29abf0ea40a697ab68185e4f9e6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13233:1487\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.332262\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[98]\tvalid_0's binary_logloss: 0.331262\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7419c863189417d9d192799e044a3e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9723842359438907\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68228fc7b3f14968af92b9d7c6459c78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 777; fold: 36\n",
      "seed: 777; fold: 37\n",
      "seed: 777; fold: 38\n",
      "seed: 777; fold: 39\n",
      "time consumed: 5 min 49 sec\n",
      "seed: 87; fold: 30\n",
      "seed: 87; fold: 31\n",
      "seed: 87; fold: 32\n",
      "seed: 87; fold: 33\n",
      "seed: 87; fold: 34\n",
      "seed: 87; fold: 35\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55c239aec40a44219c3605ff406b8e31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13247:1473\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.311204\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.311204\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c8025836008418386cab2870eac7314",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9754422446190516\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df7a5f088f404661bfa20cd62d5747e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 87; fold: 36\n",
      "seed: 87; fold: 37\n",
      "seed: 87; fold: 38\n",
      "seed: 87; fold: 39\n",
      "time consumed: 5 min 54 sec\n",
      "seed: 8787; fold: 30\n",
      "seed: 8787; fold: 31\n",
      "seed: 8787; fold: 32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69e58d1046a2477f9c3277ec57eb1f53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13232:1488\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.331539\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[84]\tvalid_0's binary_logloss: 0.330556\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45356cf3423d42fca38f92f46e28ae22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9741910787388972\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e709485346b4a3094b27d3333d63f17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 8787; fold: 33\n",
      "seed: 8787; fold: 34\n",
      "seed: 8787; fold: 35\n",
      "seed: 8787; fold: 36\n",
      "seed: 8787; fold: 37\n",
      "seed: 8787; fold: 38\n",
      "seed: 8787; fold: 39\n",
      "time consumed: 5 min 50 sec\n",
      "seed: 878787; fold: 30\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ab13bcd51544e75b320e60109a72ea8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13233:1487\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.293246\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[98]\tvalid_0's binary_logloss: 0.293094\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0425d49c4e204b749a64d33d17424a28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9808908422057437\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "232adef47d5a4e49b56da9a8602a7990",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 878787; fold: 31\n",
      "seed: 878787; fold: 32\n",
      "seed: 878787; fold: 33\n",
      "seed: 878787; fold: 34\n",
      "seed: 878787; fold: 35\n",
      "seed: 878787; fold: 36\n",
      "seed: 878787; fold: 37\n",
      "seed: 878787; fold: 38\n",
      "seed: 878787; fold: 39\n",
      "time consumed: 5 min 50 sec\n",
      "seed: 800061; fold: 30\n",
      "seed: 800061; fold: 31\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91b117561d0449b3bbca822ebc74bc3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13256:1464\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.381954\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[63]\tvalid_0's binary_logloss: 0.372951\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8588e7f1b01242f68f86c5d38f0f7a93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9668945885648619\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a835223346b4fe8acf8cefc90d8f803",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 800061; fold: 32\n",
      "seed: 800061; fold: 33\n",
      "seed: 800061; fold: 34\n",
      "seed: 800061; fold: 35\n",
      "seed: 800061; fold: 36\n",
      "seed: 800061; fold: 37\n",
      "seed: 800061; fold: 38\n",
      "seed: 800061; fold: 39\n",
      "time consumed: 5 min 56 sec\n",
      "seed: 856710; fold: 30\n",
      "seed: 856710; fold: 31\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb25568ba3d94ed198f1ea241326a741",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13241:1479\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.288739\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.288739\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbba0c55c26b41299e3fbd8e583a7997",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9753170410724243\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6dcf79501fac4c71a382b3e77c930308",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 856710; fold: 32\n",
      "seed: 856710; fold: 33\n",
      "seed: 856710; fold: 34\n",
      "seed: 856710; fold: 35\n",
      "seed: 856710; fold: 36\n",
      "seed: 856710; fold: 37\n",
      "seed: 856710; fold: 38\n",
      "seed: 856710; fold: 39\n",
      "time consumed: 5 min 50 sec\n",
      "seed: 42; fold: 30\n",
      "seed: 42; fold: 31\n",
      "seed: 42; fold: 32\n",
      "seed: 42; fold: 33\n",
      "seed: 42; fold: 34\n",
      "seed: 42; fold: 35\n",
      "seed: 42; fold: 36\n",
      "seed: 42; fold: 37\n",
      "seed: 42; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2845da6ca96d40479a080511cf7b7ce7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13233:1487\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.287232\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.287232\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a89e9c00a5a24222bb2ad651b4e170db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9781608769258074\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cb2b34214b54f278770b01d33e02cab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 42; fold: 39\n",
      "time consumed: 5 min 55 sec\n",
      "seed: 426; fold: 30\n",
      "seed: 426; fold: 31\n",
      "seed: 426; fold: 32\n",
      "seed: 426; fold: 33\n",
      "seed: 426; fold: 34\n",
      "seed: 426; fold: 35\n",
      "seed: 426; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c8580f9c9014298a394310d97c5e376",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13237:1483\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.31521\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[80]\tvalid_0's binary_logloss: 0.312358\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "181e656a64fe4ad4a49ea645933fbb6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9720805958142639\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7b40c7d66a940baa10dc9c8be4e56d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 426; fold: 37\n",
      "seed: 426; fold: 38\n",
      "seed: 426; fold: 39\n",
      "time consumed: 5 min 51 sec\n",
      "seed: 64; fold: 30\n",
      "seed: 64; fold: 31\n",
      "seed: 64; fold: 32\n",
      "seed: 64; fold: 33\n",
      "seed: 64; fold: 34\n",
      "seed: 64; fold: 35\n",
      "seed: 64; fold: 36\n",
      "seed: 64; fold: 37\n",
      "seed: 64; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0a577d948804465befa72a88e066c80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.313534\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.313534\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4339f1d4cbdf4c0aa1750680b4fe8549",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.972336297609063\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "345614c10e24404cbd8342bc1311d61a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 64; fold: 39\n",
      "time consumed: 6 min 18 sec\n",
      "seed: 8864; fold: 30\n",
      "seed: 8864; fold: 31\n",
      "seed: 8864; fold: 32\n",
      "seed: 8864; fold: 33\n",
      "seed: 8864; fold: 34\n",
      "seed: 8864; fold: 35\n",
      "seed: 8864; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2590d0ea43d4e1aaa02c770f6e7387d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13227:1493\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.292039\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[85]\tvalid_0's binary_logloss: 0.291388\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7422649a351405e9525aa0545fae9cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9736041100159571\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d226b581470e4fa8bc2b49dae972022c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 8864; fold: 37\n",
      "seed: 8864; fold: 38\n",
      "seed: 8864; fold: 39\n",
      "time consumed: 5 min 50 sec\n",
      "seed: 26; fold: 30\n",
      "seed: 26; fold: 31\n",
      "seed: 26; fold: 32\n",
      "seed: 26; fold: 33\n",
      "seed: 26; fold: 34\n",
      "seed: 26; fold: 35\n",
      "seed: 26; fold: 36\n",
      "seed: 26; fold: 37\n",
      "seed: 26; fold: 38\n",
      "seed: 26; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffa616d042c64e4a8a6227f8a29f586f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.306389\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[85]\tvalid_0's binary_logloss: 0.304296\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "acfc806d814642149d33a71b2b74eaa0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9776572296118154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "faae752f623c4b70976a7cf5f3108dfd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 5 min 58 sec\n",
      "seed: 7382; fold: 30\n",
      "seed: 7382; fold: 31\n",
      "seed: 7382; fold: 32\n",
      "seed: 7382; fold: 33\n",
      "seed: 7382; fold: 34\n",
      "seed: 7382; fold: 35\n",
      "seed: 7382; fold: 36\n",
      "seed: 7382; fold: 37\n",
      "seed: 7382; fold: 38\n",
      "seed: 7382; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee05ae1ad4014586b83d9959ef5e1158",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.348791\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[78]\tvalid_0's binary_logloss: 0.346862\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0493c2c6d9eb4370a4eb5eb80ba9b896",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9682932891621113\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9945f94c08024d71a92d4c3258f97667",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 5 min 55 sec\n",
      "seed: 1010; fold: 30\n",
      "seed: 1010; fold: 31\n",
      "seed: 1010; fold: 32\n",
      "seed: 1010; fold: 33\n",
      "seed: 1010; fold: 34\n",
      "seed: 1010; fold: 35\n",
      "seed: 1010; fold: 36\n",
      "seed: 1010; fold: 37\n",
      "seed: 1010; fold: 38\n",
      "seed: 1010; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62cedac83de44e4c8303a13881059531",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13221:1499\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.283241\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[89]\tvalid_0's binary_logloss: 0.282242\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f46645e99bce42bbbf085088b50a8092",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9758491654557846\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4dd6cbcd9d44fbab5fc219985d8c732",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 5 min 53 sec\n",
      "seed: 1001; fold: 30\n",
      "seed: 1001; fold: 31\n",
      "seed: 1001; fold: 32\n",
      "seed: 1001; fold: 33\n",
      "seed: 1001; fold: 34\n",
      "seed: 1001; fold: 35\n",
      "seed: 1001; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd1a01dc51bc44ae909daaabafcc317b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13231:1489\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.314458\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[94]\tvalid_0's binary_logloss: 0.313618\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f29e9ec5a5346f59e744a6921ff3656",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9715829863086569\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbec57fa264748fea23525a9be6e4e69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 1001; fold: 37\n",
      "seed: 1001; fold: 38\n",
      "seed: 1001; fold: 39\n",
      "time consumed: 5 min 51 sec\n",
      "seed: 2330; fold: 30\n",
      "seed: 2330; fold: 31\n",
      "seed: 2330; fold: 32\n",
      "seed: 2330; fold: 33\n",
      "seed: 2330; fold: 34\n",
      "seed: 2330; fold: 35\n",
      "seed: 2330; fold: 36\n",
      "seed: 2330; fold: 37\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bc956017a07467ca7fecf71e3453641",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.38343\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[63]\tvalid_0's binary_logloss: 0.371204\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6362201effc046008bfb4c4f12a1cdd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9623417882947685\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d969f150bca493dbfab04dc8a6f7fce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 2330; fold: 38\n",
      "seed: 2330; fold: 39\n",
      "time consumed: 5 min 58 sec\n",
      "seed: 612; fold: 30\n",
      "seed: 612; fold: 31\n",
      "seed: 612; fold: 32\n",
      "seed: 612; fold: 33\n",
      "seed: 612; fold: 34\n",
      "seed: 612; fold: 35\n",
      "seed: 612; fold: 36\n",
      "seed: 612; fold: 37\n",
      "seed: 612; fold: 38\n",
      "seed: 612; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5e01ba0b95b41c4b11ba382e29a5e13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13253:1467\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.338481\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's binary_logloss: 0.338139\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "503490e561464fc58b02794f4bd6c066",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9731813046113393\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c6a2c8cec2f4ca2ae6a1d2b3fe822d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 5 min 51 sec\n",
      "seed: 24; fold: 30\n",
      "seed: 24; fold: 31\n",
      "seed: 24; fold: 32\n",
      "seed: 24; fold: 33\n",
      "seed: 24; fold: 34\n",
      "seed: 24; fold: 35\n",
      "seed: 24; fold: 36\n",
      "seed: 24; fold: 37\n",
      "seed: 24; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5199b76d59ed414d96bd04d8458d26c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13241:1479\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.315456\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[89]\tvalid_0's binary_logloss: 0.312928\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25ff7eea5a4f4ce391dc4aac417fde6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9737841583654144\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b67d390d5bc4669afbe76cf0143d880",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 24; fold: 39\n",
      "time consumed: 6 min 6 sec\n",
      "seed: 25; fold: 30\n",
      "seed: 25; fold: 31\n",
      "seed: 25; fold: 32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af5de327b97a4a85a796dcff8a427160",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13233:1487\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.307885\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.307885\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ebe7f89a2e949dc880893d7b7782092",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9742671042009101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc467a0d92504ba9946d5f9dbbaf73f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 25; fold: 33\n",
      "seed: 25; fold: 34\n",
      "seed: 25; fold: 35\n",
      "seed: 25; fold: 36\n",
      "seed: 25; fold: 37\n",
      "seed: 25; fold: 38\n",
      "seed: 25; fold: 39\n",
      "time consumed: 6 min 21 sec\n",
      "seed: 2077; fold: 30\n",
      "seed: 2077; fold: 31\n",
      "seed: 2077; fold: 32\n",
      "seed: 2077; fold: 33\n",
      "seed: 2077; fold: 34\n",
      "seed: 2077; fold: 35\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70586b8283604c24b4f78e3ad4fdc352",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13248:1472\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.294504\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's binary_logloss: 0.293178\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1459ab6ea7e400e92e80a46f75ae77b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9767791939454669\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1d1907cda8e47f3bd94eb570fc86add",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 2077; fold: 36\n",
      "seed: 2077; fold: 37\n",
      "seed: 2077; fold: 38\n",
      "seed: 2077; fold: 39\n",
      "time consumed: 6 min 43 sec\n",
      "seed: 2049; fold: 30\n",
      "seed: 2049; fold: 31\n",
      "seed: 2049; fold: 32\n",
      "seed: 2049; fold: 33\n",
      "seed: 2049; fold: 34\n",
      "seed: 2049; fold: 35\n",
      "seed: 2049; fold: 36\n",
      "seed: 2049; fold: 37\n",
      "seed: 2049; fold: 38\n",
      "seed: 2049; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b9abe697b6a4b1083ec64934f6bc46e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13225:1495\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.314595\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's binary_logloss: 0.314227\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31c244ccf4c1435b88232f2fdeb75d02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9726399377386904\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65de8dad9f10435ca76ed55f9325b777",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 6 min 34 sec\n",
      "seed: 2045; fold: 30\n",
      "seed: 2045; fold: 31\n",
      "seed: 2045; fold: 32\n",
      "seed: 2045; fold: 33\n",
      "seed: 2045; fold: 34\n",
      "seed: 2045; fold: 35\n",
      "seed: 2045; fold: 36\n",
      "seed: 2045; fold: 37\n",
      "seed: 2045; fold: 38\n",
      "seed: 2045; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "483d9e57c03244e381e0fc32fcad99f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13231:1489\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.334002\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[98]\tvalid_0's binary_logloss: 0.333958\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8dbf856caa8e4a31a683e87504c83bdb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.968891219768062\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76d576733d1d45a9a3b76bd1f3b05829",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 6 min 31 sec\n",
      "seed: 1917; fold: 30\n",
      "seed: 1917; fold: 31\n",
      "seed: 1917; fold: 32\n",
      "seed: 1917; fold: 33\n",
      "seed: 1917; fold: 34\n",
      "seed: 1917; fold: 35\n",
      "seed: 1917; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0835ac3ae7864b8db1bcfcee3a24a36d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13225:1495\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.334055\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[95]\tvalid_0's binary_logloss: 0.332994\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c97985edfefe4f34b7d15ec5def4dc61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9742496074033165\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "771b6b0764404c8bb6b56c3ff6fe21b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 1917; fold: 37\n",
      "seed: 1917; fold: 38\n",
      "seed: 1917; fold: 39\n",
      "time consumed: 6 min 35 sec\n",
      "seed: 78667; fold: 30\n",
      "seed: 78667; fold: 31\n",
      "seed: 78667; fold: 32\n",
      "seed: 78667; fold: 33\n",
      "seed: 78667; fold: 34\n",
      "seed: 78667; fold: 35\n",
      "seed: 78667; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8e55a66cabb4ad3a70fcc566f819d93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13225:1495\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.275357\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's binary_logloss: 0.275271\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fbdc63ba32a4c6994436ebcbdb9b733",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9779828118962283\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55a80dacf257454eac696b91d5b39e2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 78667; fold: 37\n",
      "seed: 78667; fold: 38\n",
      "seed: 78667; fold: 39\n",
      "time consumed: 6 min 42 sec\n",
      "seed: 68654; fold: 30\n",
      "seed: 68654; fold: 31\n",
      "seed: 68654; fold: 32\n",
      "seed: 68654; fold: 33\n",
      "seed: 68654; fold: 34\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da6505bd9aa64a169c9af9c799752624",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13246:1474\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.29209\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[93]\tvalid_0's binary_logloss: 0.291794\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "843847f94e67404bab6003e9ef97269e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9773645513651997\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "980d2284bb5b4914ab31d0e6785ec9ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 68654; fold: 35\n",
      "seed: 68654; fold: 36\n",
      "seed: 68654; fold: 37\n",
      "seed: 68654; fold: 38\n",
      "seed: 68654; fold: 39\n",
      "time consumed: 6 min 35 sec\n",
      "seed: 56474; fold: 30\n",
      "seed: 56474; fold: 31\n",
      "seed: 56474; fold: 32\n",
      "seed: 56474; fold: 33\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b4c6ea735ee433a94e125a50208c8ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13245:1475\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.307595\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.307595\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6b460fd1f1447b1a8a0aef753b6f65d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9740524333464331\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fbbb1b4145744813a67618dc56a93cdb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 56474; fold: 34\n",
      "seed: 56474; fold: 35\n",
      "seed: 56474; fold: 36\n",
      "seed: 56474; fold: 37\n",
      "seed: 56474; fold: 38\n",
      "seed: 56474; fold: 39\n",
      "time consumed: 6 min 36 sec\n",
      "seed: 56464; fold: 30\n",
      "seed: 56464; fold: 31\n",
      "seed: 56464; fold: 32\n",
      "seed: 56464; fold: 33\n",
      "seed: 56464; fold: 34\n",
      "seed: 56464; fold: 35\n",
      "seed: 56464; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c740680cafbb49d38f96546cf87a2375",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.322531\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[95]\tvalid_0's binary_logloss: 0.320919\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9347070b408c40b3b32cadc321fe2fc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.973291101381014\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bb05c5c297148fda82f1b944490c4e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 56464; fold: 37\n",
      "seed: 56464; fold: 38\n",
      "seed: 56464; fold: 39\n",
      "time consumed: 6 min 43 sec\n",
      "seed: 54367; fold: 30\n",
      "seed: 54367; fold: 31\n",
      "seed: 54367; fold: 32\n",
      "seed: 54367; fold: 33\n",
      "seed: 54367; fold: 34\n",
      "seed: 54367; fold: 35\n",
      "seed: 54367; fold: 36\n",
      "seed: 54367; fold: 37\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdeb00bcffca4a968aa769a20a3594df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13229:1491\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.341672\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[63]\tvalid_0's binary_logloss: 0.339955\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b05cd362cfd4897bceb4008ee2ad518",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9644124611768526\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a0f9da785f241aaba18299b851d8f00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 54367; fold: 38\n",
      "seed: 54367; fold: 39\n",
      "time consumed: 6 min 0 sec\n",
      "seed: 4547; fold: 30\n",
      "seed: 4547; fold: 31\n",
      "seed: 4547; fold: 32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6a09d847e3f49149108fd36a09f57a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.27882\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.27882\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a438834e23574c0c9e25d61f2f07d2f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9773088562915148\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af84b7b33b674924a596154874986410",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 4547; fold: 33\n",
      "seed: 4547; fold: 34\n",
      "seed: 4547; fold: 35\n",
      "seed: 4547; fold: 36\n",
      "seed: 4547; fold: 37\n",
      "seed: 4547; fold: 38\n",
      "seed: 4547; fold: 39\n",
      "time consumed: 5 min 49 sec\n",
      "seed: 437; fold: 30\n",
      "seed: 437; fold: 31\n",
      "seed: 437; fold: 32\n",
      "seed: 437; fold: 33\n",
      "seed: 437; fold: 34\n",
      "seed: 437; fold: 35\n",
      "seed: 437; fold: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e00e16436926433bbdae3c937ee66458",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13229:1491\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.303346\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's binary_logloss: 0.303346\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cc83f8f28d244339ac1b143a8c54d18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9747711226050714\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3962295611ba4251bb0926953c3d9446",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 437; fold: 37\n",
      "seed: 437; fold: 38\n",
      "seed: 437; fold: 39\n",
      "time consumed: 6 min 1 sec\n",
      "seed: 485; fold: 30\n",
      "seed: 485; fold: 31\n",
      "seed: 485; fold: 32\n",
      "seed: 485; fold: 33\n",
      "seed: 485; fold: 34\n",
      "seed: 485; fold: 35\n",
      "seed: 485; fold: 36\n",
      "seed: 485; fold: 37\n",
      "seed: 485; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8428ad0c08134422b3df66ea904d097b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.291252\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[90]\tvalid_0's binary_logloss: 0.29097\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cb1c899135342d28d1418ef681f2f07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9765332322094041\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de7c0e8b0b644261a1a5f327dcd96b92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 485; fold: 39\n",
      "time consumed: 5 min 58 sec\n",
      "seed: 132; fold: 30\n",
      "seed: 132; fold: 31\n",
      "seed: 132; fold: 32\n",
      "seed: 132; fold: 33\n",
      "seed: 132; fold: 34\n",
      "seed: 132; fold: 35\n",
      "seed: 132; fold: 36\n",
      "seed: 132; fold: 37\n",
      "seed: 132; fold: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a91ce3ce2162468eb944d6973eac00b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13238:1482\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.304822\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[88]\tvalid_0's binary_logloss: 0.304042\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0315b4986c5437f84786859dd677ede",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9753755876623547\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65c3312721c748dbb3308b426aa997cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 132; fold: 39\n",
      "time consumed: 6 min 1 sec\n",
      "seed: 257; fold: 30\n",
      "seed: 257; fold: 31\n",
      "seed: 257; fold: 32\n",
      "seed: 257; fold: 33\n",
      "seed: 257; fold: 34\n",
      "seed: 257; fold: 35\n",
      "seed: 257; fold: 36\n",
      "seed: 257; fold: 37\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46a6ce33a8c04cab81d7aac5d42bcaae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13238:1482\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.336925\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[92]\tvalid_0's binary_logloss: 0.33522\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49051aa5a4414e50a1ffdedaacf569f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9668490046400823\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "401f9dcc6fce4e7fb22f7a53f2cbbbe6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 257; fold: 38\n",
      "seed: 257; fold: 39\n",
      "time consumed: 5 min 54 sec\n",
      "seed: 584; fold: 30\n",
      "seed: 584; fold: 31\n",
      "seed: 584; fold: 32\n",
      "seed: 584; fold: 33\n",
      "seed: 584; fold: 34\n",
      "seed: 584; fold: 35\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43c7af396fa049c88b80da9e2418b1bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13257:1463\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.343896\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's binary_logloss: 0.343525\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef8dc15fc61847dbaac668e6f0b2220a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9705793760055121\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "419fedf4ca1e4dd4b58dd5f71dc35917",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 584; fold: 36\n",
      "seed: 584; fold: 37\n",
      "seed: 584; fold: 38\n",
      "seed: 584; fold: 39\n",
      "time consumed: 5 min 59 sec\n",
      "seed: 931; fold: 30\n",
      "seed: 931; fold: 31\n",
      "seed: 931; fold: 32\n",
      "seed: 931; fold: 33\n",
      "seed: 931; fold: 34\n",
      "seed: 931; fold: 35\n",
      "seed: 931; fold: 36\n",
      "seed: 931; fold: 37\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8390d4fe275643d5a3e9253b10c57bee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13240:1480\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.304527\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[91]\tvalid_0's binary_logloss: 0.303705\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cdbe3b6ab8f4e3c9958f926e7d72203",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9769272078432468\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a8e4f084bdd42a7b1ce36f87d87fd97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 931; fold: 38\n",
      "seed: 931; fold: 39\n",
      "time consumed: 6 min 2 sec\n",
      "seed: 792; fold: 30\n",
      "seed: 792; fold: 31\n",
      "seed: 792; fold: 32\n",
      "seed: 792; fold: 33\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b520078c3a56460ba82318d0cc4b9bb8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13233:1487\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.323444\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[72]\tvalid_0's binary_logloss: 0.321096\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0d1865ee8584e5c8d567842c1c2aa62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9729899232887005\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1304a7a805f14776ba86d827e98221c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 792; fold: 34\n",
      "seed: 792; fold: 35\n",
      "seed: 792; fold: 36\n",
      "seed: 792; fold: 37\n",
      "seed: 792; fold: 38\n",
      "seed: 792; fold: 39\n",
      "time consumed: 6 min 0 sec\n",
      "seed: 603; fold: 30\n",
      "seed: 603; fold: 31\n",
      "seed: 603; fold: 32\n",
      "seed: 603; fold: 33\n",
      "seed: 603; fold: 34\n",
      "seed: 603; fold: 35\n",
      "seed: 603; fold: 36\n",
      "seed: 603; fold: 37\n",
      "seed: 603; fold: 38\n",
      "seed: 603; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8bda38c114a4289b5d6188b25ae99ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.269061\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[96]\tvalid_0's binary_logloss: 0.268554\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9628a27a4a5f44e3b175adb50b71a7a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9793339273233074\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba6825bad1b147a0941757ab69747503",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 6 min 14 sec\n",
      "seed: 746; fold: 30\n",
      "seed: 746; fold: 31\n",
      "seed: 746; fold: 32\n",
      "seed: 746; fold: 33\n",
      "seed: 746; fold: 34\n",
      "seed: 746; fold: 35\n",
      "seed: 746; fold: 36\n",
      "seed: 746; fold: 37\n",
      "seed: 746; fold: 38\n",
      "seed: 746; fold: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4de056231fce4e068341484ac796b284",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13287:1433\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.329716\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[80]\tvalid_0's binary_logloss: 0.326853\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bced32329f2d4dfe9c02dde9c59248d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.975584094692418\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb32ed1ac457417ba3a3c857d857316a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "time consumed: 6 min 57 sec\n",
      "seed: 480; fold: 30\n",
      "seed: 480; fold: 31\n",
      "seed: 480; fold: 32\n",
      "seed: 480; fold: 33\n",
      "seed: 480; fold: 34\n",
      "seed: 480; fold: 35\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cf3fd6ff9f24a90a7fc455f894b73d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train:test = 13220:1500\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's binary_logloss: 0.29443\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's binary_logloss: 0.29414\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79911f616f39467b879e7003cf9ac250",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=496.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "nDCG@5: 0.9742398673445793\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d58ebde2ee084bdb85184a8a20b7af15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=994.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "seed: 480; fold: 36\n",
      "seed: 480; fold: 37\n",
      "seed: 480; fold: 38\n",
      "seed: 480; fold: 39\n",
      "time consumed: 6 min 55 sec\n"
     ]
    }
   ],
   "source": [
    "seeds = {5119: 38, 519: 39, 123: 36, 1234: 36, 12345: 31, 1213: 34, 207: 34,\n",
    "         1333: 37, 2020: 36, 1115: 36, 666: 32, 2574: 39, 89983: 38, 46555: 38, 86031: 39,\n",
    "         7414: 35, 71438: 38, 777: 35, 87: 35, 8787: 32, 878787: 30, 800061: 31, 856710: 31,\n",
    "         42: 38, 426: 36, 64: 38, 8864: 36, 26: 39, 7382: 39, 1010: 39, 1001: 36,\n",
    "         2330: 37, 612: 39, 24: 38, 25: 32, 2077: 35, 2049: 39, 2045: 39, 1917: 36,\n",
    "         78667: 36, 68654: 34, 56474: 33, 56464: 36, 54367: 37, 4547: 32, 437: 36, 485: 38,\n",
    "         132: 38, 257: 37, 584: 35, 931: 37, 792: 33, 603: 39, 746: 39, 480: 35}\n",
    "folds = [i for i in range(30, 40)]\n",
    "pad_len = 30\n",
    "thd = 9999\n",
    "n_splits = 10\n",
    "\n",
    "for seed in list(seeds.keys()):\n",
    "    t0 = time.time()\n",
    "    for fold in folds:\n",
    "        print('seed: {}; fold: {}'.format(seed, fold))\n",
    "        # load model weights\n",
    "        try: model.load_state_dict(torch.load(path+'models/model_MCAN-RoBERTa_pair-cat_box_tfidf-neg_focal_all_shared_{}_{}'.format(seed, fold), map_location=device))\n",
    "        except: continue\n",
    "        # train cls\n",
    "        cls = get_cls(model, n_splits)\n",
    "        # valid\n",
    "        preds = predict(model, test, thd, cls)\n",
    "        nDCG = nDCG_score(preds, answers)\n",
    "        print('nDCG@5:', nDCG)\n",
    "        # test\n",
    "        preds = predict_all(model, testA, pad_len, cls)\n",
    "        # write to file\n",
    "        header = ['qid'] + ['p'+str(i) for i in range(pad_len)] + ['s'+str(i) for i in range(pad_len)]\n",
    "        with open('predictions/prediction_all_cls2_{}_{}.csv'.format(seed, fold), 'w', newline='') as f:\n",
    "            w = csv.writer(f)\n",
    "            w.writerow(header)\n",
    "            for qid in sorted(list(preds.keys())):\n",
    "                w.writerow([qid]+preds[qid])\n",
    "    t = round(time.time()-t0)\n",
    "    print('time consumed: {} min {} sec'.format(t//60, t%60))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}