load_bertmodels.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the inference kernel for BERT pytorch. Check out the amazing Kernel for finetuning BERT by Yuval: https://www.kaggle.com/yuval6967/toxic-bert-plain-vanila"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bert2epoch', 'bert-different-layer', 'bert2new', 'bert094048', 'bert094084', 'bert094081', 'bertlb093986', 'bert094066', 'bert-config', 'bertlb094009', 'ppbert', 'jigsaw-unintended-bias-in-toxicity-classification', 'bert094071', 'pytorchpretrainedbertnew', 'bert-pretrained-models', 'bert-2-new', 'bertlb093876']\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "package_dir_new = \"../input/pytorchpretrainedbertnew/pytorch-pretrained-bert/pytorch-pretrained-BERT\"\n",
    "package_dir = \"../input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT\"\n",
    "print(os.listdir(\"../input\"))\n",
    "# print(os.listdir(package_dir))\n",
    "# print(os.listdir(\"../input/bert-different-layer\"))\n",
    "# print(os.listdir(\"../input/b/ertlb094009\"))\n",
    "# print(os.listdir(\"../input/bertlb093876\"))\n",
    "sys.path.append(package_dir)\n",
    "# sys.path.append(package_dir_new)\n",
    "# print(os.listdir(\"../input\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "import torch.utils.data\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import warnings\n",
    "import torch.nn as nn\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification, BertAdam, BertModel\n",
    "from pytorch_pretrained_bert import BertConfig\n",
    "\n",
    "warnings.filterwarnings(action='once')\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_lines(example, max_seq_length,tokenizer):\n",
    "    max_seq_length -=2\n",
    "    all_tokens = []\n",
    "    longer = 0\n",
    "    for text in tqdm(example):\n",
    "        tokens_a = tokenizer.tokenize(text)\n",
    "        if len(tokens_a)>max_seq_length:\n",
    "            tokens_a = tokens_a[:max_seq_length]\n",
    "            longer += 1\n",
    "        one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"]+tokens_a+[\"[SEP]\"])+[0] * (max_seq_length - len(tokens_a))\n",
    "        all_tokens.append(one_token)\n",
    "    return np.array(all_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 220\n",
    "SEED = 1234\n",
    "BATCH_SIZE = 32\n",
    "BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'\n",
    "\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "# bert_config = BertConfig('../input/bert-inference/bert/bert_config.json')\n",
    "bert_config = BertConfig('../input/bert-config/bert_config.json')\n",
    "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 220\n",
    "SEED = 1234\n",
    "EPOCHS = 1\n",
    "Data_dir=\"../input/jigsaw-unintended-bias-in-toxicity-classification\"\n",
    "Input_dir = \"../input\"\n",
    "WORK_DIR = \"../working/\"\n",
    "num_to_load=1000000                         #Train size to match time limit\n",
    "valid_size= 100000                          #Validation Size\n",
    "TOXICITY_COLUMN = 'target'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 97320/97320 [01:54<00:00, 846.89it/s]\n"
     ]
    }
   ],
   "source": [
    "test_df = pd.read_csv(\"../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv\")\n",
    "test_df['comment_text'] = test_df['comment_text'].astype(str) \n",
    "X_test = convert_lines(test_df[\"comment_text\"].fillna(\"DUMMY_VALUE\"), MAX_SEQUENCE_LENGTH, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomModel(nn.Module):\n",
    "    \n",
    "    def __init__(self, config, aux_number):\n",
    "        super().__init__()\n",
    "        # 加载并冻结bert模型参数\n",
    "        self.bert = BertModel(config)\n",
    "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
    "        self.w1 = torch.nn.Linear(768, 1)\n",
    "        self.w2 = torch.nn.Linear(768, aux_number)\n",
    "    def forward(self, seqs, attention_mask=None, labels=None):\n",
    "        _, seqs = self.bert(seqs, attention_mask=attention_mask, output_all_encoded_layers=False)\n",
    "        seqs = self.dropout(seqs)\n",
    "        res = self.w1(seqs)\n",
    "        res_aux = self.w2(seqs)\n",
    "        out = torch.cat((res, res_aux), dim=-1)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): BertLayerNorm()\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (classifier): Linear(in_features=768, out_features=7, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model = BertForSequenceClassification(bert_config, num_labels=6)\n",
    "# model.load_state_dict(torch.load(\"../input/bert+pre-process/bert_pytorch.bin\"))\n",
    "# model.to(device)\n",
    "#0.93898\n",
    "model = BertForSequenceClassification(bert_config, num_labels=7)\n",
    "model.load_state_dict(torch.load(\"../input/bert2new/bert_pytorch.bin\"))\n",
    "model.to(device)\n",
    "#这个是2个out的版本\n",
    "# model = CustomModel(bert_config, 6)\n",
    "# model.load_state_dict(torch.load(\"../input/bert+preprocess/bert_pytorch.bin\"))\n",
    "# model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3042/3042 [11:55<00:00,  4.25it/s]\n"
     ]
    }
   ],
   "source": [
    "test_preds = np.zeros((len(X_test)))\n",
    "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "tk0 = tqdm(test_loader)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3042/3042 [11:54<00:00,  4.26it/s]\n"
     ]
    }
   ],
   "source": [
    "# 0.94009\n",
    "model = BertForSequenceClassification(bert_config, num_labels=7)\n",
    "model.load_state_dict(torch.load(\"../input/bertlb094009/results (1)/bert_pytorch.bin\"))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()\n",
    "\n",
    "test_preds_ = np.zeros((len(X_test)))\n",
    "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "tk0 = tqdm(test_loader)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds_[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred_ = torch.sigmoid(torch.tensor(test_preds_)).numpy().ravel()\n",
    "\n",
    "# # test_pred = (test_pred + test_pred_) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3042/3042 [11:54<00:00,  4.26it/s]\n"
     ]
    }
   ],
   "source": [
    "#0.93876\n",
    "model = CustomModel(bert_config, 6)\n",
    "model.load_state_dict(torch.load(\"../input/bertlb093876/results/bert_pytorch.bin\"))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()\n",
    "\n",
    "test_preds_2 = np.zeros((len(X_test)))\n",
    "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "tk0 = tqdm(test_loader)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds_2[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred_2 = torch.sigmoid(torch.tensor(test_preds_2)).numpy().ravel()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sys.path.append(package_dir_new)\n",
    "# #0.93986\n",
    "# from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification, BertAdam, BertModel\n",
    "# from pytorch_pretrained_bert import BertConfig\n",
    "# model = BertForSequenceClassification(bert_config, num_labels=6)\n",
    "# model.load_state_dict(torch.load(\"../input/bert+pre-process/bert_pytorch.bin\"))\n",
    "# model.to(device)\n",
    "# for param in model.parameters():\n",
    "#     param.requires_grad = False\n",
    "# model.eval()\n",
    "\n",
    "# test_preds_3 = np.zeros((len(X_test)))\n",
    "# test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "# test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "# tk0 = tqdm(test_loader)\n",
    "# for i, (x_batch,) in enumerate(tk0):\n",
    "#     pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "#     test_preds_3[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "# test_pred_3 = torch.sigmoid(torch.tensor(test_preds_3)).numpy().ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3042/3042 [11:54<00:00,  4.26it/s]\n"
     ]
    }
   ],
   "source": [
    "# \n",
    "\n",
    "model = BertForSequenceClassification(bert_config, num_labels=7)\n",
    "model.load_state_dict(torch.load(\"../input/bert094048/results/bert_pytorch.bin\"))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()\n",
    "\n",
    "test_preds_4 = np.zeros((len(X_test)))\n",
    "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "tk0 = tqdm(test_loader)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds_4[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred_4 = torch.sigmoid(torch.tensor(test_preds_4)).numpy().ravel()\n",
    "\n",
    "# test_pred = (test_pred + test_pred_) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomModel_diff(nn.Module):\n",
    "    \n",
    "    def __init__(self, config, aux_number):\n",
    "        super().__init__()\n",
    "        # 加载并冻结bert模型参数\n",
    "        self.bert = BertModel(config)\n",
    "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
    "#         self.w1 = torch.nn.Linear(768, 1)\n",
    "        self.w1 = torch.nn.Linear(768 * 2, aux_number)\n",
    "    def forward(self, seqs, attention_mask=None, labels=None):\n",
    "        seqs_list, seqs = self.bert(seqs, attention_mask=attention_mask, output_all_encoded_layers=True)\n",
    "        seqs_ = seqs_list[-1]\n",
    "        bert_seqs = seqs_\n",
    "        avg_pool = torch.mean(bert_seqs, 1)\n",
    "        # global max pooling\n",
    "        max_pool, _ = torch.max(bert_seqs, 1)\n",
    "        \n",
    "        #[batch, dim_bert * 2]\n",
    "        h_conc = torch.cat((max_pool, avg_pool), 1)\n",
    "        out = self.w1(h_conc)\n",
    "        \n",
    "#         seqs = self.dropout(seqs)\n",
    "#         res = self.w1(seqs)\n",
    "#         res_aux = self.w2(seqs)\n",
    "#         out = torch.cat((res, res_aux), dim=-1)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # model = CustomModel('../working', bert_config, 7)\n",
    "# model = CustomModel_diff(bert_config, 7)\n",
    "# model.load_state_dict(torch.load(\"../input/bert-different-layer/bert_pytorch.bin\"))\n",
    "# model.to(device)\n",
    "# for param in model.parameters():\n",
    "#     param.requires_grad = False\n",
    "# model.eval()\n",
    "\n",
    "# test_preds_5 = np.zeros((len(X_test)))\n",
    "# test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "# test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "# tk0 = tqdm(test_loader)\n",
    "# for i, (x_batch,) in enumerate(tk0):\n",
    "#     pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "#     test_preds_5[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "# test_pred_5 = torch.sigmoid(torch.tensor(test_preds_5)).numpy().ravel()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3042/3042 [11:54<00:00,  4.26it/s]\n"
     ]
    }
   ],
   "source": [
    "# 0.94066\n",
    "\n",
    "model = BertForSequenceClassification(bert_config, num_labels=7)\n",
    "model.load_state_dict(torch.load(\"../input/bert094066/results (1)/bert_pytorch.bin\"))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()\n",
    "\n",
    "test_preds_5 = np.zeros((len(X_test)))\n",
    "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "tk0 = tqdm(test_loader)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds_5[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred_5 = torch.sigmoid(torch.tensor(test_preds_5)).numpy().ravel()\n",
    "\n",
    "# test_pred = (test_pred + test_pred_) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3042/3042 [11:54<00:00,  4.26it/s]\n"
     ]
    }
   ],
   "source": [
    "# 0.94084\n",
    "\n",
    "model = BertForSequenceClassification(bert_config, num_labels=7)\n",
    "model.load_state_dict(torch.load(\"../input/bert094084/results/bert_pytorch.bin\"))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()\n",
    "\n",
    "test_preds_6 = np.zeros((len(X_test)))\n",
    "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "tk0 = tqdm(test_loader)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds_6[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred_6 = torch.sigmoid(torch.tensor(test_preds_6)).numpy().ravel()\n",
    "\n",
    "# test_pred = (test_pred + test_pred_) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 0.94081\n",
    "\n",
    "# model = BertForSequenceClassification(bert_config, num_labels=7)\n",
    "# model.load_state_dict(torch.load(\"../input/bert094081/results/bert_pytorch.bin\"))\n",
    "# model.to(device)\n",
    "# for param in model.parameters():\n",
    "#     param.requires_grad = False\n",
    "# model.eval()\n",
    "\n",
    "# test_preds_7 = np.zeros((len(X_test)))\n",
    "# test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "# test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "# tk0 = tqdm(test_loader)\n",
    "# for i, (x_batch,) in enumerate(tk0):\n",
    "#     pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "#     test_preds_7[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "# test_pred_7 = torch.sigmoid(torch.tensor(test_preds_7)).numpy().ravel()\n",
    "\n",
    "# # test_pred = (test_pred + test_pred_) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3042/3042 [11:54<00:00,  4.26it/s]\n"
     ]
    }
   ],
   "source": [
    "# 0.94071\n",
    "\n",
    "model = BertForSequenceClassification(bert_config, num_labels=7)\n",
    "model.load_state_dict(torch.load(\"../input/bert094071/results (1)/bert_pytorch.bin\"))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "model.eval()\n",
    "\n",
    "test_preds_8 = np.zeros((len(X_test)))\n",
    "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\n",
    "tk0 = tqdm(test_loader)\n",
    "for i, (x_batch,) in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "    test_preds_8[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "test_pred_8 = torch.sigmoid(torch.tensor(test_preds_8)).numpy().ravel()\n",
    "\n",
    "# test_pred = (test_pred + test_pred_) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame.from_dict({\n",
    "    'id': test_df['id'],\n",
    "    'prediction': np.average([test_pred, test_pred_,test_pred_2,test_pred_4, test_pred_5, test_pred_6, test_pred_8],axis=0,weights=[0.1,0.05,0.1,0.1,0.15,0.25,0.15])\n",
    "    # np.average([test_pred, test_pred_, test_pred_4, test_pred_5, test_pred_6, test_pred_7 ], axis=0) 0.94135\n",
    "    #np.average([test_pred, test_pred_, test_pred_4, test_pred_5, test_pred_6, test_pred_7 ], axis=0, weights=[0.1,0.1,0.1,0.2,0.3,0.2]\n",
    "    #np.average([test_pred_,test_pred_6, test_pred_5], axis=0, weights = [0.3, 0.5, 0.2]) 0.94110\n",
    "    #np.average([test_pred_,test_pred_6, test_pred], axis=0, weights = [0.3, 0.5, 0.2]) 0.94139\n",
    "    #np.average([test_pred_5,test_pred_4, test_pred_], axis=0, weights = [0.5, 0.3, 0.2]) 0.94092\n",
    "    #np.average([test_pred_,test_pred_4, test_pred], axis=0, weights = [0.3, 0.5, 0.2]) 0.94123\n",
    "#     np.average([test_pred_,test_pred_4], axis=0, weights = [0.3, 0.7]) 0.94085 [0.4, 0.6] 0.94098\n",
    "    #np.average([test_pred, test_pred_,test_pred_2], axis=0, weights = [0.2, 0.6, 0.2]) 0.94092\n",
    "    #np.average([test_pred, test_pred_,test_pred_2], axis=0, weights = [0.3, 0.5, 0.2]) 0.94052\n",
    "    #np.mean([test_pred, test_pred_], axis=0)\n",
    "})\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7000000</td>\n",
       "      <td>0.000225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7000001</td>\n",
       "      <td>0.000049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7000002</td>\n",
       "      <td>0.001499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7000003</td>\n",
       "      <td>0.000284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7000004</td>\n",
       "      <td>0.990843</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  prediction\n",
       "0  7000000    0.000225\n",
       "1  7000001    0.000049\n",
       "2  7000002    0.001499\n",
       "3  7000003    0.000284\n",
       "4  7000004    0.990843"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}