hybird_model_recall.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd  \n",
    "import numpy as np\n",
    "from tqdm import tqdm  \n",
    "from collections import defaultdict  \n",
    "import os, math, warnings, math, pickle\n",
    "from tqdm import tqdm\n",
    "import faiss\n",
    "import collections\n",
    "import random\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from datetime import datetime\n",
    "# from deepctr.feature_column import SparseFeat, VarLenSparseFeat\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "# from tensorflow.python.keras import backend as K\n",
    "# from tensorflow.python.keras.models import Model\n",
    "# from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# from deepmatch.models import *\n",
    "# from deepmatch.utils import sampledsoftmaxloss\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>click_article_id</th>\n",
       "      <th>click_timestamp</th>\n",
       "      <th>click_environment</th>\n",
       "      <th>click_deviceGroup</th>\n",
       "      <th>click_os</th>\n",
       "      <th>click_country</th>\n",
       "      <th>click_region</th>\n",
       "      <th>click_referrer_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>157507</td>\n",
       "      <td>1508211702520</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>30760</td>\n",
       "      <td>1508211672520</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>63746</td>\n",
       "      <td>1508211346889</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>289197</td>\n",
       "      <td>1508211316889</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>168401</td>\n",
       "      <td>1508211468695</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  click_article_id  click_timestamp  click_environment  \\\n",
       "0        0            157507    1508211702520                  4   \n",
       "1        0             30760    1508211672520                  4   \n",
       "2        1             63746    1508211346889                  4   \n",
       "3        1            289197    1508211316889                  4   \n",
       "4        2            168401    1508211468695                  4   \n",
       "\n",
       "   click_deviceGroup  click_os  click_country  click_region  \\\n",
       "0                  1        17              1            25   \n",
       "1                  1        17              1            25   \n",
       "2                  1        17              1            25   \n",
       "3                  1        17              1            25   \n",
       "4                  3        20              1            25   \n",
       "\n",
       "   click_referrer_type  \n",
       "0                    2  \n",
       "1                    2  \n",
       "2                    6  \n",
       "3                    6  \n",
       "4                    2  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train\n",
    "train_click = pd.read_csv(\"data/train_click_log.csv\")\n",
    "articles = pd.read_csv(\"data/articles.csv\")\n",
    "articles = articles.rename(columns={'article_id': 'click_article_id'})  #重命名，方便后续match\n",
    "articles_emb = pd.read_csv(\"data/articles_emb.csv\")\n",
    "\n",
    "# test\n",
    "test_click = pd.read_csv(\"data/testA_click_log.csv\")\n",
    "\n",
    "train_click = train_click.append(test_click)\n",
    "train_click = train_click.drop_duplicates((['user_id', 'click_article_id', 'click_timestamp']))\n",
    "\n",
    "train_click = train_click.sort_values('user_id')\n",
    "train_click.reset_index(drop=True, inplace=True)\n",
    "train_click.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "usercf_u2u2i_recall_dict = pickle.load(open('./hybird_model/usercf_u2u2i_recall.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "itemcf_u2i2i_recall_dict = pickle.load(open('./hybird_model/itemcf_u2i2i_recall.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cold_start_u2i_recall_dict = pickle.load(open('./hybird_model/cold_start_u2i_recall.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_dict = {'itemcf_sim_itemcf_recall': 1.0,\n",
    "               'usercf_sim_itemcf_recall': 0.5,\n",
    "               'cold_start_recall': 0.5}\n",
    "\n",
    "# 定义一个多路召回的字典，将各路召回的结果都保存在这个字典当中\n",
    "user_multi_recall_dict =  {'itemcf_sim_itemcf_recall': {},\n",
    "                           'usercf_sim_itemcf_recall': {},\n",
    "                           'cold_start_recall': {}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_multi_recall_dict['itemcf_sim_itemcf_recall'] = itemcf_u2i2i_recall_dict\n",
    "user_multi_recall_dict['usercf_sim_itemcf_recall'] = usercf_u2u2i_recall_dict\n",
    "user_multi_recall_dict['cold_start_recall'] = cold_start_u2i_recall_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_recall_results(user_multi_recall_dict, weight_dict=None, topk=5):\n",
    "    final_recall_items_dict = {}\n",
    "    \n",
    "    # 对每一种召回结果按照用户进行归一化，方便后面多种召回结果，相同用户的物品之间权重相加\n",
    "    def norm_user_recall_items_sim(sorted_item_list):\n",
    "        # 如果冷启动中没有文章或者只有一篇文章，直接返回，出现这种情况的原因可能是冷启动召回的文章数量太少了，\n",
    "        # 基于规则筛选之后就没有文章了, 这里还可以做一些其他的策略性的筛选\n",
    "        if len(sorted_item_list) < 2:\n",
    "            return sorted_item_list\n",
    "        \n",
    "        min_sim = sorted_item_list[-1][1]\n",
    "        max_sim = sorted_item_list[0][1]\n",
    "        \n",
    "        norm_sorted_item_list = []\n",
    "        for item, score in sorted_item_list:\n",
    "            if max_sim > 0:\n",
    "                norm_score = 1.0 * (score - min_sim) / (max_sim - min_sim) if max_sim > min_sim else 1.0\n",
    "            else:\n",
    "                norm_score = 0.0\n",
    "            norm_sorted_item_list.append((item, norm_score))\n",
    "            \n",
    "        return norm_sorted_item_list\n",
    "    \n",
    "    print('多路召回合并...')\n",
    "    for method, user_recall_items in tqdm(user_multi_recall_dict.items()):\n",
    "        print(method + '...')\n",
    "        # 在计算最终召回结果的时候，也可以为每一种召回结果设置一个权重\n",
    "        if weight_dict == None:\n",
    "            recall_method_weight = 1\n",
    "        else:\n",
    "            recall_method_weight = weight_dict[method]\n",
    "        \n",
    "        for user_id, sorted_item_list in user_recall_items.items(): # 进行归一化\n",
    "            user_recall_items[user_id] = norm_user_recall_items_sim(sorted_item_list)\n",
    "        \n",
    "        for user_id, sorted_item_list in user_recall_items.items():\n",
    "            # print('user_id')\n",
    "            final_recall_items_dict.setdefault(user_id, {})\n",
    "            for item, score in sorted_item_list:\n",
    "                final_recall_items_dict[user_id].setdefault(item, 0)\n",
    "                final_recall_items_dict[user_id][item] += recall_method_weight * score  \n",
    "    \n",
    "    final_recall_items_dict_rank = {}\n",
    "    # 多路召回时也可以控制最终的召回数量\n",
    "    for user, recall_item_dict in final_recall_items_dict.items():\n",
    "        final_recall_items_dict_rank[user] = sorted(recall_item_dict.items(), key=lambda x: x[1], reverse=True)[:topk]\n",
    "\n",
    "    # 将多路召回后的最终结果字典保存到本地\n",
    "    # pickle.dump(final_recall_items_dict_rank, open(os.path.join(save_path, 'final_recall_items_dict.pkl'),'wb'))\n",
    "\n",
    "    return final_recall_items_dict_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/3 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "多路召回合并...\n",
      "itemcf_sim_itemcf_recall...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1/3 [00:01<00:03,  1.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "usercf_sim_itemcf_recall...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 2/3 [00:02<00:01,  1.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cold_start_recall...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:04<00:00,  1.36s/it]\n"
     ]
    }
   ],
   "source": [
    "final_recall_items_dict_rank = combine_recall_results(user_multi_recall_dict, weight_dict, topk=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(194935, 1.5),\n",
       " (237870, 0.8616326270692185),\n",
       " (336221, 0.5),\n",
       " (272143, 0.44713089585246624),\n",
       " (195645, 0.4360433901218392)]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_recall_items_dict_rank[200000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50002/50002 [00:01<00:00, 34878.12it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>click_article_id</th>\n",
       "      <th>pred_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>249999</td>\n",
       "      <td>234698</td>\n",
       "      <td>1.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>249999</td>\n",
       "      <td>336223</td>\n",
       "      <td>0.976786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>249999</td>\n",
       "      <td>95716</td>\n",
       "      <td>0.894521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>249999</td>\n",
       "      <td>162655</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>249999</td>\n",
       "      <td>300082</td>\n",
       "      <td>0.368260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  click_article_id  pred_score\n",
       "0   249999            234698    1.500000\n",
       "1   249999            336223    0.976786\n",
       "2   249999             95716    0.894521\n",
       "3   249999            162655    0.500000\n",
       "4   249999            300082    0.368260"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将字典的形式转换成df\n",
    "user_item_score_list = []\n",
    "\n",
    "for user, items in tqdm(final_recall_items_dict_rank.items()):\n",
    "    for item, score in items:\n",
    "        user_item_score_list.append([user, item, score])  # final_recall_items_dict_rank\n",
    "\n",
    "recall_df = pd.DataFrame(user_item_score_list, columns=['user_id', 'click_article_id', 'pred_score'])\n",
    "recall_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 生成提交文件\n",
    "save_path = \"./\"\n",
    "\n",
    "def submit(recall_df, topk=5, model_name=None):\n",
    "    recall_df = recall_df.sort_values(by=['user_id', 'pred_score'])\n",
    "    recall_df['rank'] = recall_df.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first')\n",
    "    \n",
    "    # 判断是不是每个用户都有5篇文章及以上\n",
    "    tmp = recall_df.groupby('user_id').apply(lambda x: x['rank'].max())\n",
    "    assert tmp.min() >= topk\n",
    "    \n",
    "    del recall_df['pred_score']\n",
    "    submit = recall_df[recall_df['rank'] <= topk].set_index(['user_id', 'rank']).unstack(-1).reset_index()\n",
    "    \n",
    "    submit.columns = [int(col) if isinstance(col, int) else col for col in submit.columns.droplevel(0)]\n",
    "    # 按照提交格式定义列名\n",
    "    submit = submit.rename(columns={'': 'user_id', 1: 'article_1', 2: 'article_2', \n",
    "                                                  3: 'article_3', 4: 'article_4', 5: 'article_5'})\n",
    "    \n",
    "    save_name = save_path + model_name + '_' + datetime.today().strftime('%m-%d') + '.csv'\n",
    "    submit.to_csv(save_name, index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取测试集\n",
    "test_click = pd.read_csv(\"data/testA_click_log.csv\")\n",
    "test_click = test_click['user_id'].unique()\n",
    "\n",
    "# 从所有的召回数据中将测试集中的用户选出来\n",
    "test_recall = recall_df[recall_df['user_id'].isin(test_click)]\n",
    "\n",
    "# 生成提交文件\n",
    "submit(recall_df, topk=5, model_name='final_recall')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python-pytorch",
   "language": "python",
   "name": "qiangsunyuan-pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}