Keras-CNN(多卷积核)-LSTM-自定义词库+本地全局+类目+依赖w2v模型-predict-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# 先进行本地/全局分类，再进行类目分类\n",
    "# 两次分类，现将用户输入按照意愿进行分类（愿意还款，意图模糊，延期还款，其它），再根据不同意愿进行进一步分类\n",
    "# 理论上，意愿分类的准确率会很高，限定意愿后的进一步分类准确率也会有所提高\n",
    "# 仅使用多卷积核CNN进行特征的综合提取\n",
    "# 对于池化尺寸的大小未定，需加大样本数量进行对比\n",
    "import keras\n",
    "import xlrd, xlwt\n",
    "from xlutils import copy\n",
    "import os\n",
    "import random\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from keras.models import Model\n",
    "from keras.preprocessing.text import one_hot\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Flatten\n",
    "from keras import regularizers\n",
    "from keras.layers.embeddings import Embedding\n",
    "import jieba\n",
    "import numpy as np\n",
    "import time\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.layers import Conv1D, MaxPooling1D, GlobalMaxPooling1D, Dropout, LSTM, SpatialDropout1D, LeakyReLU, Input, concatenate\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.models import load_model\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "对用户意图进行多次分：全局/局部——粗分类——细分类\n",
    "可以有效减少多分类情境下模型难收敛的问题\n",
    "逐层逼近用户的真实意图，越底层容错率越高\n",
    "'''\n",
    "# 对神经网络模型、数据及标签进行预处理\n",
    "def preprocessing(data_set, label, num_classes, seed, mode):\n",
    "    tokenized = data_preprocessing(data_set)\n",
    "    identify = wd_encode(wd2idx, tokenized)\n",
    "    padded_data,max_len = padding_sts(identify)\n",
    "    label = label_preprocessing(label, num_classes)\n",
    "# 将映射关系构造成字典\n",
    "    dictionary = []\n",
    "    for i in range(len(tokenized)):\n",
    "        dictionary.append({'sentence':data_set[i-1], 'tokenized':tokenized[i-1],\\\n",
    "                           'id':padded_data[i-1], 'label':label[i-1]})\n",
    "    # 对数据集进行切分，生成训练集和验证集\n",
    "    train_data, validate_data, train_label, validate_label = \\\n",
    "    split_data_set(padded_data, label, 0.1, detal=True)\n",
    "    if mode == 0:\n",
    "        # mode=0: 预测模式，需要再将训练集切分为训练集和测试集\n",
    "        train_data, test_data, train_label, test_label = \\\n",
    "        split_data_set(train_data, train_label, 0.222, detal=True)\n",
    "#     预测模式下不需要测试集\n",
    "    else:\n",
    "        test_data = None\n",
    "        test_label = None\n",
    "#     构造神经网络模型\n",
    "    model = build_model(max_len, num_classes)\n",
    "    \n",
    "    return dictionary, model, train_data, validate_data, test_data, train_label, validate_label, test_label\n",
    "\n",
    "# 将标签转换为指定的形式\n",
    "def label_preprocessing(label, num_classes):\n",
    "#     0: 愿意还款; 1: 意图模糊; 2: 延期还款; 3: 其他（全局）; \n",
    "    label = np.array(label)\n",
    "    categorical_labels = to_categorical(label, num_classes=num_classes)\n",
    "    return categorical_labels\n",
    "\n",
    "# 数据集预处理(jieba分词，去停用词)\n",
    "def data_preprocessing(data):\n",
    "    total_words = []\n",
    "    tokenized = []\n",
    "# 分词\n",
    "    for pattern in data:\n",
    "#         去前停用词，本意是希望在分词前将一些无意义的单字去除。\n",
    "#         但具体效果不详，因此先不使用\n",
    "#         sts = \"\"\n",
    "#         for word in pattern:\n",
    "#             if word not in pre_stopwords:\n",
    "#                 sts += word\n",
    "        t = jieba.lcut(pattern)\n",
    "        t_s = []\n",
    "#         去停用词\n",
    "        for word in t:\n",
    "            if word not in stopwords:\n",
    "                t_s.append(word)\n",
    "        total_words.extend(t_s)\n",
    "        tokenized.append(t_s)\n",
    "\n",
    "    return tokenized\n",
    "\n",
    "# 在预测时，寻找包含新词的句子\n",
    "def find_new_wd_of_sts(data, new_wd_of_sts_idx):\n",
    "    fobj = open(pathDir+'包含新词的句子.txt','a')\n",
    "    for i in new_wd_of_sts_idx:\n",
    "        fobj.write('\\n'+data[i])\n",
    "    print('save the new word of setense')\n",
    "    fobj.close()\n",
    "\n",
    "# 短句补零，使之与神经网络模型的embedding层的输入尺寸相同\n",
    "def padding_sts(identify,max_len=None):\n",
    "#     若不预设最大长度，则计算传入样本中的最大长度\n",
    "    if max_len is None:\n",
    "        max_len = 0\n",
    "        for sts in identify:\n",
    "            if len(sts)>max_len:\n",
    "                max_len = len(sts)\n",
    "    padded_id = list(map(lambda l:l + [0]*(max_len - len(l)), identify))\n",
    "    padded_id = np.array(padded_id)\n",
    "    return padded_id, max_len\n",
    "\n",
    "# 数据集切分，默认不显示detail\n",
    "def split_data_set(data, label, ratio, detal=False):\n",
    "    data_1, data_2, label_1, label_2 = train_test_split(data, label, test_size=ratio, random_state=seed)\n",
    "# data：待划分的样本特征集\n",
    "# label：待划分的样本标签\n",
    "# ratio：划分比例。如果是浮点数，在0-1之间，表示样本占比；如果是整数，表示样本数量\n",
    "# seed：是随机数的种子。\n",
    "# detal：显示分割后的详情。默认False\n",
    "    if detal == True:\n",
    "        print(\"data_1_len: \", len(data_1),\"label_1_len: \", len(label_1),\\\n",
    "              \"\\ndata_2_len: \", len(data_2),\"lebal_2_len: \", len(label_2))\n",
    "    return data_1, data_2, label_1, label_2\n",
    "\n",
    "# 预训练词向量模型\n",
    "def build_word2vec(filename, w2v_model=None, update=True):\n",
    "#     若传入了词向量模型，则直接载入模型，默认不更新词向量模型（更新策略效果暂时不明显，待调试）\n",
    "    if  w2v_model != None and update is False:\n",
    "        print('正在载入词向量模型...')\n",
    "        w2v_model = Word2Vec.load(pathDir+model_name)\n",
    "#     若需要更新词向量或重新构建词向量，则需获取样本数据集\n",
    "    else:\n",
    "        # 获取全数据集样本（用做词向量训练）\n",
    "        total_set = xlrd.open_workbook(filename)\n",
    "        total_set_sheet1 = total_set.sheet_by_index(0)\n",
    "        total_data = []\n",
    "        for i in range(total_set_sheet1.nrows-1):\n",
    "            total_data.append(total_set_sheet1.cell(i+1,0).value)\n",
    "#         对数据集进行分词\n",
    "        tokenized = []\n",
    "        for pattern in total_data:\n",
    "            t = jieba.lcut(pattern)\n",
    "            t_s = []\n",
    "#             去后停用词\n",
    "            for word in t:\n",
    "                if word not in stopwords:\n",
    "                    t_s.append(word)\n",
    "            tokenized.append(t_s)\n",
    "#     若没有传入词向量模型，构建新的模型\n",
    "        if w2v_model is None:\n",
    "            print('正在构建新的词向量模型...')\n",
    "#             词向量维度的缺省值为200\n",
    "            try:\n",
    "                w2v_dims\n",
    "            except NameError:\n",
    "                w2v_dims = 200\n",
    "#             训练词向量模型\n",
    "            w2v_model = Word2Vec(tokenized,sg=1,size=w2v_dims,window=5,min_count=1,negative=1,sample=0.001,hs=1)\n",
    "            w2v_model.train(tokenized, total_examples=len(tokenized), epochs=5)\n",
    "#         若传入了词向量模型，则载入并更新模型\n",
    "        else:\n",
    "            print('正在载入词向量模型...')\n",
    "            w2v_model = Word2Vec.load(pathDir+model_name)\n",
    "            print('正在更新词向量模型...')        \n",
    "            w2v_model = update_w2v(w2v_model, tokenized)\n",
    "\n",
    "    return w2v_model\n",
    "\n",
    "# 更新词向量模型（仅会对出现新词了的文本进行更新）\n",
    "def update_w2v(w2v_model, tokenized):\n",
    "#     生成原词向量的词-向量映射关系\n",
    "    vocab_list = []\n",
    "    for w, _ in w2v_model.wv.vocab.items():\n",
    "        vocab_list.append(w)\n",
    "#     寻找词向量模型中没有的新词\n",
    "    new_sts = []\n",
    "    new_wd = []\n",
    "    for sts in tokenized:\n",
    "        for wd in sts:\n",
    "            if wd not in vocab_list:\n",
    "                new_wd.append(wd)\n",
    "                new_sts.append(sts)\n",
    "                break\n",
    "    if new_sts != []:\n",
    "        print('发现新词：', new_wd, '\\n已对词向量模型进行更新！')\n",
    "        w2v_model.build_vocab(new_sts, update=True)\n",
    "#         w2v_model.train(new_sts,total_examples=w2v_model.corpus_count,epochs=1)\n",
    "    else:\n",
    "        print('未发现新词，没有更新模型！')\n",
    "    return w2v_model\n",
    "\n",
    "# 对数据集中的每个词，按照词-向量的索引进行编码，若出现了生词，则填0\n",
    "def wd_encode(wd2idx, tokenized):\n",
    "    identify = []\n",
    "    for sts in tokenized:\n",
    "        id_sts = []\n",
    "        for wd in sts:\n",
    "            try:\n",
    "                id_sts.append(wd2idx[wd])\n",
    "            except:\n",
    "                print('“'+wd+'”不在词向量模型中')\n",
    "                id_sts.append('0')\n",
    "        identify.append(id_sts)\n",
    "    return identify\n",
    "\n",
    "# 构建词向量的单词索引和embedding层矩阵\n",
    "def build_wd2idx_embedMatrix(w2vModel):\n",
    "    word2idx = {\"_stopWord\": 0}  # 停用词，生词和padding填充的0。\n",
    "    vocab_list = [(w, w2vModel.wv[w]) for w, v in w2vModel.wv.vocab.items()]\n",
    "    embedMatrix = np.zeros((len(w2vModel.wv.vocab.items()) + 1, w2vModel.vector_size))\n",
    "\n",
    "    for i in range(0, len(vocab_list)):\n",
    "        word = vocab_list[i][0]\n",
    "        word2idx[word] = i + 1\n",
    "        embedMatrix[i + 1] = vocab_list[i][1]\n",
    "    return word2idx, embedMatrix\n",
    "\n",
    "# 将深度学习过程中训练的词向量矩阵更新到词向量模型中\n",
    "def embed2w2v(w2vModel, embedMatrix):\n",
    "    print('正在根据学习过程中训练的词向量矩阵对词向量模型进行更新...')\n",
    "    i = 0\n",
    "    for w,_ in w2vModel.wv.vocab.items():\n",
    "        w2vModel.wv[w] = embedMatrix[i]\n",
    "        i += 1\n",
    "    print('词向量模型更新完成')\n",
    "    return w2vModel\n",
    "\n",
    "# 设计网络模型：LSTM+六个卷积核(各10个特征)+LeakyReLU激活函数\n",
    "def build_model(max_len, num_classes):\n",
    "    # Input\n",
    "    comment_seq = Input(shape=[max_len], name='x_seq')\n",
    "    # Embedding\n",
    "    emb_comment = Embedding(len(embedMatrix), len(embedMatrix[0]), weights=[embedMatrix],\\\n",
    "                            input_length=max_len, trainable=False)(comment_seq)\n",
    "    # LSTM\n",
    "    LSTM_1 = LSTM(units=32, dropout=0.2, recurrent_dropout=0.1, return_sequences=True)(emb_comment)\n",
    "    # LSTM_1 = LSTM(200, dropout=0.2, recurrent_dropout=0.2, return_sequences=True)(emb_comment)\n",
    "    # model.add(LSTM(200, return_sequences=True))\n",
    "    # LSTM_2 = LSTM(200, return_sequences=False)(LSTM_1)\n",
    "    # conv\n",
    "    convs = []\n",
    "    kernel_size = [1,2,3,4,5,max_len]\n",
    "    for ksz in kernel_size:\n",
    "        l_conv = Conv1D(filters=10, kernel_size=ksz, strides=1, padding='valid',\\\n",
    "                       use_bias=True,kernel_initializer='glorot_uniform',\\\n",
    "                       bias_initializer='zeros',kernel_regularizer=regularizers.l2(0.0001))(LSTM_1)\n",
    "        l_conv = LeakyReLU(alpha=0.01)(l_conv)\n",
    "        l_pool = GlobalMaxPooling1D()(l_conv)\n",
    "        convs.append(l_pool)\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    dropout = Dropout(0.3)(merge)\n",
    "    # output = Dense(120)(merge)\n",
    "    # output = LeakyReLU(alpha=0.05)(output)\n",
    "    output = Dense(num_classes,activation='softmax')(dropout)\n",
    "    model = Model([comment_seq], output)\n",
    "\n",
    "    # summarize the model\n",
    "    model.summary()\n",
    "    Adam = keras.optimizers.Adam(lr=0.01)\n",
    "    model.compile(optimizer=Adam, loss='categorical_crossentropy', metrics=['acc'])\n",
    "\n",
    "    ThisTime()\n",
    "    return model\n",
    "\n",
    "def train_model(train_data, validate_data, train_label, validate_label, model, name):\n",
    "    # 训练网络模型\n",
    "    # verbose:显示日志。verbose=0: 为不在标准输出流输出日志信息 verbose=1: 为输出进度条记录 verbose=2: 为每个epoch输出一行记录\n",
    "    start_time = time.time()\n",
    "    loss_tmp = 10\n",
    "    lr_pre = 0.01\n",
    "    lr = lr_pre\n",
    "    count = 1\n",
    "    threshold = 5\n",
    "#     动态学习速率衰减\n",
    "    while lr >= lr_pre/128:\n",
    "        hist = model.fit(train_data, train_label, epochs=1, verbose=1)\n",
    "#         每次epoch后使用验证集进行验证，防止对训练集的过拟合\n",
    "        loss, _ = model.evaluate(validate_data, validate_label, verbose=0)\n",
    "        print(loss)\n",
    "\n",
    "        if loss < loss_tmp:\n",
    "            count = 1\n",
    "            loss_tmp = loss\n",
    "            model.save(pathDir+name)\n",
    "        elif count >= threshold:\n",
    "            model = load_model(pathDir+name)\n",
    "            print(\"decay the learning rate\")\n",
    "            lr = lr/2\n",
    "            Adam = keras.optimizers.Adam(lr=lr)\n",
    "            model.compile(optimizer=Adam, loss='categorical_crossentropy', metrics=['acc'])\n",
    "            count = 1\n",
    "        else:\n",
    "            count += 1\n",
    "\n",
    "    elapsed_time = time.time() - start_time\n",
    "    print (\"processing time:\", elapsed_time, \"seconds\")\n",
    "    return model\n",
    "\n",
    "def test_model(model, dictionary, test_data, test_label, detail=False):\n",
    "    # 显示识别错误的数据，并计算准确率\n",
    "    count = 0\n",
    "    sum_loss = 0\n",
    "    for i in range(len(test_data)):\n",
    "        loss, accuracy = model.evaluate(np.column_stack(test_data[i-1]), np.column_stack(test_label[i-1]), batch_size=1, verbose=0)\n",
    "        if accuracy == 0:\n",
    "            for j in dictionary:\n",
    "                if (j['id'] == test_data[i-1]).all():\n",
    "                    tmp = j['label'].tolist()\n",
    "                    print(j['sentence'], j['tokenized'], 'label: ', tmp.index(max(tmp)))\n",
    "        else:\n",
    "            count += 1\n",
    "        sum_loss += loss\n",
    "        loss = sum_loss/len(test_data)\n",
    "    accuracy = count/len(test_data)\n",
    "    if detail == True:\n",
    "        print('Accuracy: %f' % (accuracy*100))\n",
    "        print('Loss: %f' % (loss))\n",
    "        ThisTime()\n",
    "        \n",
    "# 通过迭代遍历树状结构的节点\n",
    "def find_son(count_lavel, label_index=None):\n",
    "    tmp = []\n",
    "    data = []\n",
    "    label = []\n",
    "    if count_lavel == 0:\n",
    "        label_index = [0]# 初始化label_index\n",
    "    else:\n",
    "        node = label_index[-1]\n",
    "    for i in range(sheet1.nrows-1):\n",
    "#         print(i)\n",
    "#         预读取数据集和标签\n",
    "        if count_lavel == 0:\n",
    "            data.append(sheet1.cell(i+1, 0).value)\n",
    "            label.append(sheet1.cell(i+1, 1).value)\n",
    "        elif node == sheet1.cell(i+1, count_lavel).value:\n",
    "            for j in range(len(label_index)-1):\n",
    "                if sheet1.cell(i+1, j+1).value == label_index[j+1]:\n",
    "                    flag = True\n",
    "                else:\n",
    "                    flag = False\n",
    "                    break\n",
    "            if flag is True:\n",
    "                data.append(sheet1.cell(i+1, 0).value)\n",
    "                label.append(sheet1.cell(i+1, count_lavel+1).value)\n",
    "#     print(label)\n",
    "    tmp = list(set(label))# 计算该节点下子节点的数量\n",
    "    if '' in tmp:\n",
    "        raise TypeError('The label of some data is BLANK, which is illegal!(有数据未进行标记！)')\n",
    "#     当该节点下有多个子节点时，训练该节点\n",
    "    if count_lavel == 0:\n",
    "        print('根节点下的子节点:', tmp)# 根节点没有node\n",
    "    else:    \n",
    "        print('第%d层级%d节点下的子节点:' %(count_lavel, node), tmp)\n",
    "    if len(tmp) > 1:\n",
    "        print('正在训练该节点模型...')\n",
    "#         构建模型\n",
    "        (dictionary, model, train_data, \\\n",
    "         validate_data, test_data, \\\n",
    "         train_label, validate_label, testllabel) = preprocessing(data, label, len(tmp), seed=seed, mode=devide_mode)\n",
    "#         模型命名\n",
    "        if count_lavel == 0:# 根节点没有node\n",
    "            model_name = date+'_第%d层级的模型' %count_lavel\n",
    "        else:\n",
    "            model_name = date+'_第%d层级第%d节点的模型' %(count_lavel, node)\n",
    "#         训练模型\n",
    "        model = train_model(train_data, validate_data, \\\n",
    "                            train_label, validate_label, model, model_name)\n",
    "        print('训练完成！模型名称: %s' %model_name)\n",
    "#         if count_lavel < num_label_lavel-1:# 循环遍历至倒数第二层（最后一层不存在子节点）\n",
    "#             count_lavel += 1\n",
    "#             for node in tmp:\n",
    "#                 print('遍历第%d层级%d节点下的子节点...' %(count_lavel, node))\n",
    "#                 print('_'*100)\n",
    "#                 label_index_tmp = label_index.copy()\n",
    "#                 label_index_tmp.append(node)\n",
    "#                 find_son(count_lavel, label_index_tmp)\n",
    "#             count_lavel -= 1# 循环遍历完本层的节点后，返回上一层\n",
    "#             print('返回至第%d层' %count_lavel)\n",
    "    else:\n",
    "        print('该节点无需训练模型')\n",
    "        \n",
    "    if count_lavel < num_label_lavel-1:# 循环遍历至倒数第二层（最后一层不存在子节点）\n",
    "        count_lavel += 1\n",
    "        for node in tmp:\n",
    "            print('遍历第%d层级%d节点下的子节点...' %(count_lavel, node))\n",
    "            print('_'*100)\n",
    "            label_index_tmp = label_index.copy()\n",
    "            label_index_tmp.append(node)\n",
    "            find_son(count_lavel, label_index_tmp)\n",
    "        count_lavel -= 1# 循环遍历完本层的节点后，返回上一层\n",
    "        print('返回至第%d层' %count_lavel)    \n",
    "        \n",
    "def predict_data(count_lavel, data, threshold, last_label=None):\n",
    "#     如果该节点为根节点，使用单独的名称\n",
    "#     仅在第一次迭代（根节点处）进行分词\n",
    "    if count_lavel == 0:\n",
    "        model_name =  '20190620_第0层级的模型'\n",
    "        tok = True\n",
    "    else:\n",
    "        model_name = '20190620_第%d层级第%d节点的模型' %(count_lavel, last_label)\n",
    "        tok = False\n",
    "    #     分词\n",
    "    if tok is True:\n",
    "#         单句预测时，仅对单句进行去停用词+分词即可\n",
    "        tokenized = []\n",
    "        t_s = []\n",
    "        t = jieba.lcut(data)\n",
    "        for word in t:\n",
    "            if word not in stopwords:\n",
    "                t_s.append(word)\n",
    "        tokenized.append(t_s)\n",
    "    else:\n",
    "        tokenized = data\n",
    "#     查找model_name是否存在于已加载的模型中（此处需要模型按照规范命名，加载错模型会导致predict的报错）\n",
    "    for i in range(len(model_name_lst)):\n",
    "        if model_name == model_name_lst[i]['name']:\n",
    "            model = model_name_lst[i]['model']\n",
    "            break\n",
    "        else:\n",
    "            model = None\n",
    "    if model is None:\n",
    "#         print(count_lavel, model_name)\n",
    "        if count_lavel < num_label_lavel-1:\n",
    "            count_lavel += 1\n",
    "            output = predict_data(count_lavel, tokenized, threshold, last_label=0)\n",
    "        else:\n",
    "            output = 'Wrong!'\n",
    "        return output\n",
    "    \n",
    "    max_len = model.layers[1].output_shape[1]# 获取模型文本最大长度\n",
    "    num_classes = model.layers[23].output_shape[1]# 获取模型分类数量\n",
    "\n",
    "#     根据词向量模型映射词标签\n",
    "    identify = wd_encode(wd2idx, tokenized)\n",
    "#     限定输入文本的长度（不可超过模型最大长度）\n",
    "    if len(identify[0]) > max_len:\n",
    "        print('输入过长(超出%d)，仅截取前一部分' %max_len)\n",
    "        identify[0] = identify[0][0:max_len]\n",
    "    padded_data, _ = padding_sts(identify, max_len=max_len)\n",
    "\n",
    "    accuracy = model.predict(padded_data, batch_size=1, verbose=0, steps=None).tolist()\n",
    "#     print(accuracy)\n",
    "#     准确率未超过阈值则判定为nomatch\n",
    "    if max(accuracy[0]) >= threshold:\n",
    "        label = accuracy[0].index(max(accuracy[0]))\n",
    "#         print(label)\n",
    "        output = label\n",
    "#             当迭代至最后一层，不再进行迭代\n",
    "        if count_lavel < num_label_lavel-1:\n",
    "            count_lavel += 1\n",
    "            predict_data(count_lavel, tokenized, \\\n",
    "                         threshold=predict_threshold, last_label=label)\n",
    "    else:\n",
    "#         print('nomatch\\n')\n",
    "#         fobj.write('\\t'+'nomatch')\n",
    "        output = 'nomatch'\n",
    "    return output\n",
    "        \n",
    "\n",
    "# 显示每次运行片段的时间\n",
    "def ThisTime():\n",
    "    print('This time is: ', time.strftime(\"%Y-%m-%d %H:%M:%S\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\admin\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.956 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "成功加载自定义分词词库\n",
      "构建词向量模型...\n",
      "正在构建新的词向量模型...\n",
      "词向量模型保存成功：20190620_w2v_model\n"
     ]
    }
   ],
   "source": [
    "# 定义全局变量\n",
    "global date, wd2idx, embedMatrix, max_len, seed,\\\n",
    "        w2v_dims, predict_threshold, model_name_lst, \\\n",
    "        devide_mode, num_label_lavel\n",
    "date = time.strftime(\"%Y%m%d\")# 用于保存和加载当天的模型\n",
    "seed = random.randint(1,10000)# 最近生成种子，确保每次切分得到的数据集不同\n",
    "max_len = 50# 初始化单句最大词数。超出最大长度则丢弃，不足则填0\n",
    "devide_mode = 0# 0: 训练模式; 1: 预测模式\n",
    "predict_threshold = 0.8# 预测时判定是否nomatch的阈值，缺省值为0.8\n",
    "# w2v_dims = # 若不定义，则缺省值为200\n",
    "# model_name = # 如果要载入原有的词向量模型，则再次输入模型名称\n",
    "\n",
    "# 加载自定义词典\n",
    "pathDir = \"C:/Users/admin/催收业务-意图识别/\"\n",
    "jieba.load_userdict(pathDir+\"催收文本-newdic.txt\")\n",
    "print('成功加载自定义分词词库')\n",
    "\n",
    "# 获取停用词\n",
    "stopwords = [line.strip() for line in open(pathDir+\"催收-停用词.txt\",encoding='gb18030',errors='ignore').readlines()]\n",
    "\n",
    "# 默认构建新的词向量模型。若要在原有模型基础上继续训练，build_word2vec函数需传入词向量模型\n",
    "# 若使用原有词向量模型，默认不更新模型\n",
    "print('构建词向量模型...')\n",
    "w2v_model = build_word2vec(filename=pathDir+'语音转文本_全业务数据集.xlsx', w2v_model=None, update=False)\n",
    "w2v_model.save(pathDir+date+'_w2v_model')# 保存模型\n",
    "print('词向量模型保存成功：%s_w2v_model' %date)\n",
    "\n",
    "# 将词向量模型加载为数组\n",
    "wd2idx, embedMatrix = build_wd2idx_embedMatrix(w2v_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "模型的层级数为: 3 \n",
      "构造模型树...\n",
      "____________________________________________________________________________________________________\n",
      "根节点下的子节点: [0.0]\n",
      "该节点无需训练模型\n",
      "遍历第1层级0节点下的子节点...\n",
      "____________________________________________________________________________________________________\n",
      "第1层级0节点下的子节点: [1.0]\n",
      "该节点无需训练模型\n",
      "遍历第2层级1节点下的子节点...\n",
      "____________________________________________________________________________________________________\n",
      "第2层级1节点下的子节点: [0.0, 1.0, 2.0, 3.0]\n",
      "正在训练该节点模型...\n",
      "data_1_len:  6777 label_1_len:  6777 \n",
      "data_2_len:  754 lebal_2_len:  754\n",
      "data_1_len:  5272 label_1_len:  5272 \n",
      "data_2_len:  1505 lebal_2_len:  1505\n",
      "WARNING:tensorflow:From C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "x_seq (InputLayer)              (None, 38)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 38, 200)      2635200     x_seq[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 38, 32)       29824       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 38, 10)       330         lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 37, 10)       650         lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 36, 10)       970         lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 35, 10)       1290        lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 34, 10)       1610        lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, 1, 10)        12170       lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)       (None, 38, 10)       0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)       (None, 37, 10)       0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)       (None, 36, 10)       0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)       (None, 35, 10)       0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_5 (LeakyReLU)       (None, 34, 10)       0           conv1d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_6 (LeakyReLU)       (None, 1, 10)        0           conv1d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 10)           0           leaky_re_lu_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 10)           0           leaky_re_lu_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 10)           0           leaky_re_lu_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_4 (GlobalM (None, 10)           0           leaky_re_lu_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_5 (GlobalM (None, 10)           0           leaky_re_lu_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_6 (GlobalM (None, 10)           0           leaky_re_lu_6[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 60)           0           global_max_pooling1d_1[0][0]     \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "                                                                 global_max_pooling1d_3[0][0]     \n",
      "                                                                 global_max_pooling1d_4[0][0]     \n",
      "                                                                 global_max_pooling1d_5[0][0]     \n",
      "                                                                 global_max_pooling1d_6[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 60)           0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 4)            244         dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 2,682,288\n",
      "Trainable params: 47,088\n",
      "Non-trainable params: 2,635,200\n",
      "__________________________________________________________________________________________________\n",
      "This time is:  2019-06-20 15:03:27\n",
      "WARNING:tensorflow:From C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 10s 2ms/step - loss: 0.4349 - acc: 0.8507\n",
      "0.3786322501752358\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 9s 2ms/step - loss: 0.2781 - acc: 0.9108\n",
      "0.27955996484275836\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 9s 2ms/step - loss: 0.2445 - acc: 0.9275\n",
      "0.28280772786557834\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 10s 2ms/step - loss: 0.2270 - acc: 0.9296: 0s - loss: 0.2285 - \n",
      "0.30951564116724606\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 10s 2ms/step - loss: 0.2016 - acc: 0.9374\n",
      "0.2554336272436997\n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5272/5272 [==============================] - 8s 2ms/step - loss: 0.1789 - acc: 0.9456\n",
      "0.3333304361971367\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1809 - acc: 0.9471\n",
      "0.319204714910105\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 9s 2ms/step - loss: 0.1654 - acc: 0.9543\n",
      "0.3378210580001143\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1513 - acc: 0.9588\n",
      "0.33146945780405\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1517 - acc: 0.9600\n",
      "0.3720468205229357\n",
      "decay the learning rate\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 9s 2ms/step - loss: 0.1661 - acc: 0.9476\n",
      "0.27038205088924033\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1477 - acc: 0.9549\n",
      "0.2898473550691529\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1297 - acc: 0.9579\n",
      "0.3456499733880597\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 9s 2ms/step - loss: 0.1217 - acc: 0.9683\n",
      "0.3481209622770153\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1114 - acc: 0.9678\n",
      "0.3341311381571489\n",
      "decay the learning rate\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 14s 3ms/step - loss: 0.1584 - acc: 0.9511\n",
      "0.2843172748620061\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1360 - acc: 0.9588A: 1s - loss: 0\n",
      "0.29061816494407955\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1202 - acc: 0.9655\n",
      "0.3111748055848898\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1117 - acc: 0.9653\n",
      "0.3324548059022079\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1035 - acc: 0.9681\n",
      "0.3367140234623411\n",
      "decay the learning rate\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 10s 2ms/step - loss: 0.1561 - acc: 0.9518\n",
      "0.27541022710205704\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1378 - acc: 0.9586\n",
      "0.2856347232029356\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1267 - acc: 0.9630\n",
      "0.2848228277831242\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1164 - acc: 0.9655\n",
      "0.2823940168483188\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1116 - acc: 0.9660\n",
      "0.3030237259852159\n",
      "decay the learning rate\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 10s 2ms/step - loss: 0.1574 - acc: 0.9514\n",
      "0.26984338063143926\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1423 - acc: 0.9583\n",
      "0.27237812317018484\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1361 - acc: 0.9586\n",
      "0.28444120406472084\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 9s 2ms/step - loss: 0.1228 - acc: 0.9660A: 1s - loss: 0.120\n",
      "0.2805946025038904\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1177 - acc: 0.9640\n",
      "0.27915735190996127\n",
      "decay the learning rate\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 10s 2ms/step - loss: 0.1598 - acc: 0.9541\n",
      "0.2632755761279352\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 2ms/step - loss: 0.1513 - acc: 0.9556\n",
      "0.2683127221283293\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1420 - acc: 0.9588\n",
      "0.271456012912391\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1369 - acc: 0.9585\n",
      "0.27641746235779174\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 2ms/step - loss: 0.1332 - acc: 0.9588\n",
      "0.27743245088137114\n",
      "decay the learning rate\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 12s 2ms/step - loss: 0.1646 - acc: 0.9505\n",
      "0.26029395321952253\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1534 - acc: 0.9541\n",
      "0.2659294135690684\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1469 - acc: 0.9547\n",
      "0.2671958323498936\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 2ms/step - loss: 0.1420 - acc: 0.9594\n",
      "0.27024273886604716\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 2ms/step - loss: 0.1455 - acc: 0.9571\n",
      "0.27154519292340673\n",
      "decay the learning rate\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 12s 2ms/step - loss: 0.1636 - acc: 0.9501: 0s - loss: 0.1607 - acc:\n",
      "0.2577682347765652\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1615 - acc: 0.9516A: 0s - loss: 0.1567 -\n",
      "0.2605492516917322\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 8s 1ms/step - loss: 0.1552 - acc: 0.9526\n",
      "0.26357742021507546\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1538 - acc: 0.9537\n",
      "0.2655990801218018\n",
      "Epoch 1/1\n",
      "5272/5272 [==============================] - 7s 1ms/step - loss: 0.1446 - acc: 0.9566\n",
      "0.26706047866009275\n",
      "decay the learning rate\n",
      "processing time: 473.3110737800598 seconds\n",
      "训练完成！模型名称: 20190620_第2层级第1节点的模型\n",
      "返回至第1层\n",
      "返回至第0层\n",
      "模型训练完毕！\n"
     ]
    }
   ],
   "source": [
    "readbook = xlrd.open_workbook(pathDir+'催收-还款意图模糊.xlsx')\n",
    "sheet1 = readbook.sheet_by_index(0)# 文本\n",
    "num_label_lavel = sheet1.ncols-1# 数据集构成为：一列为文本，其余为层级标签。\n",
    "# num_label_lavel = 2\n",
    "print('模型的层级数为:', num_label_lavel, '\\n构造模型树...')\n",
    "print('_'*100)\n",
    "count_lavel = 0# 为方便迭代运算，在层级结构前加入第0层（根节点）\n",
    "find_son(count_lavel, label_index=None)# 迭代训练结构中有多个子节点的节点\n",
    "print('模型训练完毕！')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_1 = load_model(pathDir+'20190620_第2层级第0节点的模型')\n",
    "model_name_lst = [{'name': '20190620_第2层级第0节点的模型', 'model': model_1}]\n",
    "num_model = len(model_name_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载词向量\n",
    "w2v_model = Word2Vec.load(pathDir+'20190618_w2v_model')\n",
    "# 加载所有神经网络模型\n",
    "model_1 = load_model(pathDir+'20190618_第0层级的模型')\n",
    "lmodel_1 = load_model(pathDir+'20190618_第1层级第0节点的模型')\n",
    "gmodel_1 = load_model(pathDir+'20190618_第1层级第1节点的模型')\n",
    "\n",
    "model_name_lst = [{'name': '20190618_第0层级的模型', 'model': model_1}, \\\n",
    "                  {'name': '20190618_第1层级第0节点的模型', 'model': lmodel_1}, \\\n",
    "                  {'name': '20190618_第1层级第1节点的模型', 'model': gmodel_1}]\n",
    "num_model = len(model_name_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n"
     ]
    }
   ],
   "source": [
    "print(model_1.layers[1].output_shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "成功加载自定义分词词库\n"
     ]
    }
   ],
   "source": [
    "# 预测新文本\n",
    "# bug1: 如果输出文本长度超过训练集最长长度，会报错\n",
    "# bug2: 出现新词，会报错\n",
    "# 加载自定义词典\n",
    "# num_label_lavel = 3\n",
    "count_lavel = 0# 为方便迭代运算，在层级结构前加入第0层（根节点）\n",
    "jieba.load_userdict(\"催收文本-newdic.txt\")\n",
    "print('成功加载自定义分词词库')\n",
    "\n",
    "# 获取停用词\n",
    "stopwords = [line.strip() for line in open(\"催收-停用词.txt\",encoding='gb18030',errors='ignore').readlines()]\n",
    "\n",
    "# 将词向量模型加载为数组\n",
    "wd2idx, embedMatrix = build_wd2idx_embedMatrix(w2v_model)\n",
    "readbook = xlrd.open_workbook(pathDir+'催收-还款意图模糊.xlsx')\n",
    "sheet2 = readbook.sheet_by_index(0)\n",
    "\n",
    "predict_threshold = 0.99\n",
    "fobj_output = open(date+'-predict.txt','a')\n",
    "\n",
    "for i in range(sheet2.nrows-1):\n",
    "    predict_set = sheet2.cell(i+1,0).value\n",
    "# predict_set = input('文本: ')\n",
    "    output = predict_data(count_lavel, predict_set, threshold=predict_threshold)\n",
    "#     print(predict_set, output, 'i:', i)\n",
    "    fobj_output.write('\\n'+str(output))\n",
    "fobj_output.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预测新文本\n",
    "# bug1: 如果输出文本长度超过训练集最长长度，会报错\n",
    "# bug2: 出现新词，会报错\n",
    "# 加载自定义词典\n",
    "# num_label_lavel = 3\n",
    "count_lavel = 0# 为方便迭代运算，在层级结构前加入第0层（根节点）\n",
    "jieba.load_userdict(\"催收文本-newdic.txt\")\n",
    "print('成功加载自定义分词词库')\n",
    "\n",
    "# 获取停用词\n",
    "stopwords = [line.strip() for line in open(\"催收-停用词.txt\",encoding='gb18030',errors='ignore').readlines()]\n",
    "\n",
    "# 将词向量模型加载为数组\n",
    "wd2idx, embedMatrix = build_wd2idx_embedMatrix(w2v_model)\n",
    "readbook = xlrd.open_workbook(pathDir+'催收-承诺还款.xlsx')\n",
    "sheet2 = readbook.sheet_by_index(1)\n",
    "\n",
    "predict_threshold = 0.98\n",
    "fobj_output = open(date+'-predict.txt','a')\n",
    "\n",
    "for i in range(sheet2.nrows-1):\n",
    "    predict_set = sheet2.cell(i+1,0).value\n",
    "# predict_set = input('文本: ')\n",
    "    output = predict_data(count_lavel, predict_set, threshold=predict_threshold)\n",
    "#     print(predict_set, output, 'i:', i)\n",
    "    fobj_output.write('\\n'+str(output))\n",
    "fobj_output.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['20190617_第1层级第1节点的模型']"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list的模糊匹配\n",
    "import difflib\n",
    "\n",
    "AA = ['20190617_第0层级的模型', '20190617_第1层级第0节点的模型', '20190617_第1层级第1节点的模型']\n",
    "A = '20190617_第1层级第4节点的模型'\n",
    "a = difflib.get_close_matches(A,AA,1, cutoff=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n"
     ]
    }
   ],
   "source": [
    "w2v_model = Word2Vec.load('20190617_w2v_model')\n",
    "model_1 = load_model('20190610_一级模型_1')\n",
    "model_2 = load_model('20190610_一级模型_2')\n",
    "model_3 = load_model('20190610_一级模型_3')\n",
    "lmodel_1 = load_mo0del('20190610_二级（本地）模型_1')\n",
    "lmodel_2 = load_model('20190610_二级（本地）模型_2')\n",
    "lmodel_3 = load_model('20190610_二级（本地）模型_3')\n",
    "gmodel_1 = load_model('20190610_二级（全局）模型_1')\n",
    "gmodel_2 = load_model('20190610_二级（全局）模型_2')\n",
    "gmodel_3 = load_model('20190610_二级（全局）模型_3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "微信 ['微信'] label:  7\n",
      "这个我不懂了 ['这个', '我', '不', '懂了'] label:  6\n",
      "为什么不提前跟我说啊我一下子哪有那么多钱 ['为什么', '不', '提前', '跟', '我', '说', '啊', '我', '一下子', '哪有', '那么', '多', '钱'] label:  15\n",
      "你这是多少利息了 ['你', '这是', '多少', '利息', '了'] label:  17\n",
      "处理中遇到问题 ['处理', '中', '遇到', '问题'] label:  7\n",
      "我的到来人工聊妈吗他那卡里的吗我怎么登不了啊 ['我的', '到来', '人工', '聊妈', '吗', '他', '那', '卡里', '吗', '我', '怎么', '登', '不了', '啊'] label:  7\n",
      "喂喂你听的见吧 ['喂', '喂', '你', '听的见', '吧'] label:  0\n",
      "呃他妈的这什么意思哦 ['呃', '他妈的', '这', '什么', '意思', '哦'] label:  3\n",
      "呃我想请问你一下你们可以听我说一下吗因为我的那个卡绑定的那个卡换掉了我的那个卡掉了现在我要重新绑定一张卡然后我把 ['呃', '我', '想', '请问', '你', '一下', '你们', '可以', '听', '我', '说', '一下', '吗', '因为', '我的', '那个', '卡', '绑定', '那个', '卡换', '掉', '了', '我的', '那个', '卡', '掉', '了', '现在', '我', '要', '重新', '绑定', '一张', '卡', '我', '把'] label:  5\n",
      "喂你给我再说啊 ['喂', '你', '给我', '再说', '啊'] label:  6\n",
      "我不知道为什么 ['我', '不知道', '为什么'] label:  7\n",
      "唉你等一下我因为我刚好今天从工作的基础里好吧你你从讲好吧 ['唉', '你', '等一下', '我', '因为', '我', '刚好', '今天', '从', '工作', '基础', '里', '好吧', '你', '你', '从', '讲', '好吧'] label:  6\n",
      "你他妈谁啊 ['你', '他', '妈', '谁', '啊'] label:  3\n",
      "呃我说我我我想问一下这个时间更改吗因为我这个发工资的日期是十号发工资还没发工资了关键是要我想问一下咱们那个 ['呃', '我', '说', '我', '我', '我', '想', '问一下', '这个', '时间', '更改', '吗', '因为', '我', '这个', '发工资', '日期', '是', '十号', '发工资', '还没', '发工资', '了', '关键', '是', '要', '我', '想', '问一下', '咱们', '那个'] label:  2\n",
      "别的银行能扣么 ['别的', '银行', '能扣', '么'] label:  7\n",
      "喂你帮我你们怎么没有那个提前还款呢 ['喂', '你', '帮', '我', '你们', '怎么', '没有', '那个', '提前', '还款', '呢'] label:  7\n",
      "你们能能不能不要用那个语音机器人啊 ['你们', '能', '能不能', '不要', '用', '那个', '语音', '机器人', '啊'] label:  4\n",
      "你好能听到我说话吗喂你好 ['你好', '能', '听到', '我', '说话', '吗', '喂', '你好'] label:  6\n",
      "听得到听得到 ['听得到', '听得到'] label:  6\n",
      "呃对什么问题 ['呃', '对', '什么', '问题'] label:  3\n",
      "没错是我 ['没错', '是', '我'] label:  6\n",
      "你会不会说话啊喂喂 ['你', '会不会', '说话', '啊', '喂', '喂'] label:  6\n",
      "不是没钱是不是给忘记处理了那个银行卡是不是要能改银行卡嘛就是啊 ['不是', '没钱', '是不是', '给', '忘记处理', '了', '那个', '银行卡', '是不是', '要', '能', '改', '银行卡', '嘛', '啊'] label:  7\n",
      "什么鬼 ['什么', '鬼'] label:  6\n",
      "不是我我前几天把银行卡丢了然后现在全人权不了我银行卡号记不得 ['不是', '我', '我', '前几天', '把', '银行卡', '丢了', '现在', '全', '人权', '不了', '我', '银行卡号', '记不得'] label:  10\n",
      "怎么办啊现在是不是最后一期了 ['怎么办', '啊', '现在', '是不是', '最后', '一期', '了'] label:  13\n",
      "手续费 ['手续费'] label:  17\n",
      "人工智能 ['人工智能'] label:  4\n",
      "就是账户专用账户中自动扣款的吗 ['账户', '专用', '账户', '中', '自动扣款', '吗'] label:  7\n",
      "然后是几天说一下 ['是', '几天', '说', '一下'] label:  12\n",
      "是客服打过来的吗 ['是', '客服', '打', '过来', '吗'] label:  4\n",
      "在哪啊 ['在', '哪', '啊'] label:  7\n",
      "我要人跟我说话 ['我', '要', '人', '跟', '我', '说话'] label:  5\n",
      "是客服是吧 ['是', '客服', '是', '吧'] label:  3\n",
      "喂他怎么了 ['喂', '他', '怎么', '了'] label:  3\n",
      "喂你好我确认客服人员还是那个系统自己打过来的 ['喂', '你好', '我', '确认', '客服', '人员', '还是', '那个', '系统', '自己', '打', '过来'] label:  4\n",
      "我现在在山里面信号不好喂 ['我', '现在', '在', '山', '里面', '信号不好', '喂'] label:  6\n",
      "需要您打电话干嘛 ['需要', '您', '打电话', '干嘛'] label:  3\n",
      "呃想问点什么的 ['呃', '想', '问点', '什么'] label:  0\n",
      "是我听的见吧 ['是', '我', '听的见', '吧'] label:  0\n",
      "已经逾期几天了啊 ['已经', '逾期', '几天', '了', '啊'] label:  12\n",
      "哎呀我也不知道这这个怎么回事 ['哎呀', '我', '也', '不知道', '这', '这个', '怎么回事'] label:  7\n",
      "Accuracy: 95.394737\n",
      "Loss: 0.234351\n",
      "This time is:  2019-06-14 15:35:51\n"
     ]
    }
   ],
   "source": [
    "# 测试模型，传入参数：用于测试的模型，字典，测试数据，测试标签\n",
    "test_model(model_1, g_dictionary, gtstd, gtstl, detail=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "readbook = xlrd.open_workbook('催收-用户输入-样本及标签-本地全局-类目.xlsx')\n",
    "sheet2 = readbook.sheet_by_name('pos_stopwords')\n",
    "sheet3 = readbook.sheet_by_name('predict')\n",
    "pos_stopwords = []\n",
    "for i in range(sheet2.nrows):\n",
    "    pos_stopwords.append(sheet2.cell(i,0).value)\n",
    "predict_set = []\n",
    "for i in range(sheet3.nrows-1):\n",
    "    predict_set.append(sheet3.cell(i+1,0).value)\n",
    "wd2idx, embedMatrix = build_wd2idx_embedMatrix(w2v_model)\n",
    "# 加载自定义词典\n",
    "jieba.load_userdict(\"催收文本-newdic.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\admin\\AppData\\Local\\Temp\\jieba.cache\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-0dc9e9d25bd1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mwd2idx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0membedMatrix\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbuild_wd2idx_embedMatrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mw2v_model\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;31m# 加载自定义词典\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mjieba\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload_userdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"催收文本-newdic.txt\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\jieba\\__init__.py\u001b[0m in \u001b[0;36mload_userdict\u001b[1;34m(self, f)\u001b[0m\n\u001b[0;32m    369\u001b[0m         \u001b[0mWord\u001b[0m \u001b[0mtype\u001b[0m \u001b[0mmay\u001b[0m \u001b[0mbe\u001b[0m \u001b[0mignored\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    370\u001b[0m         '''\n\u001b[1;32m--> 371\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    372\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstring_types\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    373\u001b[0m             \u001b[0mf_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\jieba\\__init__.py\u001b[0m in \u001b[0;36mcheck_initialized\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    166\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcheck_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minitialized\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 168\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minitialize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    169\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    170\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcalc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msentence\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mDAG\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mroute\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\jieba\\__init__.py\u001b[0m in \u001b[0;36minitialize\u001b[1;34m(self, dictionary)\u001b[0m\n\u001b[0;32m    132\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m                     \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcache_file\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mcf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 134\u001b[1;33m                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFREQ\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtotal\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmarshal\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    135\u001b[0m                     \u001b[0mload_from_cache_fail\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    136\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "readbook = xlrd.open_workbook('催收-用户输入-样本及标签-本地全局-类目.xlsx')\n",
    "sheet2 = readbook.sheet_by_name('pos_stopwords')\n",
    "pos_stopwords = []\n",
    "for i in range(sheet2.nrows):\n",
    "    pos_stopwords.append(sheet2.cell(i,0).value)\n",
    "wd2idx, embedMatrix = build_wd2idx_embedMatrix(w2v_model)\n",
    "# 加载自定义词典\n",
    "jieba.load_userdict(\"催收文本-newdic.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('估计', 0.7111688852310181), ('倒把', 0.6890275478363037), ('不照', 0.6791861057281494), ('活转', 0.6752203702926636), ('甚至', 0.6734603643417358), ('要明', 0.6718773245811462), ('微软', 0.6554404497146606), ('留款', 0.6554221510887146), ('越过', 0.6531094312667847), ('少扣', 0.6469331979751587)]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\admin\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "print(w2v_model.most_similar(positive=['可能']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\biocloo\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.945 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "好嘞 ['好', '嘞']\n"
     ]
    }
   ],
   "source": [
    "jieba.load_userdict(\"催收文本-newdic.txt\")\n",
    "sts1 = '好嘞'\n",
    "sts1_cut = jieba.lcut(sts1)\n",
    "print(sts1,sts1_cut)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}