MiniProject_2_Part2.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IsfWwT9a1bxJ"
      },
      "outputs": [],
      "source": [
        "# Importing all necessary files\n",
        "from lxml import etree, objectify\n",
        "import nltk\n",
        "import string\n",
        "from nltk.corpus import wordnet\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import WordNetLemmatizer \n",
        "import pandas as pd \n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('omw-1.4')\n",
        "nltk.download('averaged_perceptron_tagger')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LuXyIhug_Rwz"
      },
      "outputs": [],
      "source": [
        "!pip install swifter\n",
        "import swifter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cq6zSPn5CJWQ"
      },
      "outputs": [],
      "source": [
        "#Setting some global variables\n",
        "global Tree\n",
        "global TreeNew\n",
        "global SenseLemmaDictionary\n",
        "global SenseLemmaCorpusDictionary\n",
        "SenseLemmaDictionary = {}\n",
        "SenseLemmaCorpusDictionary = {}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "95lnrGRr1jtj"
      },
      "outputs": [],
      "source": [
        "# function to get senses from original dictionary: \n",
        "def get_senses(word, pos):\n",
        "    item = Tree.xpath(\"//lexelt[@item='%s.%s']\" % (word, pos))    \n",
        "    senses = []\n",
        "    if len(item) >= 1:\n",
        "        for sense in item[0].getchildren():\n",
        "            senses.append(dict(zip(sense.keys(), sense.values())))\n",
        "    return senses"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NzjoP0bg3qau"
      },
      "outputs": [],
      "source": [
        "# functions to rename columns and ignore the index column\n",
        "def rename_col(dataset):\n",
        "    dataset_new = dataset.rename(columns = {0:\"Target_Word\", 1:\"Sense_ID\", 2:\"Sentence\"})\n",
        "    dataset_new = dataset_new.reset_index(drop=True)\n",
        "    return dataset_new"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QS_2COMe3sZ1"
      },
      "outputs": [],
      "source": [
        "#convert sentence column to lower case, remove digits and punctuations\n",
        "def clean_data(dataset, colname):\n",
        "    stop = stopwords.words('english')\n",
        "    string.punctuation = string.punctuation.replace('%', '')\n",
        "    dataset[\"cleaned_data\"] = dataset[colname].apply(lambda words: ' '.join(word.lower().translate(str.maketrans(string.punctuation, ' '*len(string.punctuation))) for word in words.split()))\n",
        "    dataset[\"cleaned_data\"] = dataset[\"cleaned_data\"].str.replace('\\d+', '')\n",
        "    return dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eLZiKPiP341t"
      },
      "outputs": [],
      "source": [
        "# retrieving pos tags for the words and lemmatize it\n",
        "def get_pos_wordnet(sent):\n",
        "    lemmatizer = nltk.stem.WordNetLemmatizer()\n",
        "    sent = ' '.join(word.lower().translate(str.maketrans(string.punctuation, ' '*len(string.punctuation))) for word in sent.split())\n",
        "    list_words = sent.split()\n",
        "    final_list = []\n",
        "    for i in range (len(list_words)):\n",
        "        tag = nltk.pos_tag(list_words)[i][1][0].upper()\n",
        "        tag_dict = {\"J\": wordnet.ADJ,\n",
        "                    \"N\": wordnet.NOUN,\n",
        "                    \"V\": wordnet.VERB,\n",
        "                    \"R\": wordnet.ADV}\n",
        "        final_tag = tag_dict.get(tag, wordnet.NOUN)\n",
        "        lemmatized_word = lemmatizer.lemmatize(list_words[i],final_tag)\n",
        "        final_list.append([list_words[i],final_tag,lemmatized_word])\n",
        "    return final_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "050Kqh074Erp"
      },
      "outputs": [],
      "source": [
        "# function to remove stop words and words with length < 3\n",
        "def remove_stopwords(pos_input_list):\n",
        "    return_list = []\n",
        "    stop = stopwords.words('english')\n",
        "    for pos in pos_input_list:\n",
        "        if (pos[2] not in stop and (len(pos[2])>2 or pos[2]==\"%%\")):\n",
        "            return_list.append(pos)\n",
        "    return return_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2GE871Pi4L5l"
      },
      "outputs": [],
      "source": [
        "# function to lemmatize words and add pos tags\n",
        "def lemmatize_sentences(sent):\n",
        "    lemmatizer = nltk.stem.WordNetLemmatizer()\n",
        "    sent = ' '.join(word.lower().translate(str.maketrans(string.punctuation, ' '*len(string.punctuation))) for word in sent.split())\n",
        "    list_words = sent.split()\n",
        "    lemmatize_words = ''\n",
        "    for i in range (len(list_words)):\n",
        "        tag = nltk.pos_tag(list_words)[i][1][0].upper()\n",
        "        tag_dict = {\"J\": wordnet.ADJ,\n",
        "                    \"N\": wordnet.NOUN,\n",
        "                    \"V\": wordnet.VERB,\n",
        "                    \"R\": wordnet.ADV}\n",
        "        final_tag = tag_dict.get(tag, wordnet.NOUN)\n",
        "        lemmatize_words += \" \" + lemmatizer.lemmatize(list_words[i],final_tag)   \n",
        "    return lemmatize_words.strip()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "5ez_2Kuj4px3"
      },
      "outputs": [],
      "source": [
        "# function to lemmatize sense words\n",
        "def convert_sense_to_lemma(word_pos, sense, corpus = False):\n",
        "    word_pos = word_pos.strip()\n",
        "    key = word_pos+ \"_\"+sense.get('id')\n",
        "    sense_examples = \"\"\n",
        "    if corpus:\n",
        "        if (key in SenseLemmaCorpusDictionary):\n",
        "            sense_examples = SenseLemmaCorpusDictionary.get(key)\n",
        "        else:\n",
        "            sense_examples = (\n",
        "                lemmatize_sentences(sense.get('gloss').lower())\n",
        "                + \" | \"\n",
        "                + ('.'.join(lemmatize_sentences(sentence.lower()) for sentence in sense.get('examples').split(\".\")))\n",
        "            )\n",
        "            SenseLemmaCorpusDictionary[key] = sense_examples\n",
        "    else:\n",
        "        if (key in SenseLemmaDictionary):\n",
        "            sense_examples = SenseLemmaDictionary.get(key)\n",
        "        else:\n",
        "            sense_examples = (\n",
        "                lemmatize_sentences(sense.get('gloss').lower())\n",
        "                + \" | \"\n",
        "                + ('.'.join(lemmatize_sentences(sentence.lower()) for sentence in sense.get('examples').split(\".\")))\n",
        "            )\n",
        "            SenseLemmaDictionary[key] = sense_examples\n",
        "    return sense_examples"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "25cKsW0q7lb4"
      },
      "outputs": [],
      "source": [
        "# function for calculating accuracies \n",
        "def calculate_accuracy(dataframe, column_name):\n",
        "    accuracy_number = 0\n",
        "    i=0\n",
        "    for index, row in dataframe.iterrows():\n",
        "        if(int(row['Sense_ID'])==int(row[column_name])):\n",
        "            accuracy_number += 1\n",
        "        i += 1\n",
        "    return ((accuracy_number/i)*100)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yaBlTwFJA0V7"
      },
      "outputs": [],
      "source": [
        "# function for exporting to CSV\n",
        "def export_to_csv(input_data_frame, csv_path):\n",
        "    tmp_df = input_data_frame.drop(['Sentence', 'cleaned_data', 'pos_data'], axis=1)\n",
        "    tmp_df.to_csv(csv_path, index = False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WSeh2ZX5WHxq"
      },
      "source": [
        "**SIMPLE LESK**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OQAP5lO36p8c"
      },
      "outputs": [],
      "source": [
        "# Simple lesk Algorithm\n",
        "def simple_lesk_algo(target_word, pos_data):\n",
        "    target_data = target_word.split(\".\")\n",
        "    #fetching senses of targetword from given dictionary\n",
        "    senses = get_senses(target_data[0].strip(), target_data[1].strip()) # targetword, pos\n",
        "    score_map = {}\n",
        "    pos_sentence = []\n",
        "    for pos_word in pos_data:\n",
        "        # adds lemmatized words in pos_sentence\n",
        "        pos_sentence.append(pos_word[2]) \n",
        "        \n",
        "    for sense in senses:\n",
        "        sense_score = 0\n",
        "        #fetching lemmatized form of gloss+examples from dictionary sense passed\n",
        "        sense_examples = convert_sense_to_lemma(target_word, sense)\n",
        "        sense_example_words = sense_examples.split()\n",
        "        #Overlapping words from dictionary(gloss+examples) and lemmatized test data sentence\n",
        "        ovlap_words = set(sense_example_words).intersection( set(pos_sentence) )\n",
        "        # total no.of overlapping words as score for that sense id\n",
        "        score_map[sense.get('id')] = len(ovlap_words)\n",
        "      \n",
        "\n",
        "    max_sense = max(score_map, key=score_map.get)  # get the max sense of the score\n",
        "    return max_sense"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bzeqbNXCBGlV"
      },
      "outputs": [],
      "source": [
        "# read the dictionary file - original \n",
        "Parser = objectify.makeparser(recover=True)\n",
        "Tree = objectify.fromstring(''.join(open('dictionary.xml').readlines()), Parser)\n",
        "\n",
        "# read test data\n",
        "train_data = pd.read_csv (r'train.data',header=None,delimiter = \"|\")\n",
        "test_data = pd.read_csv (r'test.data',header=None,delimiter = \"|\")\n",
        "validation_data = pd.read_csv (r'validate.data',header=None,delimiter = \"|\")\n",
        "\n",
        "# rename columns for all the datasets\n",
        "train_data_new = rename_col(train_data)\n",
        "test_data_new = rename_col(test_data)\n",
        "validation_data_new = rename_col(validation_data)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CMOGSSWaBeB-"
      },
      "outputs": [],
      "source": [
        "#Validation Data\n",
        "simple_lesk_validation_df = validation_data_new\n",
        "# adds a new column 'cleaned_data' with lowercase sentence\n",
        "simple_lesk_validation_df = clean_data(simple_lesk_validation_df, 'Sentence') \n",
        "#get the pos data in a new column\n",
        "simple_lesk_validation_df[\"pos_data\"] = simple_lesk_validation_df['cleaned_data'].swifter.apply(lambda sentence: get_pos_wordnet(sentence)) \n",
        "# for each pos data remove stop words\n",
        "simple_lesk_validation_df[\"pos_data\"] = simple_lesk_validation_df[\"pos_data\"].swifter.apply(lambda pos_data_list: remove_stopwords(pos_data_list))\n",
        "\n",
        "#Simple Lesk\n",
        "simple_lesk_validation_df['simple_lesk_sense_id'] = simple_lesk_validation_df.swifter.apply(lambda x: simple_lesk_algo(x['Target_Word'], x['pos_data']), axis=1)\n",
        "# Calculating accuracies of validation data\n",
        "print(\"Accuracy of validation data for simple_lesk: \" + str(calculate_accuracy(simple_lesk_validation_df, \"simple_lesk_sense_id\"))) #49.08896034297964\n",
        "\n",
        "# Export validation results to CSV\n",
        "export_to_csv(simple_lesk_validation_df, r'validation_results_SimpleLesk.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "6kTnBWFECyI5"
      },
      "outputs": [],
      "source": [
        "#Training Data\n",
        "simple_lesk_train_df = train_data_new\n",
        "simple_lesk_train_df = clean_data(simple_lesk_train_df, 'Sentence')\n",
        "simple_lesk_train_df[\"pos_data\"] = simple_lesk_train_df['cleaned_data'].swifter.apply(lambda sentence: get_pos_wordnet(sentence))\n",
        "simple_lesk_train_df[\"pos_data\"] = simple_lesk_train_df[\"pos_data\"].swifter.apply(lambda pos_data_list: remove_stopwords(pos_data_list))\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Simple Lesk\n",
        "simple_lesk_train_df['simple_lesk_sense_id'] = simple_lesk_train_df.swifter.apply(lambda x: simple_lesk_algo(x['Target_Word'], x['pos_data']), axis=1)\n",
        "# Calculating accuracies of training data\n",
        "print(\"Accuracy of training data for simple_lesk: \" + str(calculate_accuracy(simple_lesk_train_df, \"simple_lesk_sense_id\"))) #47.65046452134105\n",
        "\n",
        "# Export train results to CSV\n",
        "export_to_csv(simple_lesk_train_df, r'train_results_SimpleLesk.csv')"
      ],
      "metadata": {
        "id": "-_jUL-PprBff"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1z8Y-PxXMQb1"
      },
      "source": [
        "**ORIGINAL LESK**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "oihtYM6bE1kz"
      },
      "outputs": [],
      "source": [
        "# funtion to return the lemmatized form of words from gloss and examples of the senses of context_sense passed to it (test data)\n",
        "def get_lemma_from_context_sense(context_sense):\n",
        "    dictionary_examples = \"\"\n",
        "    for context_data in context_sense:\n",
        "        for sense_data in context_data[2]:\n",
        "            dictionary_examples += convert_sense_to_lemma(context_data[1], sense_data)\n",
        "    return dictionary_examples"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "_X7AID95E5SS"
      },
      "outputs": [],
      "source": [
        "# function to return the target word, context_word.its pos tag, senses of the context word from dictionary if exists, index of context word \n",
        "def get_context_sense(target_data, pos_data, corpus=False):\n",
        "    context_sense = []\n",
        "    target_sense = []\n",
        "    sentence = pos_data\n",
        "    sentence_length = len(sentence)\n",
        "    target_word = target_data.split(\".\")[0]\n",
        "    target_pos = target_data.split(\".\")[1]\n",
        "    for k in range(len(sentence)):\n",
        "        if sentence[k][0] == \"%%\":\n",
        "            target_index = k-1 # minus 1 bcz we will get k as the second occurrence of %%\n",
        "            targetWord = sentence[target_index][0]\n",
        "            break\n",
        "    i = target_index-2 # index for word before %%\n",
        "    j = target_index+2 # index for word after %%\n",
        "    k = 0\n",
        "    while((i>=0 or j<len(sentence)) and k<30):   \n",
        "        # For previous words from target word in test data\n",
        "        if(i>=0 and len(sentence[i][2].strip())>= 3 and sentence[i][2].strip() != target_word):\n",
        "            context_word = sentence[i][2].strip()\n",
        "            context_pos = sentence[i][1].strip()\n",
        "            if(corpus):\n",
        "                sense = get_new_senses(context_word,context_pos)\n",
        "            else:\n",
        "                sense = get_senses(context_word,context_pos)\n",
        "            if len(sense) >= 1:\n",
        "                context_sense.append([targetWord, context_word+\".\"+context_pos, sense, target_index-i]) \n",
        "\n",
        "        # For next words from target word in test data\n",
        "        if(j<len(sentence) and len(sentence[j][2].strip())>= 3 and sentence[j][2].strip() != target_word):\n",
        "            context_word = sentence[j][2].strip()\n",
        "            context_pos = sentence[j][1].strip()\n",
        "            if(corpus):\n",
        "                sense = get_new_senses(context_word,context_pos)\n",
        "            else:\n",
        "                sense = get_senses(context_word,context_pos) \n",
        "            if len(sense) >= 1:\n",
        "                context_sense.append([target_word,context_word+\".\"+context_pos,sense, j-target_index])     \n",
        "        i = i-1\n",
        "        j = j+1\n",
        "        k = k+1\n",
        "                \n",
        "    return context_sense"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "gt-k61_UKbu5"
      },
      "outputs": [],
      "source": [
        "# Orginal lesk Algorithm\n",
        "def original_lesk_algo(target_word_pos, pos_without_stopwords):\n",
        "    target_word_details = target_word_pos.split(\".\")\n",
        "    # get senses from dictionary\n",
        "    target_senses = get_senses(target_word_details[0].strip(), target_word_details[1].strip()) \n",
        "    score_map = {}\n",
        "    bigram_score_map ={}\n",
        "\n",
        "    # get context of the target word from test data and then get the context word's senses from dictionary and get their dictionary's gloss+examples lemmas\n",
        "    context_sentence = get_lemma_from_context_sense(get_context_sense(target_word_pos, pos_without_stopwords)) \n",
        "    \n",
        "    for sense in target_senses:      \n",
        "        sense_examples = convert_sense_to_lemma(target_word_pos.strip(), sense)\n",
        "        sense_example_words = sense_examples.split()\n",
        "        context_example_words = context_sentence.split()\n",
        "\n",
        "        #Fetching overlap between the meaning of context and the target words senses\n",
        "        ovlap_words = set(sense_example_words).intersection( set(context_example_words) )\n",
        "        context_score = len(ovlap_words)\n",
        "        \n",
        "        #Check for bigram overlaps\n",
        "        bigrams_sense_examples = list(zip(*[sense_example_words[i:] for i in range(2)]))\n",
        "        bigrams_context_examples = list(zip(*[context_example_words[i:] for i in range(2)]))\n",
        "        bigram_ovlap = set(bigrams_sense_examples).intersection( set(bigrams_context_examples) )\n",
        "\n",
        "        #Adding to the normal score but adding twice the value in order to give more weightage\n",
        "        context_score = len(ovlap_words) + 2*len(bigram_ovlap)\n",
        "        score_map[sense.get('id')] = context_score\n",
        "        \n",
        "    max_sense = max(score_map, key = score_map.get)\n",
        "    return max_sense"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "15smlrLkKt_d"
      },
      "outputs": [],
      "source": [
        "#Validation Data\n",
        "original_lesk_validation_df = simple_lesk_validation_df\n",
        "original_lesk_validation_df['original_lesk_sense_id'] = original_lesk_validation_df.swifter.apply(lambda x: original_lesk_algo(x['Target_Word'], x['pos_data']), axis=1)\n",
        "\n",
        "#Calculating accuracy\n",
        "print(\"Accuracy of validation data for original_lesk: \" + str(calculate_accuracy(original_lesk_validation_df, \"original_lesk_sense_id\"))) # 41.37191854233655 #41.264737406216504\n",
        "\n",
        "# Export validation results to CSV\n",
        "export_to_csv(original_lesk_validation_df, r'validation_results_OriginalLesk.csv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oKPgAusGL3vZ"
      },
      "source": [
        "Validation accuracy without consecutive overlapping score : 41.37% \n",
        "\n",
        "Validation accuracy with consecutive overlapping score : 41.26%"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KWHHTdnXLPsM"
      },
      "outputs": [],
      "source": [
        "#Training Data\n",
        "original_lesk_train_df = simple_lesk_train_df\n",
        "original_lesk_train_df['original_lesk_sense_id'] = original_lesk_train_df.swifter.apply(lambda x: original_lesk_algo(x['Target_Word'], x['pos_data']), axis=1)\n",
        "\n",
        "#Calculating accuracy\n",
        "print(\"Accuracy of training data for original_lesk: \" + str(calculate_accuracy(original_lesk_train_df, \"original_lesk_sense_id\"))) #40.025133521834746"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zv10G3mpMGfa"
      },
      "source": [
        "**CORPUS LESK**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "iXv8J9JK3PLH"
      },
      "outputs": [],
      "source": [
        "# function for making the new augemented dictionary by adding training data for corpus lesk\n",
        "def new_dictionary():\n",
        "    parser = objectify.makeparser(recover=True)\n",
        "    tree = objectify.fromstring(''.join(open('dictionary.xml').readlines()), parser)\n",
        "    train_data_new = rename_col(train_data)\n",
        "    for index, row in train_data_new.iterrows():\n",
        "        target_word = row['Target_Word'].strip()\n",
        "        sense_id = str(row['Sense_ID'])\n",
        "        sentence_to_add = row['Sentence']\n",
        "        \n",
        "        item = tree.xpath(\"//lexelt[@item='%s']\" % (target_word))\n",
        "        \n",
        "        for item_sense in item[0].getchildren():\n",
        "            if (str(item_sense.attrib['id']) == sense_id):\n",
        "                item_sense.attrib['examples'] = item_sense.attrib['examples'] + sentence_to_add\n",
        "\n",
        "    xml_new = etree.tostring(tree, pretty_print=True)\n",
        "    # save your xml\n",
        "    with open(r\"new_dictionary.xml\", \"wb\") as f:\n",
        "        f.write(xml_new)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "d8MrUyCI9YOm"
      },
      "outputs": [],
      "source": [
        "# function to get sense from new dictionary\n",
        "def get_new_senses(word, pos):\n",
        "    item = TreeNew.xpath(\"//lexelt[@item='%s.%s']\" % (word, pos))    \n",
        "    senses = []\n",
        "    if len(item) >= 1:\n",
        "        for sense in item[0].getchildren():\n",
        "            senses.append(dict(zip(sense.keys(), sense.values())))\n",
        "    return senses"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "4KEdgAuWgtLn"
      },
      "outputs": [],
      "source": [
        "# Corpus lesk algorithm\n",
        "def corpus_lesk_algo(target_word, pos_data):\n",
        "    print(target_word)\n",
        "    target_data = target_word.split(\".\")\n",
        "    senses = get_new_senses(target_data[0].strip(), target_data[1].strip())\n",
        "    score_map = {}\n",
        "    pos_sentence = []\n",
        "    for pos_word in pos_data:\n",
        "        pos_sentence.append(pos_word[2])\n",
        "   \n",
        "    for sense in senses:\n",
        "        sense_score = 0\n",
        "        sense_examples = convert_sense_to_lemma(target_word, sense, corpus=True)\n",
        "        sense_example_words = sense_examples.split()\n",
        "        ovlap_words = set(sense_example_words).intersection( set(pos_sentence) )\n",
        "        score_map[sense.get('id')] = len(ovlap_words)\n",
        "   \n",
        "    max_sense = max(score_map, key=score_map.get)\n",
        "    return max_sense"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "_PbPFE777sPl"
      },
      "outputs": [],
      "source": [
        "#create new augmented dictionary\n",
        "new_dictionary()\n",
        "ParserNew = objectify.makeparser(recover=True)\n",
        "TreeNew = objectify.fromstring(''.join(open('new_dictionary.xml').readlines()), ParserNew)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2kJ02R0DTqjp"
      },
      "outputs": [],
      "source": [
        "# Validation Data\n",
        "corpus_lesk_validation_df = simple_lesk_validation_df\n",
        "corpus_lesk_validation_df['corpus_lesk_sense_id'] = corpus_lesk_validation_df.swifter.apply(lambda x: corpus_lesk_algo(x['Target_Word'], x['pos_data']), axis=1)\n",
        "\n",
        "#Calculating accuracy\n",
        "print(\"Accuracy of validation data for corpus_lesk: \" + str(calculate_accuracy(corpus_lesk_validation_df, \"corpus_lesk_sense_id\"))) #83.38692390139335\n",
        "\n",
        "# Export validation results to CSV\n",
        "export_to_csv(corpus_lesk_validation_df, r'validation_results_CorpusLesk.csv')\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fnX3qWCaDhFA"
      },
      "outputs": [],
      "source": [
        "# Training Data\n",
        "corpus_lesk_train_df = simple_lesk_train_df\n",
        "corpus_lesk_train_df['corpus_lesk_sense_id'] = corpus_lesk_train_df.swifter.apply(lambda x: corpus_lesk_algo(x['Target_Word'], x['pos_data']), axis=1)\n",
        "\n",
        "# Calculating accuracy\n",
        "print(\"Accuracy of training data for adv_original_lesk: \" + str(calculate_accuracy(corpus_lesk_train_df, \"corpus_lesk_sense_id\"))) #98.69395449037296\n",
        "\n",
        "# Export validation results to CSV\n",
        "export_to_csv(corpus_lesk_train_df, r'training_data_results_CorpusLesk.csv')\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BTQ4a6r0Rzeb"
      },
      "source": [
        "We get the maximum accuracy with the Corpus Lesk, so we shall use the corpus lesk algorithm to predict the senses of the test data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "egqTJniDHbSM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "512a64a600a54d928fcac75e19a3bda2",
            "d4adc5d40d2e42b4a155eb667ca0b11f",
            "2c4935d184df43099d98c544ce9aa39e",
            "7bb69b662e8141e8ad49d30ee21fbf9d",
            "37c53695806c4fc08f774ae042da38eb",
            "7f7f4b46be34428db38633a299b7c782",
            "3428516fd4db41b9a70e8fd835bb7af4",
            "6e093fbd77fd4a40a17954be1df6d60b",
            "8b7d09316b234829a05797671d82e62f",
            "3569b9c479ed459fab4977ab11351021",
            "1afddfe8a9d64c029cba0a55c02b4edb"
          ]
        },
        "outputId": "6be2db05-fed4-4be3-e987-598f244fa385"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "512a64a600a54d928fcac75e19a3bda2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Pandas Apply:   0%|          | 0/3918 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Test Data\n",
        "corpus_lesk_test_df = test_data_new\n",
        "corpus_lesk_test_df = clean_data(corpus_lesk_test_df, 'Sentence')\n",
        "corpus_lesk_test_df['pos_data'] = corpus_lesk_test_df['cleaned_data'].swifter.apply(lambda sentence: get_pos_wordnet(sentence))\n",
        "corpus_lesk_test_df['pos_data'] = corpus_lesk_test_df['pos_data'].swifter.apply(lambda pos_data_list: remove_stopwords(pos_data_list))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7mEBgEchDMh3"
      },
      "outputs": [],
      "source": [
        "# Corpus lesk\n",
        "corpus_lesk_test_df = corpus_lesk_test_df\n",
        "corpus_lesk_test_df['corpus_lesk_sense_id'] = corpus_lesk_test_df.swifter.apply(lambda x: corpus_lesk_algo(x['Target_Word'], x['pos_data']), axis=1)\n",
        "\n",
        "# Export validation results to CSV\n",
        "export_to_csv(corpus_lesk_test_df, r'test_data_results_CorpusLesk.csv')\n",
        "    "
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "512a64a600a54d928fcac75e19a3bda2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d4adc5d40d2e42b4a155eb667ca0b11f",
              "IPY_MODEL_2c4935d184df43099d98c544ce9aa39e",
              "IPY_MODEL_7bb69b662e8141e8ad49d30ee21fbf9d"
            ],
            "layout": "IPY_MODEL_37c53695806c4fc08f774ae042da38eb"
          }
        },
        "d4adc5d40d2e42b4a155eb667ca0b11f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f7f4b46be34428db38633a299b7c782",
            "placeholder": "​",
            "style": "IPY_MODEL_3428516fd4db41b9a70e8fd835bb7af4",
            "value": "Pandas Apply:  99%"
          }
        },
        "2c4935d184df43099d98c544ce9aa39e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6e093fbd77fd4a40a17954be1df6d60b",
            "max": 3918,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8b7d09316b234829a05797671d82e62f",
            "value": 3892
          }
        },
        "7bb69b662e8141e8ad49d30ee21fbf9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3569b9c479ed459fab4977ab11351021",
            "placeholder": "​",
            "style": "IPY_MODEL_1afddfe8a9d64c029cba0a55c02b4edb",
            "value": " 3892/3918 [36:23&lt;00:18,  1.37it/s]"
          }
        },
        "37c53695806c4fc08f774ae042da38eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f7f4b46be34428db38633a299b7c782": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3428516fd4db41b9a70e8fd835bb7af4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6e093fbd77fd4a40a17954be1df6d60b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8b7d09316b234829a05797671d82e62f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3569b9c479ed459fab4977ab11351021": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1afddfe8a9d64c029cba0a55c02b4edb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}