relation_extraction/infobox_extraction.wikia.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Infobox extraction from wikia\n",
    "\n",
    "Extracts information from the [Zeldapedia](http://zelda.wikia.com/wiki/Wiki) based on `wikia` and generates two files:\n",
    "\n",
    "### `info/infoboxes.wikia.json`:\n",
    "\n",
    "```\n",
    "\"Link.html\": {\n",
    "    \"FIRST_APPEARANCE\": [\n",
    "      [\n",
    "        \"<i>\\n<a href=\\\"The_Legend_of_Zelda.html\\\" title=\\\"The Legend of Zelda\\\">\\n           The Legend of Zelda\\n          </a>\\n</i>\",\n",
    "        \"tag\"\n",
    "      ],\n",
    "      [\n",
    "        \"(\",\n",
    "        \"string\"\n",
    "      ],\n",
    "      [\n",
    "        \"<a href=\\\"1987.html\\\" title=\\\"1987\\\">\\n          1987\\n         </a>\",\n",
    "        \"tag\"\n",
    "      ]\n",
    "    ],\n",
    "    \"APPEARS_...\n",
    "```\n",
    "\n",
    "### `info/all_properties.wikia.json`:\n",
    "\n",
    "\n",
    "```\n",
    "[\n",
    "  \"KINDRED\",\n",
    "  \"COUNTRY\",\n",
    "  \"RACE\",\n",
    "  \"WEAPON\",\n",
    "  \"HOMELAND\",...\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from glob import glob\n",
    "\n",
    "from ie_conf import get_htmls_route\n",
    "from functions import infobox_clean_url, infobox_get_canonical_relation\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import re\n",
    "import os\n",
    "\n",
    "# Where are all those htmls?\n",
    "html_route = get_htmls_route(\"wikia\")\n",
    "\n",
    "dataframe = pd.read_csv(\"info/entities.wikia.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from bs4.element import NavigableString as string\n",
    "from bs4.element import Tag as tag\n",
    "\n",
    "types = {\n",
    "    \"<class 'bs4.element.NavigableString'>\":\"string\",\n",
    "    \"<class 'bs4.element.Tag'>\":\"tag\"\n",
    "}\n",
    "\n",
    "infoboxes = {}\n",
    "all_properties = set()\n",
    "\n",
    "for i, row in dataframe.iterrows():\n",
    "    filename = row['page']\n",
    "    file = html_route + filename\n",
    "    \n",
    "    soup:BeautifulSoup = None\n",
    "    with open(file, \"r\", encoding=\"utf8\") as r:\n",
    "        soup = BeautifulSoup(r, \"lxml\")\n",
    "        \n",
    "    wikiaMainContent = soup.find('article', {'id':'WikiaMainContent'})\n",
    "    if not wikiaMainContent:\n",
    "        continue\n",
    "    \n",
    "    infobox = wikiaMainContent.find('aside', {'class':'portable-infobox'})\n",
    "    if not infobox:\n",
    "        continue\n",
    "    \n",
    "    infoboxes[filename] = {}\n",
    "    \n",
    "    items = infobox.findAll('div', {'class': 'pi-item'})\n",
    "    for item in items:\n",
    "        h3 = item.find('h3')\n",
    "        if not h3:\n",
    "            continue\n",
    "            \n",
    "        relation = infobox_get_canonical_relation(h3.text.strip())\n",
    "        all_properties.add(relation)\n",
    "        \n",
    "        values = item.find('div', {'class':'pi-data-value'}, recursive=False)\n",
    "        infoboxes[filename][relation] = [[str(c).strip(),types.get(str(type(c)), str(type(c)))] \n",
    "                                         for c \n",
    "                                         in values.contents \n",
    "                                         if str(c).strip()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open(\"info/infoboxes.wikia.json\", \"w\", encoding=\"utf8\") as w:\n",
    "    json.dump(infoboxes, w, indent=4)\n",
    "with open(\"info/all_properties.wikia.json\", \"w\", encoding=\"utf8\") as w:\n",
    "    json.dump(list(all_properties), w, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Entities %d\" % len(infoboxes))\n",
    "print(\"Possible identified relationships %d\" % len(all_properties))\n",
    "keys = list(infoboxes.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}