relation_extraction/infobox_extraction.gamepedia.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Infobox extraction from gamepedia\n",
    "\n",
    "Extracts information from the [Zelda wiki](https://zelda.gamepedia.com/Main_Page) based on `gamepedia` and generates two files:\n",
    "\n",
    "### `info/infoboxes.gamepedia.json`:\n",
    "\n",
    "```\n",
    "\"Link.html\": {\n",
    "    \"FIRST_APPEARANCE\": [\n",
    "      [\n",
    "        \"<i>\\n<a href=\\\"The_Legend_of_Zelda.html\\\" title=\\\"The Legend of Zelda\\\">\\n           The Legend of Zelda\\n          </a>\\n</i>\",\n",
    "        \"tag\"\n",
    "      ],\n",
    "      [\n",
    "        \"(\",\n",
    "        \"string\"\n",
    "      ],\n",
    "      [\n",
    "        \"<a href=\\\"1987.html\\\" title=\\\"1987\\\">\\n          1987\\n         </a>\",\n",
    "        \"tag\"\n",
    "      ]\n",
    "    ],\n",
    "    \"APPEARS_...\n",
    "```\n",
    "\n",
    "### `info/all_properties.gamepedia.json`:\n",
    "\n",
    "\n",
    "```\n",
    "[\n",
    "  \"KINDRED\",\n",
    "  \"COUNTRY\",\n",
    "  \"RACE\",\n",
    "  \"WEAPON\",\n",
    "  \"HOMELAND\",...\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from glob import glob\n",
    "\n",
    "from ie_conf import get_htmls_route\n",
    "from functions import infobox_clean_url, infobox_get_canonical_relation\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import re\n",
    "import os\n",
    "\n",
    "# Where are all those htmls?\n",
    "html_route = get_htmls_route(\"gamepedia\")\n",
    "\n",
    "dataframe = pd.read_csv(\"info/entities.gamepedia.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from bs4.element import NavigableString as string\n",
    "from bs4.element import Tag as tag\n",
    "\n",
    "types = {\n",
    "    \"<class 'bs4.element.NavigableString'>\":\"string\",\n",
    "    \"<class 'bs4.element.Tag'>\":\"tag\"\n",
    "}\n",
    "\n",
    "infoboxes = {}\n",
    "all_properties = set()\n",
    "\n",
    "for i, row in dataframe.iterrows():\n",
    "    filename = row['page']\n",
    "    file = html_route + filename\n",
    "    \n",
    "    soup:BeautifulSoup = None\n",
    "    with open(file, \"r\", encoding=\"utf8\") as r:\n",
    "        soup = BeautifulSoup(r, \"lxml\")\n",
    "        \n",
    "    bodyContent = soup.find('div', {'id':'bodyContent'})\n",
    "    if not bodyContent:\n",
    "        continue\n",
    "    \n",
    "    wikitable = bodyContent.find('table', {'class':['infobox-curse-ad', 'wikitable', 'to']})\n",
    "    if not wikitable:\n",
    "        continue\n",
    "    \n",
    "    infoboxes[filename] = {}\n",
    "    \n",
    "    trs = wikitable.findAll('tr')\n",
    "    for tr in trs:\n",
    "        th = tr.find('th')\n",
    "        values = tr.find('td', recursive=False)\n",
    "        if not th or not values:\n",
    "            continue\n",
    "        relation = infobox_get_canonical_relation(th.text.strip())\n",
    "        all_properties.add(relation)\n",
    "        \n",
    "        infoboxes[filename][relation] = [[str(c).strip(),types.get(str(type(c)), str(type(c)))] \n",
    "                                         for c \n",
    "                                         in values.contents \n",
    "                                         if str(c).strip()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open(\"info/infoboxes.gamepedia.json\", \"w\", encoding=\"utf8\") as w:\n",
    "    json.dump(infoboxes, w, indent=4)\n",
    "with open(\"info/all_properties.gamepedia.json\", \"w\", encoding=\"utf8\") as w:\n",
    "    json.dump(list(all_properties), w, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"Entities %d\" % len(infoboxes))\n",
    "print(\"Possible identified relationships %d\" % len(all_properties))\n",
    "keys = list(infoboxes.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}