examples/DKVMN/prepare_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "downloader, INFO http://base.ustc.edu.cn/data/ASSISTment/2009_skill_builder_data_corrected.zip is saved as ..\\..\\data\\2009_skill_builder_data_corrected.zip\n",
      "downloader, INFO file existed, skipped\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'..\\\\..\\\\data\\\\2009_skill_builder_data_corrected'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from EduData import get_data\n",
    "\n",
    "get_data(\"assistment-2009-2010-skill\", \"../../data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>correct</th>\n",
       "      <th>skill_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>33022537</td>\n",
       "      <td>64525</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33022709</td>\n",
       "      <td>64525</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35450204</td>\n",
       "      <td>70363</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35450295</td>\n",
       "      <td>70363</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35450311</td>\n",
       "      <td>70363</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  user_id  correct  skill_id\n",
       "0  33022537    64525        1       1.0\n",
       "1  33022709    64525        1       1.0\n",
       "2  35450204    70363        0       1.0\n",
       "3  35450295    70363        1       1.0\n",
       "4  35450311    70363        0       1.0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "import pandas as pd\n",
    "import tqdm\n",
    "import numpy as np\n",
    "\n",
    "data = pd.read_csv(\n",
    "    '../../data/2009_skill_builder_data_corrected/skill_builder_data_corrected.csv',\n",
    "    usecols=['order_id', 'user_id', 'skill_id', 'correct']\n",
    ").dropna(subset=['skill_id'])\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of skills: 123\n"
     ]
    }
   ],
   "source": [
    "raw_question = data.skill_id.unique().tolist()\n",
    "num_skill = len(raw_question)\n",
    "\n",
    "# question id from 0 to (num_skill - 1)\n",
    "questions = { p: i for i, p in enumerate(raw_question) }\n",
    "\n",
    "print(\"number of skills: %d\" % num_skill)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "parse student sequence:\t: 100%|██████████| 4163/4163 [00:08<00:00, 468.66it/s]\n"
     ]
    }
   ],
   "source": [
    "def parse_all_seq(students):\n",
    "    all_sequences = []\n",
    "    for student_id in tqdm.tqdm(students, 'parse student sequence:\\t'):\n",
    "        student_sequence = parse_student_seq(data[data.user_id == student_id])\n",
    "        all_sequences.extend([student_sequence])\n",
    "    return all_sequences\n",
    "\n",
    "\n",
    "def parse_student_seq(student):\n",
    "    seq = student.sort_values('order_id')\n",
    "    q = [questions[q] for q in seq.skill_id.tolist()]\n",
    "    a = seq.correct.tolist()\n",
    "    return q, a\n",
    "\n",
    "\n",
    "# [(question_sequence_0, answer_sequence_0), ..., (question_sequence_n, answer_sequence_n)]\n",
    "sequences = parse_all_seq(data.user_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test_split(data, train_size=.7, shuffle=True):\n",
    "    if shuffle:\n",
    "        random.shuffle(data)\n",
    "    boundary = round(len(data) * train_size)\n",
    "    return data[: boundary], data[boundary:]\n",
    "\n",
    "\n",
    "train_sequences, test_sequences = train_test_split(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "write into file: 100%|██████████| 2914/2914 [00:00<00:00, 12347.44it/s]\n",
      "write into file: 100%|██████████| 1249/1249 [00:00<00:00, 11783.26it/s]\n",
      "write into file: 100%|██████████| 145/145 [00:00<00:00, 16111.85it/s]\n",
      "write into file: 100%|██████████| 62/62 [00:00<00:00, 15496.50it/s]\n"
     ]
    }
   ],
   "source": [
    "def sequences2tl(sequences, trgpath):\n",
    "    with open(trgpath, 'a', encoding='utf8') as f:\n",
    "        for seq in tqdm.tqdm(sequences, 'write into file: '):\n",
    "            questions, answers = seq\n",
    "            seq_len = len(questions)\n",
    "            f.write(str(seq_len) + '\\n')\n",
    "            f.write(','.join([str(q) for q in questions]) + '\\n')\n",
    "            f.write(','.join([str(a) for a in answers]) + '\\n')\n",
    "\n",
    "\n",
    "# save triple line format for other tasks\n",
    "sequences2tl(train_sequences, '../../data/2009_skill_builder_data_corrected/train.txt')\n",
    "sequences2tl(test_sequences, '../../data/2009_skill_builder_data_corrected/test.txt')\n",
    "\n",
    "\n",
    "# reduce the amount of data for example running faster\n",
    "percentage = 0.05\n",
    "train_sequences = train_sequences[:int(len(train_sequences)*percentage)]\n",
    "test_sequences = test_sequences[:int(len(test_sequences)*percentage)]\n",
    "sequences2tl(train_sequences, '../../data/2009_skill_builder_data_corrected/example_train.txt')\n",
    "sequences2tl(test_sequences, '../../data/2009_skill_builder_data_corrected/example_test.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}