examples/LBKT/data_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "downloader, INFO http://base.ustc.edu.cn/data/ASSISTment/2009_skill_builder_data_corrected.zip is saved as ../../data/2009_skill_builder_data_corrected.zip\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading ../../data/2009_skill_builder_data_corrected.zip 100.00%: 8.66MB | 8.66MB"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "downloader, INFO ../../data/2009_skill_builder_data_corrected.zip is unzip to ../../data/2009_skill_builder_data_corrected\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from EduData import get_data\n",
    "import os\n",
    "\n",
    "data_path = '../../data/2009_skill_builder_data_corrected/'\n",
    "file_name = data_path + 'skill_builder_data_corrected.csv'\n",
    "if not os.path.exists(file_name):\n",
    "    get_data(\"assistment-2009-2010-skill\", \"../../data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tqdm\n",
    "\n",
    "data = pd.read_csv(\n",
    "    file_name,\n",
    "    usecols=['user_id', 'problem_id', 'skill_id', 'attempt_count', 'hint_count', 'correct','ms_first_response'], encoding='utf-8'\n",
    ").dropna(subset=['skill_id', 'problem_id'])\n",
    "data = data[data['ms_first_response'] > 0]\n",
    "data['time_first_res'] = data['ms_first_response'] / 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of users: 4163\n",
      "number of skills: 123\n",
      "number of problems: 17716\n"
     ]
    }
   ],
   "source": [
    "skills = data.skill_id.unique().tolist()\n",
    "problems = data.problem_id.unique().tolist()\n",
    "users = data.user_id.unique()\n",
    "\n",
    "# question id from 1 to #num_skill\n",
    "skill2id = { p: i+1 for i, p in enumerate(skills) }\n",
    "problem2id = { p: i+1 for i, p in enumerate(problems) }\n",
    "\n",
    "\n",
    "print(\"number of users: %d\" % len(users))\n",
    "print(\"number of skills: %d\" % len(skills))\n",
    "print(\"number of problems: %d\" % len(problems))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "problem2skill = {}\n",
    "for s, p in zip(np.array(data.skill_id), np.array(data.problem_id)):\n",
    "    problem2skill[problem2id[p]] = skill2id[s]\n",
    "with open(data_path + 'problem2skill', 'w', encoding='utf-8') as f:\n",
    "    f.write(str(problem2skill))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish merging\n",
      "Finish processing time features \n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from scipy.stats import norm\n",
    "from scipy.stats import poisson\n",
    "\n",
    "train_student_ids, test_student_ids = train_test_split(users, test_size=0.2, random_state=42)\n",
    "\n",
    "train_data = data[data['user_id'].isin(train_student_ids)]\n",
    "\n",
    "\n",
    "# compute the mean and variance of the response time for each question\n",
    "question_time_stats = train_data.groupby('problem_id')['time_first_res'].agg(['mean', 'std']).reset_index()\n",
    "\n",
    "# merge the time statistics to the original data\n",
    "data = pd.merge(data, question_time_stats, on='problem_id')\n",
    "data['std'] = data['std'].fillna(0)\n",
    "print(\"finish merging\")\n",
    "\n",
    "# compute the time factor with its distribution\n",
    "data['time_factor'] = data.apply(lambda row: 1 if row['std'] == 0 else norm(row['mean'], row['std']).cdf(np.log(row['time_first_res'])), axis=1)\n",
    "data = data.dropna(subset = ['time_factor'])\n",
    "print(\"Finish processing time features \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish processing attempt features \n"
     ]
    }
   ],
   "source": [
    "# compute the mean of the attempts\n",
    "question_attempt_stats = train_data.groupby('problem_id')['attempt_count'].mean().reset_index()\n",
    "question_attempt_stats.rename(columns = {'attempt_count':'mean_attempt'}, inplace = True)\n",
    "# merge the attempts statistics to the original data\n",
    "data = pd.merge(data, question_attempt_stats, on='problem_id', suffixes=('', '_attempt'))\n",
    "\n",
    "# compute the attempt factor with its distribution\n",
    "data['attempt_factor'] = 1 - poisson(data['mean_attempt']).cdf(data['attempt_count'] - 1)\n",
    "print(\"Finish processing attempt features \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish processing hint features \n"
     ]
    }
   ],
   "source": [
    "# compute the mean of the hints\n",
    "question_hint_stats = train_data.groupby('problem_id')['hint_count'].agg('mean').reset_index()\n",
    "question_hint_stats.rename(columns = {'hint_count':'mean_hint'}, inplace = True)\n",
    "# merge the hints statistics to the original data\n",
    "data = pd.merge(data, question_hint_stats, on='problem_id')\n",
    "\n",
    "# compute the hint factor with its distribution\n",
    "data['hint_factor'] = 1 - poisson(data['mean_hint']).cdf(data['hint_count'] - 1)\n",
    "\n",
    "print(\"Finish processing hint features \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "parse student sequence:\t: 100%|█████████████████████████████████████████████████████████████████████████████████████████| 3330/3330 [00:01<00:00, 1689.24it/s]\n",
      "parse student sequence:\t: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 833/833 [00:00<00:00, 1659.13it/s]\n"
     ]
    }
   ],
   "source": [
    "def parse_all_seq(students):\n",
    "    all_sequences = []\n",
    "    for student_id in tqdm.tqdm(students, 'parse student sequence:\\t'):\n",
    "        student_sequence = parse_student_seq(data[data.user_id == student_id])\n",
    "        all_sequences.extend([student_sequence])\n",
    "    return all_sequences\n",
    "\n",
    "\n",
    "def parse_student_seq(student):\n",
    "    seq = student\n",
    "    s = [skill2id[q] for q in seq.skill_id.tolist()]\n",
    "    a = seq.correct.tolist()\n",
    "    p = [problem2id[p] for p in seq.problem_id.tolist()]\n",
    "    time_factor = seq.time_factor.tolist()\n",
    "    attempt_factor = seq.attempt_factor.tolist()\n",
    "    hint_factor = seq.hint_factor.tolist()\n",
    "\n",
    "    return s, a, p, time_factor, attempt_factor, hint_factor\n",
    "\n",
    "\n",
    "train_data = np.array(parse_all_seq(train_student_ids), dtype=object)\n",
    "test_data = np.array(parse_all_seq(test_student_ids), dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "write data into file: ../../data/2009_skill_builder_data_corrected/train.txt: 100%|█████████████████████████████████████| 3330/3330 [00:00<00:00, 6110.41it/s]\n",
      "write data into file: ../../data/2009_skill_builder_data_corrected/test.txt: 100%|████████████████████████████████████████| 833/833 [00:00<00:00, 5741.12it/s]\n"
     ]
    }
   ],
   "source": [
    "def sequences2l(sequences, trg_path):\n",
    "    with open(trg_path, 'w', encoding='utf8') as f:\n",
    "        for seq in tqdm.tqdm(sequences, 'write data into file: %s' % trg_path):\n",
    "            s_seq, a_seq, p_seq, time_seq, attempt_seq, hint_seq = seq\n",
    "            seq_len = len(s_seq)\n",
    "            f.write(str(seq_len) + '\\n')\n",
    "            f.write(','.join([str(s) for s in s_seq]) + '\\n')\n",
    "            f.write(','.join([str(a) for a in a_seq]) + '\\n')\n",
    "            f.write(','.join([str(p) for p in p_seq]) + '\\n')\n",
    "            f.write(','.join([format(t, '.6f') for t in time_seq]) + '\\n')\n",
    "            f.write(','.join([format(att, '.6f') for att in attempt_seq]) + '\\n')\n",
    "            f.write(','.join([format(h, '.6f') for h in hint_seq]) + '\\n')\n",
    "\n",
    "sequences2l(train_data, data_path + 'train.txt')\n",
    "sequences2l(test_data, data_path + 'test.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}