code/2_process_mturk_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process MTurk Results\n",
    "\n",
    "Computes agreement scores of hand-labeled sample using Mechnical Turk; checks for bad workers.\n",
    "\n",
    "## Inputs\n",
    "List of labeled tweets repeated over multiple workers with filename format 'myth_{myth_name}_sample_{sample_size}_{date}-results.csv'\n",
    "\n",
    "## Outputs\n",
    "N/A (agreement scores, worker validation)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import statistics\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "myth_name = str()\n",
    "sample_size = str()\n",
    "date = str()\n",
    "\n",
    "file_name = f'../data/samples/{myth_name}/myth_{myth_name}_sample_{sample_size}_{date}-results.csv'\n",
    "\n",
    "df = pd.read_csv(file_name)\n",
    "list(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in list(df):\n",
    "    if col.startswith('Input.full_text'):\n",
    "        full_text_col_name = col\n",
    "        break\n",
    "df['text_len'] = df[full_text_col_name].apply(len)\n",
    "df.plot(kind='scatter',x='text_len',y='WorkTimeInSeconds',color='blue')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Group by WorkerID, and take the average time to complete to look\n",
    "# for outliers that may have completed the tasks too fast.\n",
    "df.groupby('WorkerId')['WorkTimeInSeconds']\\\n",
    "  .agg(['mean', 'count'])\\\n",
    "  .sort_values(by='mean')\\\n",
    "  .plot(subplots=True, kind='bar', title='MTurk Worker Statistics')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The radio buttons from the MTurk form store their boolean result in a\n",
    "# single column; we will combine these into their respective questions.\n",
    "\n",
    "null_task_ids = set()\n",
    "\n",
    "def merge_radios(row, topic):\n",
    "    \"\"\"\n",
    "    To be used in an `apply` method to combine boolean radio buttons into a\n",
    "    single column\n",
    "    \"\"\"\n",
    "    if row['Answer.{}_yes.on'.format(topic)]:\n",
    "        return 'yes'\n",
    "    elif row['Answer.{}_no.on'.format(topic)]:\n",
    "        return 'no'\n",
    "    elif row['Answer.{}_unsure.on'.format(topic)]:\n",
    "        return 'unsure'\n",
    "    elif 'Answer.{}_broken_links.on'.format(topic) in list(df) and row['Answer.{}_broken_links.on'.format(topic)]:\n",
    "        return 'broken_links'\n",
    "    else:\n",
    "        if topic == 'myth_supports':\n",
    "            # Because this question is conditonal, some tasks might not have this field\n",
    "            # Assume they are NO\n",
    "            return 'no'\n",
    "        \n",
    "        raise ValueError(\"The chosen choice is not defined.\") # If the worker didn't choose any choices\n",
    "        # print(row.AssignmentId)\n",
    "        # null_task_ids.add(row.HITId)\n",
    "        # return None\n",
    "    \n",
    "df['is_myth'] = df.apply (lambda row: merge_radios(row, 'myth'), axis=1)\n",
    "df['is_myth_supports'] = df.apply (lambda row: merge_radios(row, 'myth_supports'), axis=1)\n",
    "\n",
    "print('There are {} tasks ids with null values'.format(len(null_task_ids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop tasks with null\n",
    "df = df[~df['HITId'].isin(list(null_task_ids))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop all tasks performed by rejected workers\n",
    "rejected_workers = [\"AAXYYH9MI3PJM\"]\n",
    "\n",
    "df = df[~df['WorkerId'].isin(rejected_workers)]\n",
    "\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# is_myth Answer Statistics\n",
    "\n",
    "def check_agree_on(col):\n",
    "    # Group the results by Task ID and their answer on whether tweet contains the focal myth, \n",
    "    # then count the number of records in those groups. This determines how many answers\n",
    "    # there were per choice, per task. Rename the count column.\n",
    "    df_gun_violence_counts = df.groupby(['HITId', col]).size().to_frame().reset_index()\n",
    "    df_gun_violence_counts.rename(columns={0: 'count'}, inplace=True)\n",
    "\n",
    "    # All three workers answered 'unsure'\n",
    "    unsure = df_gun_violence_counts[col] == 'unsure'\n",
    "    three = df_gun_violence_counts['count'] >= 2\n",
    "    num_all_unsure = len(df_gun_violence_counts[unsure & three])\n",
    "\n",
    "    # All three workers answered 'yes'\n",
    "    yes = df_gun_violence_counts[col] == 'yes'\n",
    "    num_all_yes = len(df_gun_violence_counts[yes & three])\n",
    "\n",
    "    # All three workers answered 'no'\n",
    "    no = df_gun_violence_counts[col] == 'no'\n",
    "    num_all_no = len(df_gun_violence_counts[no & three])\n",
    "\n",
    "    print('Agreed on \"yes\": {}\\nAgreed on \"No\": {}\\nAgreed on \"Unsure\": {}'.format(num_all_yes, num_all_no, num_all_unsure))\n",
    "\n",
    "    # There was no majority, answers were 'yes', 'no', and 'unsure'\n",
    "    df_gun_violence_count_size = df_gun_violence_counts.groupby('HITId').size().to_frame().rename(columns={0: 'count'})\n",
    "    # df_gun_violence_count_size[df_gun_violence_count_size['count'] == 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_agree_on(col=\"is_myth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_agree_on(col=\"is_myth_supports\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check data validity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These columns must not be None\n",
    "answer_cols = [ e for e in list(df) if e.startswith(\"Answers.\") ]\n",
    "for col in answer_cols:\n",
    "\n",
    "    for v in df[col]:\n",
    "        if v is None:\n",
    "            raise ValueError('None exists in {}'.format(col))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check that each task has three rated values\n",
    "rater_num = int() # Declare rater number\n",
    "for v in df.groupby('HITId').size():\n",
    "    if v != rater_num:\n",
    "        raise ValueError(\"There is a task with raters not equal to {}, found {}.\".format(rater_num, v))\n",
    "print('Every task has {} workers for each'.format(rater_num))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agreement Computation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Task-based agreement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_task_based_rating(df, answer_col, rater_num=5):\n",
    "    \"\"\"\n",
    "    Task-based rating score computation\n",
    "    \n",
    "    Args:\n",
    "        df:\n",
    "            A dataFrame with columns ['WorkerId', 'HITId', 'is_parenting'] ordered by ['HITId', 'WorkerId']\n",
    "        answer_col:\n",
    "            Column name to compute rating scores\n",
    "        rater_num:\n",
    "            Number of raters for each task\n",
    "            \n",
    "    Return:\n",
    "        A list of task-based rating scores\n",
    "    \"\"\"\n",
    "    \n",
    "    rating_scores = []\n",
    "    for i in range(0, df.shape[0], rater_num):\n",
    "        if len(set(df.iloc[i:i + rater_num]['HITId'])) != 1:\n",
    "            raise ValueError('Each task must contains {} rates, wrong at {}'.format(rater_num, i))\n",
    "        answers = df.iloc[i:i + rater_num][answer_col].tolist()\n",
    "        \n",
    "        # Get number of majority\n",
    "        majority_num = max(df.iloc[i:i + rater_num].groupby(answer_col).size())\n",
    "        # Store rating score\n",
    "        rating_scores.append(round(float(majority_num / rater_num), 2))\n",
    "        \n",
    "    return rating_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gun_violence\n",
    "def check_task_based_score(col):\n",
    "    df_task_based = df[['WorkerId', 'HITId', col]].sort_values(by=['HITId', 'WorkerId'])\n",
    "\n",
    "    # DataFrame for rating scores\n",
    "    df_task_based_rates = df_task_based.drop_duplicates(subset=['HITId'])\n",
    "    df_task_based_rates.drop(['WorkerId', col], axis=1, inplace=True)\n",
    "\n",
    "    # Compute scores\n",
    "    rating_scores = get_task_based_rating(df_task_based, answer_col=col, rater_num=rater_num)\n",
    "    df_task_based_rates = df_task_based_rates.assign(rating_scores=rating_scores)\n",
    "\n",
    "    # Sort and check statistics\n",
    "    df_task_based_rates.sort_values(by=['rating_scores'], inplace=True)\n",
    "\n",
    "    # Check stats\n",
    "    stats_task_based_gun_violence = df_task_based_rates.groupby('rating_scores').size().to_frame().reset_index().rename(columns={0: 'count'})\n",
    "    ax = stats_task_based_gun_violence.plot(kind='bar', x='rating_scores', y='count', color='blue', \\\n",
    "                   title='{}: stats of task-based agreement'.format(col))\n",
    "\n",
    "    for i, v in enumerate(stats_task_based_gun_violence['count']):\n",
    "        ax.text(i-0.05, v+5, str(v), va='center', fontsize=10, fontweight='bold')\n",
    "        \n",
    "    df_gun_violence_task_based_rates = df_task_based_rates.sort_values(by=['rating_scores'], ascending=False)\n",
    "    print(df_gun_violence_task_based_rates.head(10))\n",
    "    \n",
    "    only_high_task_based_tweet_ids = df_gun_violence_task_based_rates[df_gun_violence_task_based_rates['rating_scores'] >= 0.8]['HITId'].values\n",
    "    print(\"There are {} tweets with high task-based scores\".format(only_high_task_based_tweet_ids.shape[0]))\n",
    "    \n",
    "    avg_score = statistics.mean(df_gun_violence_task_based_rates['rating_scores'].values)\n",
    "    print(\"Avg task-based score for {} = {}\".format(col, avg_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_task_based_score(col='is_myth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_task_based_score(col='is_myth_supports')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Worker-based agreement\n",
    "\n",
    "Compute the score by comparing each task of each worker to the other. If an answer of a worker is equal to its majority of the corresponding task, then the number of correct answers of the work increases by one. The avg_score of each worker is computed by dividing the number of correct answers by total number of tasks that the worker have done."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_worker_based_rating(df, answer_col):\n",
    "    \"\"\"\n",
    "    Worker-based rating score computation\n",
    "    \n",
    "    Args:\n",
    "        df:\n",
    "            A dataFrame with columns ['WorkerId', 'HITId', 'is_parenting'] ordered by ['WorkerId', 'HITId']\n",
    "        answer_col:\n",
    "            Column name to compute rating scores\n",
    "            \n",
    "    Return:\n",
    "        A list of worker-based rating scores\n",
    "    \"\"\"\n",
    "    df = df.reset_index(drop=True)\n",
    "    \n",
    "    # Variables for rating score computation\n",
    "    current_woker_id = \"\"\n",
    "    total_task_num = 0\n",
    "    correct_answer_num = 0\n",
    "    \n",
    "    rating_scores = []\n",
    "    total_task_nums = []\n",
    "    for idx, row in df.iterrows():\n",
    "        worker_id = row['WorkerId']\n",
    "        task_id = row['HITId']\n",
    "        answer = row[answer_col]\n",
    "        \n",
    "        # Get next worker ID\n",
    "        if worker_id != current_woker_id:\n",
    "            current_woker_id = worker_id\n",
    "            \n",
    "            if idx > 0:\n",
    "                # Compute rating scores\n",
    "                rating_scores.append(correct_answer_num / total_task_num)\n",
    "                total_task_nums.append(total_task_num)\n",
    "\n",
    "                # Reset variables\n",
    "                total_task_num = 0\n",
    "                correct_answer_num = 0\n",
    "            \n",
    "        # Check whether the answer is the same as majority answer\n",
    "        df_answer_counts = df[df['HITId'] == task_id].groupby(answer_col).size().to_frame().rename(columns={0: 'count'}).sort_values(by=['count'], ascending=False).reset_index()\n",
    "        majority_count = df_answer_counts.iloc[0]['count']\n",
    "        \n",
    "        try:\n",
    "            answer_count = df_answer_counts[df_answer_counts[answer_col] == answer].iloc[0]['count']\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            print(df[df['HITId'] == task_id])\n",
    "        \n",
    "        if answer_count == majority_count and majority_count > 1:\n",
    "            is_majority = True\n",
    "        else:\n",
    "            is_majority = False\n",
    "        \n",
    "        # Counting if the answer is the same as the majority vote\n",
    "        if is_majority:\n",
    "            correct_answer_num += 1\n",
    "        total_task_num += 1\n",
    "        \n",
    "        # Last task\n",
    "        if idx + 1 > max(df.index):\n",
    "            # Compute rating scores\n",
    "            rating_scores.append(correct_answer_num / total_task_num)\n",
    "            total_task_nums.append(total_task_num)\n",
    "    \n",
    "    # print(len(set(df['WorkerId'].values)))\n",
    "    # print(\"Rating score: \" + str(len(rating_scores)))\n",
    "    # print(\"Total task number: \" + str(len(total_task_nums)))\n",
    "    \n",
    "    return rating_scores, total_task_nums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gun violence\n",
    "def check_worker_based_score(col):\n",
    "    df_worker_based = df[['WorkerId', 'HITId', col]].sort_values(by=['WorkerId', 'HITId'])\n",
    "\n",
    "    df_worker_based_rates = df_worker_based.drop_duplicates(subset=['WorkerId'])\n",
    "    df_worker_based_rates.drop(['HITId', col], axis=1, inplace=True)\n",
    "\n",
    "    # # Compute scores\n",
    "    rating_scores, total_task_nums = get_worker_based_rating(df_worker_based, answer_col=col)\n",
    "    df_worker_based_rates = df_worker_based_rates.assign(rating_scores=rating_scores)\n",
    "    df_worker_based_rates = df_worker_based_rates.assign(total_task_nums=total_task_nums)\n",
    "\n",
    "    # Filter workers that have done only one task\n",
    "    df_worker_based_rates = df_worker_based_rates[df_worker_based_rates['total_task_nums'] > 1]\n",
    "\n",
    "    # Check stats\n",
    "    stats_worker_based_gun_violence = pd.cut(df_worker_based_rates['rating_scores'].values, \\\n",
    "                       bins=[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], \\\n",
    "                       include_lowest=True).value_counts()\n",
    "    stats_worker_based_gun_violence.plot(kind='bar', x='rating_scores', y='count', color='blue', \\\n",
    "                   title='{}: stats of worker-based agreement\\n({} workers)'.format(col, df_worker_based_rates.shape[0]))\n",
    "    \n",
    "    df_gun_violence_worker_based_rates = df_worker_based_rates.sort_values(by=['rating_scores'], ascending=False)\n",
    "\n",
    "    avg_score = statistics.mean(df_gun_violence_worker_based_rates['rating_scores'].values)\n",
    "    \n",
    "    print(\"Avg worker-based score for {} = {}\".format(col, avg_score))\n",
    "    \n",
    "    done_enough_gun_violence_worker_ids = df_gun_violence_worker_based_rates[(df_gun_violence_worker_based_rates['rating_scores'] >= 0.5)]\\\n",
    "['WorkerId'].values\n",
    "    print(\"There are {} good workers\".format(len(done_enough_gun_violence_worker_ids)))\n",
    "    \n",
    "    bad_gun_violence_worker_ids = df_gun_violence_worker_based_rates[(df_gun_violence_worker_based_rates['rating_scores'] < 0.5)]\\\n",
    "['WorkerId'].values\n",
    "    print(\"There are {} bad workers\".format(len(bad_gun_violence_worker_ids)))\n",
    "    print(bad_gun_violence_worker_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_worker_based_score(\"is_myth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_worker_based_score(\"is_myth_supports\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 Alpha agreement score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.metrics import agreement\n",
    "\n",
    "data = df[['WorkerId', 'HITId', 'is_myth']].values\n",
    "\n",
    "data = [ e for e in data if e[2] != 'unsure' ]\n",
    "\n",
    "rating = agreement.AnnotationTask(data=data)\n",
    "\n",
    "#print(\"kappa \" + str(rating.kappa()))\n",
    "#print(\"fleiss \" + str(rating.multi_kappa()))\n",
    "print(\"alpha \" + str(rating.alpha()))\n",
    "#print(\"scotts \" + str(rating.pi()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = df[['WorkerId', 'HITId', 'is_myth_supports']].values\n",
    "\n",
    "data = [ e for e in data if e[2] != 'unsure' ]\n",
    "\n",
    "rating = agreement.AnnotationTask(data=data)\n",
    "\n",
    "#print(\"kappa \" + str(rating.kappa()))\n",
    "#print(\"fleiss \" + str(rating.multi_kappa()))\n",
    "print(\"alpha \" + str(rating.alpha()))\n",
    "#print(\"scotts \" + str(rating.pi()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "worker_ids = list(set(df['WorkerId']))\n",
    "print('There are {} raters'.format(len(worker_ids)))\n",
    "\n",
    "task_ids = list(set(df['HITId']))\n",
    "print('There are {} tweets'.format(len(task_ids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[['is_myth']].groupby('is_myth').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df[['is_myth_supports']].groupby('is_myth_supports').size()"
   ]
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}