code/5_sample_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample tweets for hand-labeling based on classification scores\n",
    "\n",
    "Uses classifiers trained on labeled tweets (about a myth vs. not) to filter tweets from March to August 2020 (especially April-May) to those with a higher probability of being in the minority class than the majority class. To help the classifier learn to detect both classes amidst our imbalanced data, the new sample is predicted to be 90% minority class and 10% minority class. This sample will be used to select tweets for hand-coding that fall into minority classes, which are hard to capture from the first round of ML models. Data source is tweets with hashtags related to Covid-19.\n",
    "\n",
    "### Inputs\n",
    "Unlabeled tweets; ML models to use for predictions\n",
    "\n",
    "### Outputs\n",
    "Sample of unlabeled tweets--mostly predicted to be misinformation--for hand-labeling, with filename format '{myth_name}_sample_{sample_size}_{date}.csv'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re, csv, os\n",
    "from datetime import date\n",
    "from random import sample\n",
    "from collections import Counter\n",
    "import gcsfs # for quick loading of data from gcloud\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "import time\n",
    "import emoji\n",
    "import math\n",
    "\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "stemmer = WordNetLemmatizer()\n",
    "import joblib\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "from pandas.plotting import scatter_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Custom function for loading data\n",
    "from utils import gcs_read_json_gz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define filepaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_fp = 'gs://COVID_TWEETS_DIR/'\n",
    "MYTH = str() # short-hand for myth\n",
    "\n",
    "# paths for ML model and vectorizer\n",
    "myth_mod_fp = f'{MYTH}_classifier.joblib'\n",
    "myth_vec_fp = f'{MYTH}_vectorizer.joblib'\n",
    "\n",
    "n_sample = 180\n",
    "thisday = date.today().strftime(\"%d%m%y\")\n",
    "\n",
    "# output path for sample\n",
    "myth_sam_fp = f'{MYTH}_sample_{str(n_sample)}_{str(thisday)}.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_tweets (dirname, \n",
    "                 num_rand_files = 0, \n",
    "                 start_week_num = -1, \n",
    "                 end_week_num = 0, \n",
    "                 levels = 1, \n",
    "                 nrows = 50, \n",
    "                 previous_tweets = [],\n",
    "                 language = 'en'):\n",
    "    \n",
    "    \"\"\"\n",
    "    Read latest Twitter data from JSON files. \n",
    "    Options: take random number of files from each folder, \n",
    "    look only at folders for weeks within specified range, and/or \n",
    "    read files in single folder (1 level) or folder of folders (2 levels).\n",
    "    \n",
    "    Args:\n",
    "        dirname: folder in Google Cloud Compute Server where Twitter JSON files live\n",
    "        num_rand_files: number of random files to draw from each folder. Useful when randomly sampling from files with lots of tweets\n",
    "        start_week_num: first week in range (to take tweets from), inclusive\n",
    "        end_week_num: last week in range (to take tweets from), inclusive\n",
    "        levels: number of folders in hierarchy ('dirname/file1, dirname/file2, etc.' = one level)\n",
    "        nrows: number of tweets to read from file\n",
    "        previous_tweets: list of previous samples to remove duplicate tweets\n",
    "        language: filter to only tweets in this language\n",
    "    Returns:\n",
    "        df: DataFrame with loaded in Twitter data\n",
    "    \"\"\"\n",
    "    \n",
    "    fs = gcsfs.GCSFileSystem(project=\"PROJECT\", token=\"cloud\")\n",
    "    \n",
    "    if levels == 2: # two levels (multiple folders)\n",
    "        files = []\n",
    "        folders = [folder for folder in fs.ls(dirname) \n",
    "                   if folder != dirname \n",
    "                   and os.path.join(\"gs://\", folder) != dirname] # don't keep duplicate folders = same as dirname\n",
    "        \n",
    "        for folder in folders:\n",
    "            # only process folders where: start_week_num < week_number < end_week_num\n",
    "            week_num = int(folder.split('/')[-1][9:]) # get week number: last part of file path, anything after '2020-week'\n",
    "            if week_num < start_week_num or (end_week_num > 0 and week_num > end_week_num):\n",
    "                continue # skip if not in week range\n",
    "            \n",
    "            # for folders of weeks in desired range, get their files\n",
    "            listf = fs.ls(os.path.join(\"gs://\", folder)) # get list of files in this folder\n",
    "            listf = [re.sub(r\"#\", r\"%23\", file) for file in listf] # in each file name, replace '#' with '%23' so pandas can read it\n",
    "            if num_rand_files > 0:\n",
    "                listf = sample(listf, num_rand_files) # get specified number of random files\n",
    "            files.extend(listf) # add each file to list\n",
    "            files = [x for x in files if x.endswith(\".json\") or x.endswith(\".json.gz\")]\n",
    "            \n",
    "    else: # one level (just one folder)\n",
    "        files = fs.ls(dirname)\n",
    "        files = [x for x in files if x.endswith(\".json\") or x.endswith(\".json.gz\")]\n",
    "        if num_rand_files > 0:\n",
    "            files = sample(listf, num_rand_files) # get specified number of random files\n",
    "    \n",
    "    print(dirname)\n",
    "    files = [x.replace(\"%23\", \"#\") for x in files]\n",
    "    print(files)\n",
    "    \n",
    "    print(f\"Reading in tweets from {len(files)} JSON files...\")\n",
    "    \n",
    "    # Load and merge files as DFs\n",
    "    dfs = []\n",
    "    for f in tqdm(files):\n",
    "        #print(\"gs://{}\".format(f))\n",
    "        # thisdf = pd.read_json(\"gs://{}\".format(f), nrows = nrows, lines=True, compression = 'gzip')\n",
    "        thisdf = gcs_read_json_gz(\"gs://{}\".format(f), nrows=nrows)\n",
    "        if 'lang' in thisdf.columns:\n",
    "            thisdf['language'] = thisdf['lang'] # funnel 'lang' to 'language' column\n",
    "            thisdf.drop(columns = 'lang', inplace = True) # erase 'lang' column (now a duplicate)\n",
    "        thisdf = thisdf[(thisdf['language'] == 'null') | (thisdf['language']==language)] # Filter to only tweets in language\n",
    "        dfs.append(thisdf)\n",
    "    df = pd.concat(dfs, ignore_index = True)\n",
    "    df['id'] = df['id'].astype(str)\n",
    "    \n",
    "    # Remove duplicate tweets\n",
    "    for id in previous_tweets:\n",
    "        df = df[df['id'] != id] \n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load & inspect data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load ML model(s) for classifying myths\n",
    "myth_mod = joblib.load(myth_mod_fp)\n",
    "\n",
    "# Load vectorizer(s) to keep vocab consistent with training data\n",
    "myth_vec = joblib.load(myth_vec_fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load previous samples for duplicate check\n",
    "\n",
    "previous_fps = [f'{MYTH}_sample_A.csv',\n",
    "                f'{MYTH}_sample_B.csv'\n",
    "              ]\n",
    "previous_dfs = []\n",
    "for fp in previous_fps:\n",
    "    previous_dfs.append(pd.read_csv(fp, names=['id']))\n",
    "previous_ids = pd.concat(previous_dfs, ignore_index = True)\n",
    "previous_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample Tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": false
   },
   "source": [
    "### Load and inspect the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# April - May\n",
    "df = read_tweets(bucket_fp, \n",
    "                 num_rand_files = 1, \n",
    "                 start_week_num = 14, \n",
    "                 end_week_num = 22, \n",
    "                 levels = 2, \n",
    "                 nrows = 5000,\n",
    "                 previous_tweets = previous_ids,\n",
    "                 language='en')\n",
    "\n",
    "print('Number of rows (tweets) and cols in DF:', str(df.shape))\n",
    "print()\n",
    "print('Columns in tweets DF:\\n', str(df.columns))\n",
    "print()\n",
    "\n",
    "# See examples of two tweets. \n",
    "# Have usernames and URLs already been replaced?\n",
    "print(\"Example tweet 1:\\n\", df['full_text'].iloc[0])\n",
    "print()\n",
    "print(\"Example tweet 2:\\n\", df['full_text'].iloc[10])\n",
    "print()\n",
    "\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess text data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "### Tweet Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def process_tweets(tweet):\n",
    "    '''\n",
    "    Preprocesses raw text of a tweet, skipping any retweets. \n",
    "    Steps: lower-casing; removing punctuation, newlines, URLs, usernames, and emojis;\n",
    "    stripping whitespace, replacing hashtags, and finally, lemmatization.\n",
    "    \n",
    "    args:\n",
    "        tweet: raw text of a tweet\n",
    "    \n",
    "    returns:\n",
    "        string: cleaned tweet text\n",
    "    '''\n",
    "    \n",
    "    # Skip retweets and non-strings\n",
    "    retweet_pattern = r'^RT\\s+' # recognize retweets by starting with 'RT'\n",
    "    if not isinstance(tweet, str) or re.search(retweet_pattern, tweet):\n",
    "        return ''\n",
    "    \n",
    "    # Convert to lowercase\n",
    "    tweet = tweet.lower()\n",
    "    \n",
    "    # Repair hashtag and remove newline character\n",
    "    # from text_helpers.tweet_text_cleanup\n",
    "    tweet = tweet.replace(\"# \", \"#\")\n",
    "    tweet = tweet.replace(\"\\n\", \" \")\n",
    "    \n",
    "    # remove URLs and @mentions\n",
    "    # Simple regular expression to match URLs starting with `https` or `http`\n",
    "    # More complex regex an be found here: https://mathiasbynens.be/demo/url-regex\n",
    "    url_regex = r\"https?://\\S*\"\n",
    "    # Regex to match mentions\n",
    "    mention_regex = r\"@\\S*\"\n",
    "    tweet = re.sub(url_regex, \"\", tweet)\n",
    "    tweet = re.sub(mention_regex, \"\", tweet)\n",
    "        \n",
    "    # Remove additional white spaces\n",
    "    whitespace_pattern = r'\\s+'\n",
    "    tweet = re.sub(whitespace_pattern, ' ', tweet) # strip whitespaces in between words\n",
    "    tweet = tweet.strip() # strip whitespaces at start & end\n",
    "    \n",
    "    # Replace #word with word\n",
    "    tweet = re.sub(r'#([^\\s]+)', r'\\1', tweet)\n",
    "    \n",
    "    # Remove emojis\n",
    "    tweet = emoji.get_emoji_regexp().sub(u'', tweet)\n",
    "    \n",
    "    # Lemmatization\n",
    "    tweet = tweet.split()\n",
    "    tweet = ' '.join([stemmer.lemmatize(word) for word in tweet])\n",
    "    \n",
    "    \n",
    "    return tweet\n",
    "\n",
    "print(\"Preprocessing tweets...\")\n",
    "df['text_cleaned'] = df['full_text'].progress_apply(lambda x: process_tweets(x))\n",
    "df = df[df['text_cleaned']!=''] # Filter to only non-empty text_cleaned tweets\n",
    "df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take a look at two (probably different) tweets post-preprocessing\n",
    "print(\"Example tweet 1 (cleaned):\\n\", df['text_cleaned'].iloc[0])\n",
    "print()\n",
    "print(\"Example tweet 2 (cleaned):\\n\", df['text_cleaned'].iloc[10])\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Check out vocab size after cleaning\n",
    "# Add words from each cleaned tweet to empty list:\n",
    "tweet_tokens_cleaned = []\n",
    "print(\"Tokenizing words for counting purposes...\")\n",
    "df['text_cleaned'].progress_apply(lambda x: tweet_tokens_cleaned.extend(word_tokenize(x))) # add each word to tokens list\n",
    "\n",
    "print('Vocabulary size for preprocessed tweets:', str(len(set(tweet_tokens_cleaned))))\n",
    "\n",
    "# Check out most frequent words in preprocessed text\n",
    "freq = Counter(tweet_tokens_cleaned)\n",
    "print('20 most frequent words in cleaned tweets:')\n",
    "freq.most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean up: Filter to key columns, including date of tweet via created_at\n",
    "final_df = df[['id','created_at','full_text','text_cleaned']]\n",
    "final_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute predictions for each tweet using model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def compute_prediction(tweet_text_col, vectorizer_model, class_model):\n",
    "    '''\n",
    "    Predicts the label for an input tweet using a given model trained to classify Covid-19-related myths in tweets. \n",
    "    Uses vectorizer_model to restrict the vocab of the input tweets so it's consistent with vocab in class_model (avoids errors).\n",
    "    \n",
    "    Args:\n",
    "        tweet_text_col: array of preprocessed tweet texts\n",
    "        vectorizer_model: fitted text vectorizer\n",
    "        class_model: trained classification model\n",
    "    Returns:\n",
    "        label: label for tweet_text predicted by model, false for tie\n",
    "        prob: probability for label\n",
    "    '''\n",
    "    \n",
    "    X = vectorizer_model.transform(tweet_text_col)\n",
    "    probabilities = class_model.predict_proba(X)\n",
    "    \n",
    "    label = 'no'\n",
    "    prob_no = probabilities[0][0]\n",
    "    prob_yes = probabilities[0][1]\n",
    "    \n",
    "    # predicted label is one with greater probability\n",
    "    if probabilities[0][0] < probabilities[0][1]:\n",
    "        label = 'yes'\n",
    "        \n",
    "    return label, prob_yes, prob_no\n",
    "\n",
    "final_df[['prediction_myth','prediction_myth_prob_yes','prediction_myth_prob_no']] = final_df['text_cleaned'].progress_apply(lambda x: pd.Series(compute_prediction([x], myth_vec, myth_mod)))\n",
    "\n",
    "final_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize distributions of labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_threshold(tweets_to_find, df):\n",
    "    '''\n",
    "    Calculates greatest threshold to find requested number of tweets in a dataframe series\n",
    "    \n",
    "    Args:\n",
    "        tweets_to_find: number of tweets to find\n",
    "        df: df series to search through\n",
    "    Returns:\n",
    "        threshold: minimum value of probability\n",
    "    \n",
    "    '''\n",
    "    threshold = .500\n",
    "    tweets_num = df[df > threshold].size\n",
    "    \n",
    "    while tweets_to_find > tweets_num:\n",
    "        threshold = round(threshold - .001,3)\n",
    "        tweets_num = df[df > threshold].size\n",
    "    return threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Probability Difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(final_df['prediction_myth_prob_no'] - final_df['prediction_myth_prob_yes']).hist(bins=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Minority Class (yes) Distribution\n",
    "We're interested in \"yes\" cases here because it's easy to find \"no\" cases. <br>\n",
    "Priority is to make sure our classifier can find \"yes\" cases (whether or not they are in majority in coded data)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df['prediction_myth_prob_no'].hist(bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = calculate_threshold(225, final_df['prediction_myth_prob_yes'])\n",
    "num = len(final_df[final_df['prediction_myth_prob_yes'] > threshold])\n",
    "prop = (num/len(final_df))*100\n",
    "print(f'{str(round(prop,3))}% ({num}) of cases are above {str(threshold)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select tweets for sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get sample of new tweets composed of 90% minority class and 10% majority class.<br>\n",
    "To improve fidelity of model, make the tweets selected majority class fuzzy/unreliable, so model gets better at labeling these.<br>\n",
    "First, filter into new DFs, one for minority class and one for (fuzzy) majority class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set thresholds for minority cases for each myth\n",
    "# set these as low as possible to capture ~140 tweets likely to fall in minority class\n",
    "capture_num = 162\n",
    "\n",
    "minority_threshold = calculate_threshold(capture_num,final_df['prediction_myth_prob_yes'])\n",
    "\n",
    "# filter using threshold\n",
    "df_myth_minority = final_df[final_df['prediction_myth_prob_yes'] > minority_threshold]\n",
    "\n",
    "# check out results\n",
    "print('Number of minority cases selected for each myth:')\n",
    "\n",
    "print(f'{len(df_myth_minority)} for {MYTH}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def check_pred_fuzzy(row, \n",
    "                     pred_labels = ['yes', 'no'], \n",
    "                     myth_labels = [MYTH],\n",
    "                     upper_threshold=0.3, \n",
    "                     lower_threshold=0.10):\n",
    "    \n",
    "    '''\n",
    "    Checks whether prediction is fuzzy/unreliable. Use this to determine if a tweet is worth hand-coding.\n",
    "    Rationale: by only coding tweets with unreliable labels, we can improve the classifier's ability to detect 'unsure' cases.\n",
    "    \n",
    "    Function focuses on this difference: prob(predicted label) - prob(some other label).\n",
    "    If difference is greater than lower_threshold (minimum for hand-coding of tweet to be possible), \n",
    "    but lesser than upper_threshold (maximum for hand-coding to be necessary), then it IS worth coding, so return True. \n",
    "    If difference is not between these, then we either it can't be reliably coded, or already have a reliable prediction,\n",
    "    so we don't need to hand-code --> return False.\n",
    "\n",
    "    To help select a threshold, ask: To what extent do we want the uncertainty to be, to help inform our sample selection?    \n",
    "    For example, if a tweet is labeled as POS with 90% prob and NEG 10% --> this is very obvious sample, so don't bother coding.\n",
    "    On the other hand, if the predictions are 51% NEG and 49 POS, then we need ppl to label this to update our model. \n",
    "    If our upper_threshold is 20% (0.20), then if a tweet has 61% NEG 39% POS probabilities, we don't choose it. \n",
    "    If some other tweet has prob 59% POS 41% NEG, we do select it for coding.\n",
    "    \n",
    "    Args:\n",
    "        row: row corresponding to tweet, with predictions in format...\n",
    "        pred_labels: labels for probabilities to use--used for naming columns\n",
    "        myth_labels: labels for COVID-19 myths to detect--used for naming columns\n",
    "        upper_threshold: max difference between predicted probs\n",
    "        lower_threshold: min difference between predicted probs\n",
    "    \n",
    "    Returns:\n",
    "        Array: True if tweet should be hand-coded, otherwise False. Array contains determinations for all myths\n",
    "    '''\n",
    "    \n",
    "    worth_coding = []\n",
    "    \n",
    "    for myth in myth_labels:\n",
    "        pred_label = row[f'prediction_{myth}'].strip() # get label of prediction for tweet--must be one of those in possible_labels!\n",
    "        pred_score = float(row[f'prediction_{myth}_prob_{pred_label}']) # get probability of predicted label (probably high)\n",
    "        \n",
    "        for pred in pred_labels: # Look at each label\n",
    "            pred = pred.strip() # clean label text\n",
    "            \n",
    "            if pred != pred_label: # if this label isn't the predicted one...\n",
    "                difference = pred_score - float(row[f'prediction_{myth}_prob_{pred}']) # ...then look at their difference in probability\n",
    "                if lower_threshold <= difference <= upper_threshold:\n",
    "                 # if difference in probs is > lower_threshold but < upper_threshold, then pred is fuzzy and we should code\n",
    "                    worth_coding.append(True) # worth coding\n",
    "                \n",
    "                else: worth_coding.append(False)\n",
    "                    \n",
    "    if len(worth_coding) == 1:\n",
    "        return worth_coding[0]\n",
    "    \n",
    "    else: return worth_coding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_myth_fuzzy = final_df[final_df.progress_apply(lambda x: check_pred_fuzzy(x, myth_labels = [MYTH], upper_threshold = .65), axis=1)]\n",
    "\n",
    "print(f'df_myth_fuzzy: {df_myth_fuzzy.shape}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compile sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_tweets(df, df_minority, df_fuzzy, sample_size=150, prop_maj=.1):\n",
    "    '''\n",
    "    Preliminary script to sample tweets using the previously determined minority and fuzzy df.\n",
    "    Does not take into account distribution of dates.\n",
    "    \n",
    "    Args:\n",
    "        df: original df with tweet id and full text\n",
    "        df_minority: minority df with tweets above threshold\n",
    "        sample_size: number of total tweets to sample\n",
    "        prop_maj: proportion of total tweets to be majority label/fuzzy\n",
    "        \n",
    "    '''\n",
    "    df_sample = pd.DataFrame()\n",
    "    min_size = int(sample_size * (1-prop_maj))\n",
    "    maj_size = sample_size - min_size\n",
    "    \n",
    "    for id in tqdm(df_minority['id'].sample(n=min_size)):\n",
    "        df_sample = df_sample.append(df.loc[df['id'] == id][['id','created_at','full_text']],ignore_index=True)\n",
    "    for id in tqdm(df_fuzzy['id'].sample(n=maj_size)):\n",
    "        df_sample = df_sample.append(df.loc[df['id'] == id][['id','created_at','full_text']],ignore_index=True)\n",
    "    df_sample = df_sample.sample(frac=1).reset_index(drop=True)\n",
    "    return df_sample\n",
    "\n",
    "myth_sample = sample_tweets(final_df, \n",
    "                            df_myth_minority, \n",
    "                            df_myth_fuzzy, \n",
    "                            sample_size = n_sample)\n",
    "print(myth_sample.head(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check date distribution and remove column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "myth_sample['created_at'].hist(bins = 30, xrot = 90)\n",
    "myth_sample.drop(columns = 'created_at', inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "myth_sample.to_csv(myth_sam_fp,\\\n",
    "    escapechar='\\\"', \\\n",
    "    quotechar='\\\"',\\\n",
    "    quoting=csv.QUOTE_ALL,\\\n",
    "    index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}