SentenceLevelPred.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\hetan\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "c:\\Users\\hetan\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "c:\\Users\\hetan\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\transformers\\utils\\generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  _torch_pytree._register_pytree_node(\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from summac.model_summac import SummaCZS, SummaCImager\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, roc_auc_score,f1_score\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Zero-Shot SummaC Article Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_zs = SummaCZS(granularity=\"sentence\", model_name=\"vitc\", device=\"cpu\")  # Use GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_threshold(validation_df,model):\n",
    "    true_labels = []\n",
    "    predicted_scores = []\n",
    "\n",
    "    for index, row in tqdm(validation_df.iterrows(),total=len(validation_df)):\n",
    "        document = row['Scraped Content']\n",
    "        claim = row['Headline']\n",
    "        score = model.score([str(document)], [str(claim)])\n",
    "\n",
    "        true_labels.append(row['Actual Decision'])\n",
    "        predicted_scores.append(score[\"scores\"][0])\n",
    "\n",
    "    # Evaluate performance at different thresholds\n",
    "    thresholds = [i * 0.01 for i in range(-100, 101)]\n",
    "    best_threshold = 0\n",
    "    best_accuracy = 0\n",
    "\n",
    "    for threshold in thresholds:\n",
    "        predicted_labels = [1 if score >= threshold else 0 for score in predicted_scores]\n",
    "\n",
    "        accuracy = accuracy_score(true_labels, predicted_labels)\n",
    "        precision = precision_score(true_labels, predicted_labels, zero_division=1)\n",
    "        recall = recall_score(true_labels, predicted_labels)\n",
    "        f1 = f1_score(true_labels, predicted_labels)\n",
    "\n",
    "        print(f\"Threshold: {threshold:.2f}\")\n",
    "        print(f\"Accuracy: {accuracy:.3f}\")\n",
    "        print(f\"Precision: {precision:.3f}\")\n",
    "        print(f\"Recall: {recall:.3f}\")\n",
    "        print(f\"F1 Score: {f1:.3f}\")\n",
    "        print('-' * 50)\n",
    "\n",
    "        if accuracy > best_accuracy:\n",
    "            best_accuracy = accuracy\n",
    "            best_threshold = threshold\n",
    "\n",
    "    print(f\"Best Threshold: {best_threshold:.2f}\")\n",
    "    print(f\"Best Accuracy: {best_accuracy:.3f}\")\n",
    "\n",
    "    roc_auc = roc_auc_score(true_labels, predicted_scores)\n",
    "    print(f\"ROC-AUC: {roc_auc:.3f}\")\n",
    "    return best_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST SET\n",
    "def test_accuracy(test_df, best_threshold, model, imager):\n",
    "    true_test_labels = []\n",
    "    predicted_test_scores = []\n",
    "    \n",
    "    # Create lists to store interesting cases\n",
    "    correct_predictions = []\n",
    "    incorrect_predictions = []\n",
    "    \n",
    "    for index, row in tqdm(test_df.iterrows(), total=len(test_df)):\n",
    "        document = row['Scraped Content']\n",
    "        claim = row['Headline']\n",
    "        score = model.score([str(document)], [str(claim)])\n",
    "        true_label = row['Actual Decision']\n",
    "        predicted_score = score[\"scores\"][0]\n",
    "        \n",
    "        predicted_label = 1 if predicted_score >= best_threshold else 0\n",
    "        \n",
    "        # Store interesting cases\n",
    "        if predicted_label == true_label:\n",
    "            correct_predictions.append(row)\n",
    "        else:\n",
    "            incorrect_predictions.append(row)\n",
    "            \n",
    "        true_test_labels.append(true_label)\n",
    "        predicted_test_scores.append(predicted_score)\n",
    "    \n",
    "    # Calculate metrics\n",
    "    predicted_test_labels = [1 if score >= best_threshold else 0 for score in predicted_test_scores]\n",
    "    test_accuracy = accuracy_score(true_test_labels, predicted_test_labels)\n",
    "    test_precision = precision_score(true_test_labels, predicted_test_labels, zero_division=1)\n",
    "    test_recall = recall_score(true_test_labels, predicted_test_labels)\n",
    "    test_f1 = f1_score(true_test_labels, predicted_test_labels)\n",
    "    test_roc_auc = roc_auc_score(true_test_labels, predicted_test_scores)\n",
    "    \n",
    "    print(f\"Test Accuracy: {test_accuracy:.3f}\")\n",
    "    print(f\"Test Precision: {test_precision:.3f}\")\n",
    "    print(f\"Test Recall: {test_recall:.3f}\")\n",
    "    print(f\"Test F1 Score: {test_f1:.3f}\")\n",
    "    print(f\"Test ROC-AUC: {test_roc_auc:.3f}\")\n",
    "    \n",
    "    return correct_predictions, incorrect_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: The `best_threshold` is manually set from our experiments in `Pipeline_Results_SummaC`. \\\n",
    " \\\n",
    "Additionally, the results obtained using `best_threshold` pertaining to *F1* and *Accuracy* are calcualted on a subset of the dataset of 16 claims."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/16 [00:00<?, ?it/s]c:\\Users\\hetan\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\huggingface_hub\\file_download.py:795: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "c:\\Users\\hetan\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\transformers\\utils\\generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  _torch_pytree._register_pytree_node(\n",
      "c:\\Users\\hetan\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\transformers\\utils\\generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  _torch_pytree._register_pytree_node(\n",
      "100%|██████████| 16/16 [05:59<00:00, 22.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.875\n",
      "Test Precision: 1.000\n",
      "Test Recall: 0.750\n",
      "Test F1 Score: 0.857\n",
      "Test ROC-AUC: 0.859\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Initialize the imager\n",
    "imager = SummaCImager(model_name=\"vitc\", device=\"cpu\")  # Use GPU if available\n",
    "\n",
    "# Your existing code with visualization added\n",
    "df = pd.read_csv('./Efficiency test/SummaC_Pipeline_Article.csv', delimiter='|')\n",
    "test_df, validation_df = train_test_split(df, test_size=0.2, random_state=42, stratify=df['Actual Decision'])\n",
    "\n",
    "# Get best threshold from the Pipeline_Results_SummaC.ipynb for Article_Pipeline with SummaC_ZS\n",
    "best_threshold = -0.03 \n",
    "\n",
    "# Run test with visualization\n",
    "correct_preds, incorrect_preds = test_accuracy(test_df, best_threshold, model_zs, imager)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Showing the document segment with the most impact on sentence level granularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\hetan\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\huggingface_hub\\file_download.py:795: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Analyzing sample predictions...\n",
      "\n",
      "=== TRUE CLAIMS ===\n",
      "\n",
      "=== Analysis Results ===\n",
      "\n",
      "CLAIM: Sepsis tests take days  putting patients at risk. A new method may cut wait time\n",
      "\n",
      "DOCUMENT TEXT:\n",
      "\u001b[1;32mSepsis tests take days, putting patients at risk.\u001b[0m\n",
      "A new method may cut wait time  Every print subscription comes with full digital access The test is crucial to figuring out what is causing a bloodstream infection, and fighting it A nurse holds tubes of collected blood from a patient.\n",
      "A new method of sepsis testing could speed treatment that targets the specific bacteria causing a blood infection.\n",
      "FLUXFACTORY/GETTY IMAGES By Claire Yuan JULY 24, 2024 AT 11:00 AM When bloodstream infections set in, fast treatment is crucial — but it can take several days to identify the bacteria responsible.\n",
      "A new, rapid-diagnosis sepsis test could cut down on the wait, reducing testing time from as much as a few days to about 13 hours by cutting out a lengthy blood culturing step, researchers report July 24 in Nature.\n",
      "“They are pushing the limits of rapid diagnostics for bloodstream infections,” says Pak Kin Wong, a biomedical engineer at Penn State who was not involved in the research.\n",
      "“They are driving toward a direction that will dramatically improve the clinical management of bloodstream infections and sepsis.” Sepsis — an immune system overreaction to an infection — is a life-threatening condition that strikes nearly 2 million people per year in the United States, killing more than 250,000 (SN: 5/18/08).\n",
      "The condition can also progress to septic shock, a steep drop in blood pressure that damages the kidneys, lungs, liver and other organs.\n",
      "It can be caused by a broad range of different bacteria, making species identification key for personalized treatment of each patient.\n",
      "In conventional sepsis testing, the blood collected from the patient must first go through a daylong blood culturing step to grow more bacteria for detection.\n",
      "The sample then goes through a second culture for purification before undergoing testing to find the best treatment.\n",
      "During the two to three days required for testing, patients are placed on broad-spectrum antibiotics — a blunt tool designed to stave off a mystery infection that’s better treated by targeted antibiotics after figuring out the specific bacteria causing the infection.\n",
      "Nanoengineer Tae Hyun Kim and colleagues found a way around the initial 24-hour blood culture.\n",
      "The workaround starts by injecting a blood sample with nanoparticles decorated with a peptide designed to bind to a wide range of blood-borne pathogens.\n",
      "Magnets then pull out the nanoparticles, and the bound pathogens come with them.\n",
      "Those bacteria are sent directly to the pure culture.\n",
      "Thanks to this binding and sorting process, the bacteria can grow faster without extraneous components in the sample, like blood cells and the previously given broad-spectrum antibiotics, says Kim, of Seoul National University in South Korea.\n",
      "Cutting out the initial blood culturing step also relies on a new imaging algorithm, Kim says.\n",
      "To test bacteria’s susceptibility to antibiotics, both are placed in the same environment, and scientists observe if and how the antibiotics stunt the bacteria’s growth or kill them.\n",
      "The team’s image detection algorithm can detect subtler changes than the human eye can.\n",
      "So it can identify the species and antibiotic susceptibility with far fewer bacteria cells than the conventional method, thereby reducing the need for long culture times to produce larger colonies.\n",
      "Though the new method shows promise, Wong says, any new test carries a risk of false negatives, missing bacteria that are actually present in the bloodstream.\n",
      "That in turn can lead to not treating an active infection, and “undertreatment of bloodstream infection can be fatal,” he says.\n",
      "“While the classical blood culture technique is extremely slow, it is very effective in avoiding false negatives.” Following their laboratory-based experiments, Kim and colleagues tested their new method clinically, running it in parallel with conventional sepsis testing on 190 hospital patients with suspected infections.\n",
      "The testing obtained a 100 percent match on correct bacterial species identification, the team reports.\n",
      "Though more clinical tests are needed, these accuracy results are encouraging so far, Kim says.\n",
      "The team is continuing to refine their design in hopes of developing a fully automated sepsis blood test that can quickly produce results, even when hospital laboratories are closed overnight.\n",
      "“We really wanted to commercialize this and really make it happen so that we could make impacts to the patients,” Kim says.\n",
      "Questions or comments on this article?\n",
      "E-mail us at feedback@sciencenews.org   Reprints FAQ T.H.\n",
      "Blood culture-free ultrarapid antimicrobial susceptibility testing.\n",
      "Published online July 24, 2024. doi: 10.1038/s41586-024-07725-1.\n",
      "Claire Yuan is the 2024 AAAS Mass Media Fellow at Science News.\n",
      "She is an undergraduate at Harvard University studying chemistry & physics and history of science.\n",
      "We are at a critical time and supporting science journalism is more important than ever.\n",
      "Science News and our parent organization, the Society for Science, need your help to strengthen scientific literacy and ensure that important societal decisions are made with science in mind.\n",
      "Please subscribe to Science News and add $16 to expand science literacy and understanding.\n",
      "Science News was founded in 1921 as an independent, nonprofit source of accurate information on the latest news of science, medicine and technology.\n",
      "Today, our mission remains the same: to empower people to evaluate the news and the world around them.\n",
      "It is published by the Society for Science, a nonprofit 501(c)(3) membership organization dedicated to public engagement in scientific research and education (EIN 53-0196483).\n",
      "© Society for Science & the Public 2000–2024.\n",
      "All rights reserved.\n",
      "\n",
      "DETAILED RELATIONSHIPS:\n",
      "\n",
      "Segment: Sepsis tests take days, putting patients at risk.\n",
      "Relationship: ENTAILMENT\n",
      "Scores - E: 0.993 | C: 0.001 | N: 0.006\n",
      "\n",
      "Score Distribution:\n",
      "Entailment:     █████████████████████████████████████████████████\n",
      "Contradiction:  \n",
      "Neutral:        \n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "=== Analysis Results ===\n",
      "\n",
      "CLAIM: Say goodbye to back pain  patients go for advanced endoscopy spine surgery for sciatica\n",
      "\n",
      "DOCUMENT TEXT:\n",
      "\u001b[1;32mSay goodbye to back pain, patients go for advanced endoscopy spine surgery for sciatica  Kapil Dixit is a graduate of the Indian Institute of Mass Communications.\u001b[0m\n",
      "He has covered crime at regional as well as state level.\n",
      "His hobbies include reading, writing and meeting people with diverse interests.\n",
      "\n",
      "DETAILED RELATIONSHIPS:\n",
      "\n",
      "Segment: Say goodbye to back pain, patients go for advanced endoscopy spine surgery for sciatica  Kapil Dixit is a graduate of the Indian Institute of Mass Communications.\n",
      "Relationship: ENTAILMENT\n",
      "Scores - E: 0.976 | C: 0.003 | N: 0.022\n",
      "\n",
      "Score Distribution:\n",
      "Entailment:     ████████████████████████████████████████████████\n",
      "Contradiction:  \n",
      "Neutral:        █\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "=== FALSE CLAIMS ===\n",
      "\n",
      "=== Analysis Results ===\n",
      "\n",
      "CLAIM: \"OpenAI unveils a new search engine dubbed SearchGPT; Alphabet shares soar\"\n",
      "\n",
      "DOCUMENT TEXT:\n",
      "\u001b[1;31mOpenAI announces a search engine called SearchGPT; Alphabet shares dip                                                                                                                      In this article OpenAI on Thursday announced a prototype of its search engine, called SearchGPT, which aims to give users “fast and timely answers with clear and relevant sources.” The company said it eventually plans to integrate the tool, which is currently being tested with a small group of users, into its ChatGPT chatbot.\u001b[0m\n",
      "The rollout could have implications for Google and its dominant search engine.\n",
      "Since the launch of ChatGPT in November 2022, Alphabet investors have been concerned that OpenAI could take market share from Google in search by giving consumers new ways to seek information online.\n",
      "With this prototype, OpenAI is testing the waters for doing just that, promising users the chance to “search in a more natural, intuitive way” and ask follow-up questions “just like you would in a conversation.” “We think there is room to make search much better than it is today,” OpenAI CEO Sam Altman wrote Thursday in a post on X.\n",
      "Alphabet shares fell more than 3% on Thursday to close at $167.28, while the Nasdaq was down less than 1%.\n",
      "In May, Google launched AI Overview, which CEO Sundar Pichai called the biggest change in search in 25 years, to a limited audience, allowing users to see a summary of answers to queries at the very top of Google Search.\n",
      "Though Google had been working on AI Overview for more than a year, public criticism mounted after  users quickly noticed that queries returned nonsensical or inaccurate results within the AI feature — without any way to opt out.\n",
      "“Google has been kind of shaking in their boots a little bit since this stuff first popped off,” said Daniel Faggella, founder and head of research at Emerj Artificial Intelligence Research, referring to generative artificial intelligence.\n",
      "“We haven’t seem their company crumble in the interim, but we have seen them kind of fumble.”  The SearchGPT announcement follows OpenAI’s launch last Thursday of a new AI model, “GPT-4o mini.” The new model is an offshoot of GPT-4o, the startup’s fastest and most powerful model to date, which it launched in May during a livestreamed event with executives.\n",
      "OpenAI, backed by Microsoft, has been valued at more than $80 billion by investors.\n",
      "The company, founded in 2015, is under pressure to stay on top of the generative AI market while finding ways to make money as it spends massive sums on processors and infrastructure to build and train its models.\n",
      "Last Month, OpenAI announced the hiring of two top executives as well as a partnership with Apple that includes a ChatGPT-Siri integration.\n",
      "Sarah Friar, previously CEO of Nextdoor and finance chief at Square, joined as chief financial officer, and Kevin Weil, an ex-president at Planet Labs and former senior vice president at Twitter and a vice president at Facebook and Instagram, joined as chief product officer.\n",
      "OpenAI is bolstering its C-suite as its large language models gain importance across the tech sector and as competition rapidly emerges in the burgeoning generative AI market.\n",
      "Both OpenAI’s new mini AI model and the prototype of SearchGPT are also part of the company’s push to be at the forefront of “multimodality,” or the ability to offer a wide range of types of AI-generated media, like text, images, audio, video and search, inside one tool: ChatGPT.\n",
      "For SearchGPT, OpenAI’s blog post said the tool’s visual results will lead to “richer understanding” for users.\n",
      "Last year, OpenAI Chief Operating Officer Brad Lightcap told CNBC, “The world is multimodal.” He added that as humans “engage with the world, we see things, we hear things, we say things,” so limiting interactions to text is insufficient.\n",
      "WATCH: AI’s trillion dollar time bomb Got a confidential news tip?\n",
      "We want to hear from you.\n",
      "Sign up for free newsletters and get more CNBC delivered to your inbox Get this delivered to your inbox, and more info about our products and services.\n",
      "© 2024 CNBC LLC.\n",
      "All Rights Reserved.\n",
      "A Division of NBCUniversal Data is a real-time snapshot *Data is delayed at least 15 minutes.\n",
      "Global Business and Financial News, Stock Quotes, and Market Data and Analysis.\n",
      "Data also provided by\n",
      "\n",
      "DETAILED RELATIONSHIPS:\n",
      "\n",
      "Segment: OpenAI announces a search engine called SearchGPT; Alphabet shares dip                                                                                                                      In this article OpenAI on Thursday announced a prototype of its search engine, called SearchGPT, which aims to give users “fast and timely answers with clear and relevant sources.” The company said it eventually plans to integrate the tool, which is currently being tested with a small group of users, into its ChatGPT chatbot.\n",
      "Relationship: CONTRADICTION\n",
      "Scores - E: 0.247 | C: 0.712 | N: 0.040\n",
      "\n",
      "Score Distribution:\n",
      "Entailment:     ████████████\n",
      "Contradiction:  ███████████████████████████████████\n",
      "Neutral:        ██\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "=== Analysis Results ===\n",
      "\n",
      "CLAIM: \"Global crypto market experiences unexpected surge in market cap as bitcoin briefly soars above $50,000\"\n",
      "\n",
      "DOCUMENT TEXT:\n",
      "\u001b[1;95mFor further information, do not hesitate to contact us.\u001b[0m\n",
      "Ref: 35.240.133.145 2024-09-08T18:12:49.937Z\n",
      "\n",
      "DETAILED RELATIONSHIPS:\n",
      "\n",
      "Segment: For further information, do not hesitate to contact us.\n",
      "Relationship: NEUTRAL\n",
      "Scores - E: 0.006 | C: 0.042 | N: 0.953\n",
      "\n",
      "Score Distribution:\n",
      "Entailment:     \n",
      "Contradiction:  ██\n",
      "Neutral:        ███████████████████████████████████████████████\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "=== TRUE CLAIMS ===\n",
      "\n",
      "=== Analysis Results ===\n",
      "\n",
      "CLAIM: Tomba del Cerbero: Archaeologists unseal 2 000-year-old tomb in Italy\n",
      "\n",
      "DOCUMENT TEXT:\n",
      "\u001b[1;31mTomba del Cerbero: Archaeologists unseal 2,000-year-old tomb in Italy  In Giugliano in Italy, archaeologists were taken aback while examining the interior of the Tomba del Cerbero.\u001b[0m\n",
      "The experts made a surprising discovery during their investigation using a microcamera.\n",
      "They found that it was feasible to physically unseal and open the 2,000-year-old stone coffin, also known as a sarcophagus, which had remained untouched for centuries (Photo: Superintendence archaeology fine arts & landscape for the naples metropolitan area) In 2023, archaeologists made a remarkable discovery in Giugliano when they unearthed the perimeter of an ancient necropolis.\n",
      "This finding led them to uncover a structure that was later identified as the entrance to a chamber tomb, now known as the 'tomb of Cerberus.'\n",
      "Upon further investigation, the archaeologists were astounded by the presence of a captivating fresco within the tomb.\n",
      "The fresco showcased Cerberus, the legendary three-headed dog from ancient Greek mythology.\n",
      "(Photo: Superintendence archaeology fine arts & landscape for the Naples metropolitan area) Guigliano, a town situated in the Campania region of Italy, has a rich history dating back to ancient times.\n",
      "The area was settled by Greek colonizers during the period spanning the 8th and 7th centuries BC.\n",
      "Archaeologists from the Superintendency of Archaeology, Fine Arts, and Landscape made a startling discovery when they uncovered a tomb containing remarkably well-preserved bodies.\n",
      "The team was astonished not only by the condition of the remains but also by the unique preservation technique employed.\n",
      "The mummies, which have yet to be publicly showcased through images, were treated with a distinctive mixture of herbs.\n",
      "The ancient embalmers utilized a cream consisting of chenopodium, \"a genus of perennial herb colloquially known as goosefoot,\" and \"absinthium (wormwood)\" to preserve the bodies.\n",
      "(Photo: Superintendence archaeology fine arts & landscape for the Naples metropolitan area) Mariano Nuzzo, an archaeology superintendent working for the Italian Ministry of Culture, stated in a press release that \"The Tomb of Cerberus continues to provide valuable information on the Phlegraean territory near Liternum, expanding knowledge of the past, and, and offering opportunities for research also of a multidisciplinary nature.\"\n",
      "(Photo: Superintendence archaeology fine arts & landscape for the Naples metropolitan area) 10 places struggling with overtourism right now Deepika and Ranveer blessed with a baby girl, celebs who became parents this year 11 animals that have extraordinary night vision Allu Arjun, Nayanthara, Varun Tej Konidela: Ganesh Chaturthi Celebrations of South Celebs\n",
      "\n",
      "DETAILED RELATIONSHIPS:\n",
      "\n",
      "Segment: Tomba del Cerbero: Archaeologists unseal 2,000-year-old tomb in Italy  In Giugliano in Italy, archaeologists were taken aback while examining the interior of the Tomba del Cerbero.\n",
      "Relationship: CONTRADICTION\n",
      "Scores - E: 0.156 | C: 0.736 | N: 0.108\n",
      "\n",
      "Score Distribution:\n",
      "Entailment:     ███████\n",
      "Contradiction:  ████████████████████████████████████\n",
      "Neutral:        █████\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "=== Analysis Results ===\n",
      "\n",
      "CLAIM: Neurodivergent children more likely to develop chronic fatigue as teen  study finds\n",
      "\n",
      "DOCUMENT TEXT:\n",
      "\u001b[1;95mcould not scrape content\u001b[0m\n",
      "\n",
      "DETAILED RELATIONSHIPS:\n",
      "\n",
      "Segment: could not scrape content\n",
      "Relationship: NEUTRAL\n",
      "Scores - E: 0.020 | C: 0.128 | N: 0.853\n",
      "\n",
      "Score Distribution:\n",
      "Entailment:     \n",
      "Contradiction:  ██████\n",
      "Neutral:        ██████████████████████████████████████████\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "=== FALSE CLAIMS ===\n"
     ]
    }
   ],
   "source": [
    "def color_score(score, threshold=0.3):\n",
    "    if score > threshold:\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "def analyze_text_comparison(document, claim, imager):\n",
    "    \"\"\"\n",
    "    Analyze and display text-based comparison with the full document and color-coded segments\n",
    "    Green: Entailment, Red: Contradiction, Pink: Neutral\n",
    "    \"\"\"\n",
    "    doc_segments = imager.split_text(document, granularity=\"sentence\")\n",
    "    claim_segments = imager.split_text(claim, granularity=\"sentence\")\n",
    "    image = imager.build_image(document, claim)\n",
    "    \n",
    "    print(\"\\n=== Analysis Results ===\")\n",
    "    print(f\"\\nCLAIM: {claim}\\n\")\n",
    "    \n",
    "    # Store scores and segments with their relationships\n",
    "    entail_scores = []\n",
    "    contra_scores = []\n",
    "    neutral_scores = []\n",
    "    segment_relationships = {}  # Store indices and their highest scoring relationship\n",
    "    \n",
    "    # First pass to identify relationships\n",
    "    for i, doc_seg in enumerate(doc_segments):\n",
    "        for j, claim_seg in enumerate(claim_segments):\n",
    "            try:\n",
    "                entail_score = float(image[0][i][j])\n",
    "                contra_score = float(image[1][i][j])\n",
    "                neutral_score = float(image[2][i][j])\n",
    "                \n",
    "                entail_scores.append(entail_score)\n",
    "                contra_scores.append(contra_score)\n",
    "                neutral_scores.append(neutral_score)\n",
    "                \n",
    "                # Determine the dominant relationship for this segment\n",
    "                max_score = max(entail_score, contra_score, neutral_score)\n",
    "                if max_score > 0.3:\n",
    "                    if max_score == entail_score:\n",
    "                        segment_relationships[i] = 'entailment'\n",
    "                    elif max_score == contra_score:\n",
    "                        segment_relationships[i] = 'contradiction'\n",
    "                    elif max_score == neutral_score:\n",
    "                        segment_relationships[i] = 'neutral'\n",
    "                    \n",
    "            except (IndexError, Exception):\n",
    "                continue\n",
    "    \n",
    "    # Print the full document with color-coded segments\n",
    "    print(\"DOCUMENT TEXT:\")\n",
    "    for i, segment in enumerate(doc_segments):\n",
    "        if i in segment_relationships:\n",
    "            relationship = segment_relationships[i]\n",
    "            if relationship == 'entailment':\n",
    "                print(f\"\\033[1;32m{segment}\\033[0m\")  # Green for entailment\n",
    "            elif relationship == 'contradiction':\n",
    "                print(f\"\\033[1;31m{segment}\\033[0m\")  # Red for contradiction\n",
    "            elif relationship == 'neutral':\n",
    "                print(f\"\\033[1;95m{segment}\\033[0m\")  # Pink for neutral\n",
    "        else:\n",
    "            print(segment)\n",
    "    \n",
    "    print(\"\\nDETAILED RELATIONSHIPS:\")\n",
    "    # Print detailed scores for segments with relationships\n",
    "    for i in segment_relationships:\n",
    "        for j, claim_seg in enumerate(claim_segments):\n",
    "            try:\n",
    "                entail_score = float(image[0][i][j])\n",
    "                contra_score = float(image[1][i][j])\n",
    "                neutral_score = float(image[2][i][j])\n",
    "                \n",
    "                max_score = max(entail_score, contra_score, neutral_score)\n",
    "                if max_score > 0.3:\n",
    "                    print(f\"\\nSegment: {doc_segments[i]}\")\n",
    "                    print(f\"Relationship: {segment_relationships[i].upper()}\")\n",
    "                    print(f\"Scores - E: {entail_score:.3f} | C: {contra_score:.3f} | N: {neutral_score:.3f}\")\n",
    "            except (IndexError, Exception):\n",
    "                continue\n",
    "    \n",
    "    # Create simple ASCII histogram\n",
    "    print(\"\\nScore Distribution:\")\n",
    "    print(\"Entailment:     \" + \"█\" * int(sum(entail_scores)/len(entail_scores) * 50))\n",
    "    print(\"Contradiction:  \" + \"█\" * int(sum(contra_scores)/len(contra_scores) * 50))\n",
    "    print(\"Neutral:        \" + \"█\" * int(sum(neutral_scores)/len(neutral_scores) * 50))\n",
    "    print(\"\\n\" + \"-\"*80 + \"\\n\")\n",
    "\n",
    "def analyze_sample_cases_text(df, imager, n_samples=2):\n",
    "    \"\"\"\n",
    "    Analyze sample cases with cleaner output\n",
    "    \"\"\"\n",
    "    true_cases = df[df['Actual Decision'] == 1]\n",
    "    false_cases = df[df['Actual Decision'] == 0]\n",
    "    \n",
    "    n_true_samples = min(n_samples, len(true_cases))\n",
    "    n_false_samples = min(n_samples, len(false_cases))\n",
    "    \n",
    "    print(\"\\n=== TRUE CLAIMS ===\")\n",
    "    for idx, row in true_cases.sample(n=n_true_samples).iterrows():\n",
    "        doc_content = str(row['Scraped Content']) if pd.notna(row['Scraped Content']) else \"\"\n",
    "        headline = str(row['Headline']) if pd.notna(row['Headline']) else \"\"\n",
    "        \n",
    "        if doc_content and headline:\n",
    "            analyze_text_comparison(doc_content, headline, imager)\n",
    "    \n",
    "    print(\"\\n=== FALSE CLAIMS ===\")\n",
    "    for idx, row in false_cases.sample(n=n_false_samples).iterrows():\n",
    "        doc_content = str(row['Scraped Content']) if pd.notna(row['Scraped Content']) else \"\"\n",
    "        headline = str(row['Headline']) if pd.notna(row['Headline']) else \"\"\n",
    "        \n",
    "        if doc_content and headline:\n",
    "            analyze_text_comparison(doc_content, headline, imager)\n",
    "print(\"\\nAnalyzing sample predictions...\")\n",
    "if len(correct_preds) > 0:\n",
    "    analyze_sample_cases_text(pd.DataFrame(correct_preds), imager, n_samples=2)\n",
    "if len(incorrect_preds) > 0:\n",
    "    analyze_sample_cases_text(pd.DataFrame(incorrect_preds), imager, n_samples=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}