Statistics_and_Graphs.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rmaacario/spatial-semantics-translation/blob/main/Statistics_and_Graphs.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#%load_ext cudf.pandas"
      ],
      "metadata": {
        "id": "ow_I8Avn7jtB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Tc2IVAFJ6Twe"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import scipy.stats as stats\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.patches import Polygon\n",
        "import scipy.stats as stats\n",
        "from matplotlib.colors import LinearSegmentedColormap\n",
        "import plotnine as p\n",
        "from plotnine import aes\n",
        "from IPython.display import display\n",
        "from plotnine import stat_summary, ggplot, aes, geom_tile, scale_fill_gradientn, labs\n",
        "from scipy.stats import mannwhitneyu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "13naoEda6Twh"
      },
      "outputs": [],
      "source": [
        "def compute_statistics(scores):\n",
        "    \"\"\"Compute statistics for a given series of scores.\"\"\"\n",
        "    numeric_scores = [float(score) for score in scores if isinstance(score, (int, float))]\n",
        "\n",
        "    median = np.median(numeric_scores)\n",
        "    q1, q3 = np.percentile(numeric_scores, [25, 75])\n",
        "    iqr = q3 - q1\n",
        "    whisker_low = max(np.min(numeric_scores), q1 - 1.5 * iqr)\n",
        "    whisker_high = min(np.max(numeric_scores), q3 + 1.5 * iqr)\n",
        "\n",
        "    filtered_scores = [score for score in numeric_scores if whisker_low <= score <= whisker_high]\n",
        "    outliers = [score for score in numeric_scores if score not in filtered_scores]\n",
        "\n",
        "    mean_score = np.mean(filtered_scores)\n",
        "    std_dev = np.std(filtered_scores)\n",
        "    ci = stats.t.interval(0.95, len(filtered_scores)-1, loc=mean_score, scale=stats.sem(filtered_scores))\n",
        "\n",
        "    return mean_score, median, q1, q3, whisker_low, whisker_high, outliers, std_dev, ci\n",
        "\n",
        "\n",
        "def analyze_scores(folder_path, score_labels, model_labels):\n",
        "    \"\"\"Analyze scores from CSV files in the given folder.\"\"\"\n",
        "    scores_dict = {score_name: [] for score_name in score_labels}\n",
        "\n",
        "    for filename in os.listdir(folder_path):\n",
        "        if filename.endswith('.csv'):\n",
        "            file_path = os.path.join(folder_path, filename)\n",
        "            df = pd.read_csv(file_path).drop(columns=['Unnamed: 0'], errors='ignore')\n",
        "\n",
        "            model_name = df.columns[3] if len(df.columns) > 3 else None\n",
        "            if not model_name:\n",
        "                print(f\"Skipping file {filename}: Model name not found.\")\n",
        "                continue\n",
        "\n",
        "            for score_name in score_labels:\n",
        "                if score_name not in df.columns:\n",
        "                    print(f\"Skipping score {score_name} in file {filename}: Score column not found.\")\n",
        "                    continue\n",
        "\n",
        "                scores = df[score_name].dropna().tolist()\n",
        "                statistics = compute_statistics(scores)\n",
        "                model_name_label = model_labels.get(model_name, model_name)\n",
        "                scores_dict[score_name].append((model_name_label, *statistics))\n",
        "\n",
        "    return scores_dict"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import ttest_ind\n",
        "\n",
        "def compare_groups(scores_dict, score_labels, model_labels, llms, nmt):\n",
        "    p_values = {}\n",
        "\n",
        "    # Iterate over each score type\n",
        "    for score_name, score_data in scores_dict.items():\n",
        "        # Create a DataFrame from the score data\n",
        "        df = pd.DataFrame(score_data, columns=['Model', 'Mean', 'Median', 'Q1', 'Q3',\n",
        "                                               'Whisker Low', 'Whisker High', 'Outliers',\n",
        "                                               'Standard Deviation', 'Confidence Interval'])\n",
        "        # Filter the DataFrame for LLMs and NMTs\n",
        "        llm_df = df[df['Model'].isin(llms)]\n",
        "        nmt_df = df[df['Model'].isin(nmt)]\n",
        "\n",
        "        # Perform t-test for each pair of models\n",
        "        p_values[score_name] = {}\n",
        "        for llm_model in llms:\n",
        "            for nmt_model in nmt:\n",
        "                # Filter data for the current pair of models\n",
        "                llm_data = llm_df[llm_df['Model'] == llm_model]['Mean'].tolist()\n",
        "                nmt_data = nmt_df[nmt_df['Model'] == nmt_model]['Mean'].tolist()\n",
        "\n",
        "                try:\n",
        "                    # Check variance and sample size\n",
        "                    if len(llm_data) > 1 and len(nmt_data) > 1:\n",
        "                        if max(llm_data) - min(llm_data) > 1e-5 and max(nmt_data) - min(nmt_data) > 1e-5:\n",
        "                            # Perform t-test\n",
        "                            _, p_value = ttest_ind(llm_data, nmt_data, equal_var=False)\n",
        "                            p_values[score_name][(llm_model, nmt_model)] = p_value\n",
        "                        else:\n",
        "                            p_values[score_name][(llm_model, nmt_model)] = None\n",
        "                    else:\n",
        "                        p_values[score_name][(llm_model, nmt_model)] = None\n",
        "                except Exception as e:\n",
        "                    print(f\"Error computing p-value for {score_name}: {e}\")\n",
        "                    p_values[score_name][(llm_model, nmt_model)] = None\n",
        "\n",
        "    return p_values\n"
      ],
      "metadata": {
        "id": "EuH_2gM132v-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import plotnine as p\n",
        "\n",
        "def plot_boxplots(scores_dict, score_labels, model_labels, max_outliers=3, max_confidence_intervals=3):\n",
        "    \"\"\"Plot boxplot for each score, with all metrics plotted in a single boxplot.\"\"\"\n",
        "    melted_dfs = []  # Initialize an empty list to store melted DataFrames\n",
        "    for score_name, score_data in scores_dict.items():\n",
        "        print(f\"Processing score: {score_name}\")\n",
        "        print(\"Score data:\")\n",
        "        print(score_data)\n",
        "\n",
        "        df = pd.DataFrame(score_data, columns=['Model', 'Mean', 'Median', 'Q1', 'Q3',\n",
        "                                               'Whisker Low', 'Whisker High', 'Outliers',\n",
        "                                               'Standard Deviation', 'Confidence Interval'])\n",
        "        print(\"DataFrame:\")\n",
        "        print(df)\n",
        "\n",
        "        # Melt the DataFrame to have all metrics in one column\n",
        "        melted_df = pd.melt(df, id_vars=['Model'], value_vars=['Mean', 'Median', 'Q1', 'Q3',\n",
        "                                                               'Whisker Low', 'Whisker High',\n",
        "                                                               'Standard Deviation'],\n",
        "                             var_name='Metric', value_name='Value')\n",
        "\n",
        "        print(\"Melted DataFrame:\")\n",
        "        print(melted_df)\n",
        "\n",
        "        melted_dfs.append(melted_df)  # Append the melted DataFrame to the list\n",
        "        # Define colors based on model titles\n",
        "        colors = {model: 'lightblue' if model in ['Gemma-7B', 'LLaMA-2-7B', 'LLaMA-2-13B', 'LLaMA-3-8B',  'Mistral-7B', 'Mixtral-8x7B'] else 'lightgray' for model in melted_df['Model'].unique()}\n",
        "\n",
        "        # Calculate the dynamic y-axis limits\n",
        "        y_min = melted_df['Value'].min()\n",
        "        y_max = melted_df['Value'].max()\n",
        "        y_padding = (y_max - y_min) * 0.1  # Add some padding to the range\n",
        "\n",
        "        plot = (\n",
        "          p.ggplot(melted_df, p.aes(x='Model', y='Value', fill='factor(Model)')) +\n",
        "          p.geom_boxplot(stat='boxplot', colour=\"black\", show_legend=False) +\n",
        "          p.stat_summary(geom=\"point\", shape='o', size=3, color=\"red\", fill=\"white\") +\n",
        "          p.labs(x=\"Model\", y=score_labels[score_name]) +\n",
        "          p.theme_gray() +\n",
        "          p.scale_fill_manual(values=colors) +\n",
        "          p.theme(axis_text_x=p.element_text(angle=45, vjust=1, hjust=-1, size=8)) +\n",
        "          p.ylim(y_min - y_padding, y_max + y_padding) +  # Set y-axis limits\n",
        "          p.guides(fill=p.guide_legend(title=\"Models:\", override_aes={'lightblue': 'LLMs', 'lightgray': 'NMT'}))\n",
        "        )\n",
        "\n",
        "        display(plot)\n",
        "\n",
        "    if not melted_dfs:\n",
        "        print(\"No melted DataFrames found. Check input data.\")\n",
        "\n",
        "    # Concatenate all melted DataFrames into a single DataFrame\n",
        "    final_df = pd.concat(melted_dfs, ignore_index=True)\n",
        "    return final_df"
      ],
      "metadata": {
        "id": "5lU_Rq1O8NmV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CaoyEGgM6Twq"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from plotnine import *\n",
        "\n",
        "def plot_heatmap(p_values, llm_models, nmt_models, metric):\n",
        "    data = np.full((len(llm_models), len(nmt_models)), np.nan)\n",
        "\n",
        "    for i, llm_model in enumerate(llm_models):\n",
        "        for j, nmt_model in enumerate(nmt_models):\n",
        "            data[i, j] = p_values.get((llm_model, nmt_model), np.nan)\n",
        "\n",
        "    df = pd.DataFrame(data, index=llm_models, columns=nmt_models)\n",
        "\n",
        "    # Define a custom color map\n",
        "    custom_cmap = [\"blue\", \"green\", \"yellow\", \"orange\", \"red\"]\n",
        "\n",
        "    # Plot the heatmap using ggplot\n",
        "    plot = (\n",
        "        ggplot(df.melt(ignore_index=False).reset_index(), aes(x=\"variable\", y=\"index\", fill=\"value\")) +\n",
        "        geom_tile() +\n",
        "        scale_fill_gradientn(colors=custom_cmap, limits=(0, 1), breaks=np.linspace(0, 1, len(custom_cmap)),\n",
        "                             labels=[\"0\", \"0.25\", \"0.5\", \"0.75\", \"1\"], na_value='gray') +\n",
        "        labs(title=f\"P-values for {metric}\", x=\"NMT Models\", y=\"LLM Models\")\n",
        "    )\n",
        "    print(plot)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def plot_heatmap(p_values, llm_models, nmt_models, metric):\n",
        "    data = np.full((len(llm_models), len(nmt_models)), np.nan)\n",
        "\n",
        "    for i, llm_model in enumerate(llm_models):\n",
        "        for j, nmt_model in enumerate(nmt_models):\n",
        "            data[i, j] = p_values.get((llm_model, nmt_model), np.nan)\n",
        "\n",
        "    df = pd.DataFrame(data, index=llm_models, columns=nmt_models)\n",
        "\n",
        "    # Define a custom color map\n",
        "    custom_cmap = [\"blue\", \"green\", \"yellow\", \"orange\", \"red\"]\n",
        "\n",
        "    # Plot the heatmap using seaborn\n",
        "    plt.figure(figsize=(10, 8))\n",
        "    sns.heatmap(df, cmap=custom_cmap, vmin=0, vmax=1, annot=True, fmt=\".2f\", linewidths=.5)\n",
        "    plt.title(f\"P-values for {metric}\")\n",
        "    plt.xlabel(\"NMT Models\")\n",
        "    plt.ylabel(\"LLM Models\")\n",
        "    plt.show()"
      ],
      "metadata": {
        "id": "CBMyleng-R6A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "scrolled": false,
        "id": "dn--iEEa6Twv"
      },
      "outputs": [],
      "source": [
        "def main():\n",
        "    folder_path = '/content/'\n",
        "    score_labels = {\n",
        "        'bleu_score': 'BLEU',\n",
        "        'meteor_score': 'METEOR',\n",
        "        'bert_score': 'BERTScore',\n",
        "        'comet_score': 'COMET',\n",
        "        'ter_score': 'TER'\n",
        "    }\n",
        "    model_labels = {\n",
        "\n",
        "        'gemma:7b': 'Gemma-7B',\n",
        "        'llama2:7b': 'LLaMA-2-7B',\n",
        "        'llama2:13b': 'LLaMA-2-13B',\n",
        "        'llama3': 'LLaMA-3-8B',\n",
        "        'deepl': 'DeepL',\n",
        "        'mistral': 'Mistral-7B',\n",
        "        'amazon-stock': 'Amazon (Stock)',\n",
        "        'amazon-custom': 'Amazon (Custom)',\n",
        "        'googletrans': 'Google',\n",
        "        'mixtral': 'Mixtral-8x7B'\n",
        "    }\n",
        "\n",
        "    llms = ['Gemma-7B', 'LLaMA-2-7B', 'LLaMA-2-13B', 'LLaMA-3-8B', 'Mistral-7B', 'Mixtral-8x7B']\n",
        "    nmt = ['DeepL', 'Amazon (Stock)', 'Amazon (Custom)', 'Google']\n",
        "\n",
        "\n",
        "    scores_dict = analyze_scores(folder_path, score_labels, model_labels)\n",
        "\n",
        "    plot_boxplots(scores_dict, score_labels, model_labels)\n",
        "\n",
        "    #plot_means_confidence_intervals(scores_dict, score_labels)\n",
        "\n",
        "\n",
        "    #for score_label, score_name in score_labels.items():\n",
        "    #    p_values = compare_groups(scores_dict, score_labels, model_labels, llms, nmt)\n",
        "    #    print(f\"P-values for {score_name}:\")\n",
        "    #    print(p_values)\n",
        "    #    plot_heatmap(p_values, llms, nmt, score_name)\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scores_dict"
      ],
      "metadata": {
        "id": "atGSjxQ5-r0D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pd.read_csv(\"/content/TEDTalks.en_pt-br.llama3.csv\")"
      ],
      "metadata": {
        "id": "yeJWMbWiLd1i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mWgipmtC6Twz"
      },
      "outputs": [],
      "source": [
        "spatial_df = df['spatial_sense'].value_counts().reset_index()\n",
        "spatial_df.columns = ['spatial_sense', 'quantity']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_6tBsEkt6Tw0"
      },
      "outputs": [],
      "source": [
        "spatial_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "46MslaqM6Tw1"
      },
      "outputs": [],
      "source": [
        "across = df[df['spatial_sense'] == 'Into(5)']\n",
        "across"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CWbQ0ZJA6Tw1"
      },
      "outputs": [],
      "source": [
        "#across['source'][292]\n",
        "#df.loc[df['inner_id'] == 98716, 'spatial_sense']  = 'Into(1)'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6S0dsnNS6Tw4"
      },
      "outputs": [],
      "source": [
        "#df.to_csv('NMT ANALYSIS/TEDTalks.en_pt-br.DeepL.ANALYZED.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0HLvGmLM6Tw6"
      },
      "outputs": [],
      "source": [
        "# Define legend mapping\n",
        "legend_mapping = {\n",
        "    'Across': '(1) Perpendicular position; (2) Movement over a surface; (3) Opposite location; (4) Distribution; (5) Non-spatial',\n",
        "    'Through': '(1) Movement within a passage; (2) Movement within an open area; (3) Movement penetrating a barrier; (4) Part of a route; (5) Non-spatial',\n",
        "    'Into': '(1) Movement/direction leading to enclosure; (2) Movement resulting in physical contact/collision; (3) Non-spatial',\n",
        "    'Onto': '(1) Movement to a location on a surface; (2) Sense of attachment; (3) Non-spatial'\n",
        "}\n",
        "\n",
        "# Create figure\n",
        "fig = go.Figure(data=[go.Bar(x=spatial_df['spatial_sense'], y=spatial_df['quantity'])])\n",
        "\n",
        "# Define custom legend items as annotations with smaller font size and white background\n",
        "legend_annotations = [\n",
        "    dict(x=1.1, y=1-i*0.1, xref=\"paper\", yref=\"paper\",\n",
        "         text=f\"{category}: {description}\", showarrow=False,\n",
        "         font=dict(size=10))\n",
        "    for i, (category, description) in enumerate(legend_mapping.items())\n",
        "]\n",
        "\n",
        "# Add legend annotations to the layout\n",
        "fig.update_layout(\n",
        "    title='Histogram of Spatial Sense Quantity',\n",
        "    xaxis_title='Meaning',\n",
        "    yaxis_title='Quantity',\n",
        "    annotations=legend_annotations,\n",
        "    legend_tracegroupgap=50,  # Adjust spacing between legend items\n",
        ")\n",
        "\n",
        "# Add a white rectangle behind the legend annotations\n",
        "fig.add_shape(\n",
        "    type=\"rect\",\n",
        "    xref=\"paper\",\n",
        "    yref=\"paper\",\n",
        "    x0=1.05,\n",
        "    y0=1,\n",
        "    x1=1.25,\n",
        "    y1=0.5,\n",
        "    fillcolor=\"white\",\n",
        "    layer=\"below\",\n",
        "    opacity=1,\n",
        "    line_width=0\n",
        ")\n",
        "\n",
        "# Show the plot\n",
        "fig.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dbhKRiV86Tw7"
      },
      "outputs": [],
      "source": [
        "total_spatial"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "t5HaU6oQ6Tw_"
      },
      "outputs": [],
      "source": [
        "total_non_spatial"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TuCKuhjC6TxA"
      },
      "outputs": [],
      "source": [
        "# Define the classification function\n",
        "def classify_sense(sense):\n",
        "    if sense.endswith('(5)') or sense == 'Into(3)' or sense == 'Onto(3)':\n",
        "        return 'Non-Spatial'\n",
        "    else:\n",
        "        return 'Spatial'\n",
        "\n",
        "# Apply the classification function to the 'spatial_sense' column\n",
        "spatial_df['sense_type'] = spatial_df['spatial_sense'].apply(classify_sense)\n",
        "\n",
        "# Sort DataFrame by quantity in descending order\n",
        "spatial_df = spatial_df.sort_values(by='quantity', ascending=False)\n",
        "\n",
        "# Separate spatial and non-spatial senses\n",
        "spatial_senses = spatial_df[spatial_df['sense_type'] == 'Spatial']\n",
        "non_spatial_senses = spatial_df[spatial_df['sense_type'] == 'Non-Spatial']\n",
        "\n",
        "# Create traces for spatial and non-spatial senses with assigned colors\n",
        "trace_spatial = go.Bar(x=spatial_senses['spatial_sense'],\n",
        "                       y=spatial_senses['quantity'], name='Spatial', marker=dict(color='darkblue'))\n",
        "trace_non_spatial = go.Bar(x=non_spatial_senses['spatial_sense'],\n",
        "                           y=non_spatial_senses['quantity'], name='Non-Spatial', marker=dict(color='royalblue'))\n",
        "\n",
        "# Create figure\n",
        "fig = go.Figure(data=[trace_spatial, trace_non_spatial])\n",
        "\n",
        "# Update layout with legend\n",
        "fig.update_layout(\n",
        "    title='Spatial vs Non-Spatial Meaning Frequency',\n",
        "    xaxis_title='Meanings',\n",
        "    yaxis_title='Frequency',\n",
        "    legend_title='Meaning:',\n",
        "    legend_traceorder='reversed',\n",
        "    legend_tracegroupgap=50,  # Adjust spacing between legend items\n",
        ")\n",
        "\n",
        "# Show the plot\n",
        "fig.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q01e4L856TxC"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jr-iyiYW6TxD"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7yHA40c56TxE"
      },
      "outputs": [],
      "source": [
        "scores_dict"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "euoB4EWp6TxF"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "91k-idOx6TxF"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1wZvebXN6TxF"
      },
      "outputs": [],
      "source": [
        "scores_dict = analyze_scores(folder_path, score_labels, model_labels)\n",
        "p_values = pairwise_ttest(scores_dict, 'BLEU')  # or 'METEOR'\n",
        "print(p_values)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7ZcDqmdF6TxG"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.5"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}