visualizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "vU3vDD5NnHbe",
        "B6j9HEUktPbU"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#Package Setup"
      ],
      "metadata": {
        "id": "L6a-YUzXtKMi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Installations"
      ],
      "metadata": {
        "id": "vU3vDD5NnHbe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install umap-learn\n",
        "!pip install sentence-transformers\n",
        "!pip install pyvis"
      ],
      "metadata": {
        "id": "ld6DUOwPnOx4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Imports"
      ],
      "metadata": {
        "id": "GWU8thTgtDwK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import csv\n",
        "import umap\n",
        "from scipy import spatial\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sentence_transformers import SentenceTransformer\n",
        "import matplotlib.pyplot as plt\n",
        "from collections import defaultdict\n",
        "import pyvis\n",
        "from pyvis.network import Network\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import branca.colormap as cm\n",
        "import branca\n",
        "import pandas as pd\n",
        "import re\n",
        "from textwrap import wrap\n",
        "import json\n",
        "\n",
        "project_path = 'insert your path here\""
      ],
      "metadata": {
        "id": "iqovyEuCkmEk"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Visualization Creation Process"
      ],
      "metadata": {
        "id": "B6j9HEUktPbU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create Embeddings from Raw Data"
      ],
      "metadata": {
        "id": "5Fu_9ijYFeR8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Read attendees and their responses from a CSV file, replace attendees.csv with own link or file name\n",
        "attendees_map = {}\n",
        "with open(project_path+ 'WAT.ai Project Summary - Sheet1.csv', newline='') as csvfile:\n",
        "    attendees = csv.reader(csvfile, delimiter=',', quotechar='\"')\n",
        "    next(attendees)  # Skip the header row\n",
        "    for row in attendees:\n",
        "        name, paragraph = row\n",
        "        attendees_map[paragraph] = name\n",
        "\n",
        "# Generate sentence embeddings\n",
        "model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')\n",
        "paragraphs = list(attendees_map.keys())\n",
        "embeddings = model.encode(paragraphs)\n",
        "\n",
        "# Create a dictionary to store embeddings for each person\n",
        "person_embeddings = {attendees_map[paragraph]: embedding for paragraph, embedding in zip(paragraphs, embeddings)}\n"
      ],
      "metadata": {
        "id": "A20046f5kvzd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reducing dimensionality of embedding data, scaling to coordinate domain/range\n"
      ],
      "metadata": {
        "id": "yhQYxCZ-Fkmp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Reducing dimensionality of embedding data, scaling to coordinate domain/range\n",
        "reducer = umap.UMAP()\n",
        "scaler = StandardScaler()\n",
        "scaled_data = scaler.fit_transform(list(person_embeddings.values()))\n",
        "reduced_data = reducer.fit_transform(scaled_data)\n"
      ],
      "metadata": {
        "id": "PLufwJPWkoWp"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create Visualization Image"
      ],
      "metadata": {
        "id": "oD5e7ge2Fn3e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating lists of coordinates with accompanying labels\n",
        "x = [row[0] for row in reduced_data]\n",
        "y = [row[1] for row in reduced_data]\n",
        "label = list(person_embeddings.keys())\n",
        "\n",
        "# Plotting and annotating data points\n",
        "plt.scatter(x,y)\n",
        "for i, name in enumerate(label):\n",
        "    plt.annotate(name, (x[i], y[i]), fontsize=\"3\")\n",
        "\n",
        "# Clean-up and Export\n",
        "plt.axis('off')\n",
        "plt.savefig(project_path+'a_visualization.png', dpi=800)"
      ],
      "metadata": {
        "id": "aFhtKXwvktLO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Find the  top N matches to a node"
      ],
      "metadata": {
        "id": "RDfz17paFvR3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Providing top matches\n",
        "top_matches = {}\n",
        "all_personal_pairs = defaultdict(list)\n",
        "for person in attendees_map.values():\n",
        "    for person1 in attendees_map.values():\n",
        "        all_personal_pairs[person].append([spatial.distance.cosine(person_embeddings[person1], person_embeddings[person]), person1])\n",
        "\n",
        "n = 5\n",
        "# Collect the top n most similarity nodes\n",
        "data_day_list = []\n",
        "for person in attendees_map.values():\n",
        "    top_matches[person] = sorted(all_personal_pairs[person], key=lambda x: x[0])[1:n+1] # drop yourself, take top 5\n",
        "    a = sorted(all_personal_pairs[person], key=lambda x: x[0])[1:n+1]\n",
        "    mini_df = pd.DataFrame(a, columns=['Weight', 'Target'])\n",
        "    mini_df['Source'] = person\n",
        "    data_day_list.append(mini_df)\n",
        "\n",
        "# output this information as a json\n",
        "with open(project_path + 'b_top5_matches.json', 'w') as json_file:\n",
        "    json.dump(top_matches, json_file)\n",
        "\n",
        "# Output this information as a csv\n",
        "df = pd.concat(data_day_list)\n",
        "df.to_csv(project_path + 'b_top5_matches.csv')\n"
      ],
      "metadata": {
        "id": "DkzlqdbFrVCt"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Add Colour/Paragraph Information to Dataframe"
      ],
      "metadata": {
        "id": "-GMH_-ioGB-T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the colour pallette\n",
        "colour = sns.color_palette(\"pastel\",len(x)).as_hex()\n",
        "\n",
        "# Add colour pallette to the df\n",
        "df1 = pd.DataFrame([label,colour])\n",
        "df1 = df1.T\n",
        "df1.rename(columns={0: 'Source', 1: 'Colour'},inplace=True)\n",
        "df = df.set_index('Source').join(df1.set_index('Source'))\n",
        "df['Source'] = df.index\n",
        "df = df.reset_index(drop=True)\n",
        "\n",
        "# Add colour pallette for both the df Target and Source:\n",
        "df1.rename(columns={'Source': 'Target'},inplace=True)\n",
        "df = df.set_index('Target').join(df1.set_index('Target'),lsuffix='_Source', rsuffix='_Target')\n",
        "df['Target'] = df.index\n",
        "df = df.reset_index(drop=True)\n",
        "print(df)\n",
        "\n",
        "# Add paragraphs to the df\n",
        "df2 = pd.DataFrame([label,paragraphs])\n",
        "df2 = df2.T\n",
        "df2.rename(columns={0: 'Source', 1: 'Paragraphs'},inplace=True)\n",
        "df = df.set_index('Source').join(df2.set_index('Source'))\n",
        "df['Source'] = df.index\n",
        "df = df.reset_index(drop=True)\n",
        "print(df)\n",
        "\n",
        "# Create a cleaned Dataframe of just the Source and and Paragraph information\n",
        "df_new = df[[\"Source\",\"Paragraphs\"]]\n",
        "df_new = df_new.drop_duplicates()\n",
        "df_new.set_index('Source', inplace=True)"
      ],
      "metadata": {
        "id": "f423QJS90vPj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Build Interative Network Visualization (Simple)"
      ],
      "metadata": {
        "id": "_hIev_UiGLei"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Intitalize bucket size and colour palettes\n",
        "buckets = [100] * len(x)\n",
        "colour = sns.color_palette(\"pastel\",len(x)).as_hex()\n",
        "\n",
        "# Initialize network\n",
        "g = Network(height=\"750px\", width=\"100%\", bgcolor=\"#222222\", font_color=\"white\")\n",
        "\n",
        "# Add unconnected nodes to the network\n",
        "g.add_nodes(list(range(1,len(x)+1)), value=buckets,\n",
        "                         title=paragraphs,\n",
        "                         x=np.array(x).astype(np.float64),\n",
        "                         y=np.array(y).astype(np.float64),\n",
        "                         label=label,\n",
        "                         color=colour)\n",
        "\n",
        "# Output the visualization\n",
        "g.toggle_physics(True)\n",
        "g.show(project_path+'c_simple_viz.html', notebook=False)"
      ],
      "metadata": {
        "id": "4uNRCq8srzXg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Build Interative Network Visualization (Complex)\n"
      ],
      "metadata": {
        "id": "k25-lsF-GQMP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize network\n",
        "got_net = Network(height=\"750px\", width=\"100%\", bgcolor=\"#222222\", font_color=\"white\",select_menu=True,cdn_resources='remote')\n",
        "\n",
        "# Create a dictionary of Important information\n",
        "sources = df['Source']\n",
        "targets = df['Target']\n",
        "weights = df['Weight']\n",
        "color_targets = df['Colour_Target']\n",
        "color_sources = df['Colour_Source']\n",
        "\n",
        "edge_data = zip(sources, targets, weights,color_targets,color_sources)\n",
        "\n",
        "# Add nodes and edges to the network\n",
        "for e in edge_data:\n",
        "                src = e[0]\n",
        "                dst = e[1]\n",
        "                w = e[2]\n",
        "                c_t= e[3]\n",
        "                c_s= e[4]\n",
        "                got_net.add_node(src, src, title=src,color=c_s)\n",
        "                got_net.add_node(dst, dst, title=dst)\n",
        "                got_net.add_edge(src, dst, value=w)#,color = \"#c79910\") # if you  want a solide colour for edges\n",
        "\n",
        "# Add paragraphs to the popup\n",
        "for i,node in enumerate(got_net.nodes):\n",
        "               content =df_new.loc[node.get(\"title\"),\"Paragraphs\"]\n",
        "               node[\"title\"] += \": \"+ \"\\n \\n\" +'\\n'.join(wrap(content, width=50))\n",
        "\n",
        "## Output the visualization\n",
        "got_net.show(project_path+'c_complex_viz.html', notebook=False)"
      ],
      "metadata": {
        "id": "gN4tOTrjyjGT"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}