Yelp_Data_EDA.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YXAL6gpkijkz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03f7ddf8-3b94-44d2-d97a-585f6fdafc8e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 24.2 MB 797 kB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 15.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 4.4 MB 14.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 596 kB 85.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 101 kB 13.9 MB/s \n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 57.9 MB/s \n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install gensim==3.8.3 --quiet\n",
        "!pip install pydot --quiet\n",
        "!pip install sentencepiece --quiet\n",
        "!pip install transformers --quiet"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.cloud import storage\n",
        "import google.oauth2.credentials\n",
        "import json\n",
        "import seaborn as sns\n",
        "\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "from tensorflow.keras.layers import Embedding, Input, Dense, Lambda\n",
        "from tensorflow.keras.models import Model\n",
        "import tensorflow.keras.backend as K\n",
        "import tensorflow_datasets as tfds\n",
        "\n",
        "import sklearn as sk\n",
        "import nltk\n",
        "from nltk.corpus import reuters\n",
        "from nltk.data import find\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import re\n",
        "\n",
        "#This continues to work with gensim 3.8.3.  It doesn't yet work with 4.x.  \n",
        "#Make sure your pip install command specifies gensim==3.8.3\n",
        "import gensim\n"
      ],
      "metadata": {
        "id": "Kd0xo5RHVbg-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load Yelp data\n",
        "Get both the reviews data set and the corresponding business data set"
      ],
      "metadata": {
        "id": "UaJca2JcFTH3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Access to google drive where I have downloded the data\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Umy49AUj-QH",
        "outputId": "b0079c21-6c73-4728-c8b1-e01088e1f8de"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "data_file = open(\"/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_academic_dataset_business.json\")\n",
        "data = []\n",
        "for line in data_file:\n",
        "  data.append(json.loads(line))\n",
        "business_df = pd.DataFrame(data)\n",
        "data_file.close()"
      ],
      "metadata": {
        "id": "I-Eth0DBZQGj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tDQAHQNEznJD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 600
        },
        "outputId": "83582fca-f1d3-4906-efa7-2845763e4db0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "              business_id                      name  \\\n",
              "0  Pns2l4eNsfO8kk83dixA6A  Abby Rappoport, LAC, CMQ   \n",
              "1  mpf3x-BjTdTEA3yCZrAYPw             The UPS Store   \n",
              "2  tUFrWirKiKi_TAnsVWINQQ                    Target   \n",
              "3  MTSW4McQd7CbVtyjqoe9mw        St Honore Pastries   \n",
              "4  mWMc6_wTdE0EUBKIGXDVfA  Perkiomen Valley Brewery   \n",
              "\n",
              "                           address           city state postal_code  \\\n",
              "0           1616 Chapala St, Ste 2  Santa Barbara    CA       93101   \n",
              "1  87 Grasso Plaza Shopping Center         Affton    MO       63123   \n",
              "2             5255 E Broadway Blvd         Tucson    AZ       85711   \n",
              "3                      935 Race St   Philadelphia    PA       19107   \n",
              "4                    101 Walnut St     Green Lane    PA       18054   \n",
              "\n",
              "    latitude   longitude  stars  review_count  is_open  \\\n",
              "0  34.426679 -119.711197    5.0             7        0   \n",
              "1  38.551126  -90.335695    3.0            15        1   \n",
              "2  32.223236 -110.880452    3.5            22        0   \n",
              "3  39.955505  -75.155564    4.0            80        1   \n",
              "4  40.338183  -75.471659    4.5            13        1   \n",
              "\n",
              "                                          attributes  \\\n",
              "0                      {'ByAppointmentOnly': 'True'}   \n",
              "1             {'BusinessAcceptsCreditCards': 'True'}   \n",
              "2  {'BikeParking': 'True', 'BusinessAcceptsCredit...   \n",
              "3  {'RestaurantsDelivery': 'False', 'OutdoorSeati...   \n",
              "4  {'BusinessAcceptsCreditCards': 'True', 'Wheelc...   \n",
              "\n",
              "                                          categories  \\\n",
              "0  Doctors, Traditional Chinese Medicine, Naturop...   \n",
              "1  Shipping Centers, Local Services, Notaries, Ma...   \n",
              "2  Department Stores, Shopping, Fashion, Home & G...   \n",
              "3  Restaurants, Food, Bubble Tea, Coffee & Tea, B...   \n",
              "4                          Brewpubs, Breweries, Food   \n",
              "\n",
              "                                               hours  \n",
              "0                                               None  \n",
              "1  {'Monday': '0:0-0:0', 'Tuesday': '8:0-18:30', ...  \n",
              "2  {'Monday': '8:0-22:0', 'Tuesday': '8:0-22:0', ...  \n",
              "3  {'Monday': '7:0-20:0', 'Tuesday': '7:0-20:0', ...  \n",
              "4  {'Wednesday': '14:0-22:0', 'Thursday': '16:0-2...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d1fe8f0d-9f6b-4220-a411-bd493d69fe8d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>business_id</th>\n",
              "      <th>name</th>\n",
              "      <th>address</th>\n",
              "      <th>city</th>\n",
              "      <th>state</th>\n",
              "      <th>postal_code</th>\n",
              "      <th>latitude</th>\n",
              "      <th>longitude</th>\n",
              "      <th>stars</th>\n",
              "      <th>review_count</th>\n",
              "      <th>is_open</th>\n",
              "      <th>attributes</th>\n",
              "      <th>categories</th>\n",
              "      <th>hours</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Pns2l4eNsfO8kk83dixA6A</td>\n",
              "      <td>Abby Rappoport, LAC, CMQ</td>\n",
              "      <td>1616 Chapala St, Ste 2</td>\n",
              "      <td>Santa Barbara</td>\n",
              "      <td>CA</td>\n",
              "      <td>93101</td>\n",
              "      <td>34.426679</td>\n",
              "      <td>-119.711197</td>\n",
              "      <td>5.0</td>\n",
              "      <td>7</td>\n",
              "      <td>0</td>\n",
              "      <td>{'ByAppointmentOnly': 'True'}</td>\n",
              "      <td>Doctors, Traditional Chinese Medicine, Naturop...</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>mpf3x-BjTdTEA3yCZrAYPw</td>\n",
              "      <td>The UPS Store</td>\n",
              "      <td>87 Grasso Plaza Shopping Center</td>\n",
              "      <td>Affton</td>\n",
              "      <td>MO</td>\n",
              "      <td>63123</td>\n",
              "      <td>38.551126</td>\n",
              "      <td>-90.335695</td>\n",
              "      <td>3.0</td>\n",
              "      <td>15</td>\n",
              "      <td>1</td>\n",
              "      <td>{'BusinessAcceptsCreditCards': 'True'}</td>\n",
              "      <td>Shipping Centers, Local Services, Notaries, Ma...</td>\n",
              "      <td>{'Monday': '0:0-0:0', 'Tuesday': '8:0-18:30', ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>tUFrWirKiKi_TAnsVWINQQ</td>\n",
              "      <td>Target</td>\n",
              "      <td>5255 E Broadway Blvd</td>\n",
              "      <td>Tucson</td>\n",
              "      <td>AZ</td>\n",
              "      <td>85711</td>\n",
              "      <td>32.223236</td>\n",
              "      <td>-110.880452</td>\n",
              "      <td>3.5</td>\n",
              "      <td>22</td>\n",
              "      <td>0</td>\n",
              "      <td>{'BikeParking': 'True', 'BusinessAcceptsCredit...</td>\n",
              "      <td>Department Stores, Shopping, Fashion, Home &amp; G...</td>\n",
              "      <td>{'Monday': '8:0-22:0', 'Tuesday': '8:0-22:0', ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>MTSW4McQd7CbVtyjqoe9mw</td>\n",
              "      <td>St Honore Pastries</td>\n",
              "      <td>935 Race St</td>\n",
              "      <td>Philadelphia</td>\n",
              "      <td>PA</td>\n",
              "      <td>19107</td>\n",
              "      <td>39.955505</td>\n",
              "      <td>-75.155564</td>\n",
              "      <td>4.0</td>\n",
              "      <td>80</td>\n",
              "      <td>1</td>\n",
              "      <td>{'RestaurantsDelivery': 'False', 'OutdoorSeati...</td>\n",
              "      <td>Restaurants, Food, Bubble Tea, Coffee &amp; Tea, B...</td>\n",
              "      <td>{'Monday': '7:0-20:0', 'Tuesday': '7:0-20:0', ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>mWMc6_wTdE0EUBKIGXDVfA</td>\n",
              "      <td>Perkiomen Valley Brewery</td>\n",
              "      <td>101 Walnut St</td>\n",
              "      <td>Green Lane</td>\n",
              "      <td>PA</td>\n",
              "      <td>18054</td>\n",
              "      <td>40.338183</td>\n",
              "      <td>-75.471659</td>\n",
              "      <td>4.5</td>\n",
              "      <td>13</td>\n",
              "      <td>1</td>\n",
              "      <td>{'BusinessAcceptsCreditCards': 'True', 'Wheelc...</td>\n",
              "      <td>Brewpubs, Breweries, Food</td>\n",
              "      <td>{'Wednesday': '14:0-22:0', 'Thursday': '16:0-2...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d1fe8f0d-9f6b-4220-a411-bd493d69fe8d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d1fe8f0d-9f6b-4220-a411-bd493d69fe8d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d1fe8f0d-9f6b-4220-a411-bd493d69fe8d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "business_df = business_df.convert_dtypes()\n",
        "business_df[0:5]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_file = open(\"/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_academic_dataset_review.json\")\n",
        "data = []\n",
        "for line in data_file:\n",
        "  data.append(json.loads(line))\n",
        "review_df = pd.DataFrame(data)\n",
        "data_file.close()"
      ],
      "metadata": {
        "id": "ToTT6hytkhAL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "review_df = review_df.convert_dtypes()\n",
        "review_df[0:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 721
        },
        "id": "K7h60gNclF6f",
        "outputId": "67c04b42-d25c-4e95-e265-231261066408"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                review_id                 user_id             business_id  \\\n",
              "0  KU_O5udG6zpxOg-VcAEodg  mh_-eMZ6K5RLWhZyISBhwA  XQfwVwDr-v0ZS3_CbbE5Xw   \n",
              "1  BiTunyQ73aT9WBnpR9DZGw  OyoGAe7OKpv6SyGZT5g77Q  7ATYjTIgM3jUlt4UM3IypQ   \n",
              "2  saUsX_uimxRlCVr67Z4Jig  8g_iMtfSiwikVnbP2etR0A  YjUWPpI6HXG530lwP-fb2A   \n",
              "3  AqPFMleE6RsU23_auESxiA  _7bHUi9Uuf5__HHc_Q8guQ  kxX2SOes4o-D3ZQBkiMRfA   \n",
              "4  Sx8TMOWLNuJBWer-0pcmoA  bcjbaE6dDog4jkNY91ncLQ  e4Vwtrqf-wpJfwesgvdgxQ   \n",
              "\n",
              "   stars  useful  funny  cool  \\\n",
              "0      3       0      0     0   \n",
              "1      5       1      0     1   \n",
              "2      3       0      0     0   \n",
              "3      5       1      0     1   \n",
              "4      4       1      0     1   \n",
              "\n",
              "                                                text                 date  \n",
              "0  If you decide to eat here, just be aware it is...  2018-07-07 22:09:11  \n",
              "1  I've taken a lot of spin classes over the year...  2012-01-03 15:28:18  \n",
              "2  Family diner. Had the buffet. Eclectic assortm...  2014-02-05 20:30:30  \n",
              "3  Wow!  Yummy, different,  delicious.   Our favo...  2015-01-04 00:01:03  \n",
              "4  Cute interior and owner (?) gave us tour of up...  2017-01-14 20:54:15  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e7272698-2b24-4ac1-bc02-3a1af1b091ee\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review_id</th>\n",
              "      <th>user_id</th>\n",
              "      <th>business_id</th>\n",
              "      <th>stars</th>\n",
              "      <th>useful</th>\n",
              "      <th>funny</th>\n",
              "      <th>cool</th>\n",
              "      <th>text</th>\n",
              "      <th>date</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>KU_O5udG6zpxOg-VcAEodg</td>\n",
              "      <td>mh_-eMZ6K5RLWhZyISBhwA</td>\n",
              "      <td>XQfwVwDr-v0ZS3_CbbE5Xw</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>If you decide to eat here, just be aware it is...</td>\n",
              "      <td>2018-07-07 22:09:11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>BiTunyQ73aT9WBnpR9DZGw</td>\n",
              "      <td>OyoGAe7OKpv6SyGZT5g77Q</td>\n",
              "      <td>7ATYjTIgM3jUlt4UM3IypQ</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>I've taken a lot of spin classes over the year...</td>\n",
              "      <td>2012-01-03 15:28:18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>saUsX_uimxRlCVr67Z4Jig</td>\n",
              "      <td>8g_iMtfSiwikVnbP2etR0A</td>\n",
              "      <td>YjUWPpI6HXG530lwP-fb2A</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Family diner. Had the buffet. Eclectic assortm...</td>\n",
              "      <td>2014-02-05 20:30:30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AqPFMleE6RsU23_auESxiA</td>\n",
              "      <td>_7bHUi9Uuf5__HHc_Q8guQ</td>\n",
              "      <td>kxX2SOes4o-D3ZQBkiMRfA</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>Wow!  Yummy, different,  delicious.   Our favo...</td>\n",
              "      <td>2015-01-04 00:01:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Sx8TMOWLNuJBWer-0pcmoA</td>\n",
              "      <td>bcjbaE6dDog4jkNY91ncLQ</td>\n",
              "      <td>e4Vwtrqf-wpJfwesgvdgxQ</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>Cute interior and owner (?) gave us tour of up...</td>\n",
              "      <td>2017-01-14 20:54:15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e7272698-2b24-4ac1-bc02-3a1af1b091ee')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e7272698-2b24-4ac1-bc02-3a1af1b091ee button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e7272698-2b24-4ac1-bc02-3a1af1b091ee');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# join data sets together\n",
        "review_df = review_df.join(business_df.set_index('business_id'), how = \"left\", lsuffix='_review', on = 'business_id')"
      ],
      "metadata": {
        "id": "Mm0KGOmolWZj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get only reviews for businesses categorized as a 'Restaurant'\n",
        "review_df = review_df[(review_df.categories.str.contains('Restaurants'))]"
      ],
      "metadata": {
        "id": "FQ6tainFQVCf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "review_df.describe()"
      ],
      "metadata": {
        "id": "_AgLIXBVUU88",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 364
        },
        "outputId": "de86cdfa-1b94-4afd-c9ca-11d83185d5dc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       stars_review        useful         funny          cool      latitude  \\\n",
              "count  4.724471e+06  4.724471e+06  4.724471e+06  4.724471e+06  4.724471e+06   \n",
              "mean   3.793798e+00  9.840992e-01  3.013692e-01  4.795771e-01  3.587026e+01   \n",
              "std    1.391227e+00  2.739150e+00  1.717190e+00  2.159917e+00  5.337905e+00   \n",
              "min    1.000000e+00 -1.000000e+00 -1.000000e+00 -1.000000e+00  2.756446e+01   \n",
              "25%    3.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00  2.995987e+01   \n",
              "50%    4.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00  3.859945e+01   \n",
              "75%    5.000000e+00  1.000000e+00  0.000000e+00  0.000000e+00  3.994374e+01   \n",
              "max    5.000000e+00  4.200000e+02  7.920000e+02  4.040000e+02  5.367920e+01   \n",
              "\n",
              "          longitude         stars  review_count       is_open  \n",
              "count  4.724471e+06  4.724471e+06  4.724471e+06  4.724471e+06  \n",
              "mean  -8.903109e+01  3.792529e+00  4.925801e+02  7.987709e-01  \n",
              "std    1.429694e+01  6.258164e-01  8.502007e+02  4.009189e-01  \n",
              "min   -1.200837e+02  1.000000e+00  5.000000e+00  0.000000e+00  \n",
              "25%   -9.023588e+01  3.500000e+00  9.000000e+01  1.000000e+00  \n",
              "50%   -8.615980e+01  4.000000e+00  2.210000e+02  1.000000e+00  \n",
              "75%   -7.545180e+01  4.000000e+00  5.080000e+02  1.000000e+00  \n",
              "max   -7.466446e+01  5.000000e+00  7.568000e+03  1.000000e+00  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-66020141-51cb-4032-a111-5dbd87305069\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>stars_review</th>\n",
              "      <th>useful</th>\n",
              "      <th>funny</th>\n",
              "      <th>cool</th>\n",
              "      <th>latitude</th>\n",
              "      <th>longitude</th>\n",
              "      <th>stars</th>\n",
              "      <th>review_count</th>\n",
              "      <th>is_open</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>4.724471e+06</td>\n",
              "      <td>4.724471e+06</td>\n",
              "      <td>4.724471e+06</td>\n",
              "      <td>4.724471e+06</td>\n",
              "      <td>4.724471e+06</td>\n",
              "      <td>4.724471e+06</td>\n",
              "      <td>4.724471e+06</td>\n",
              "      <td>4.724471e+06</td>\n",
              "      <td>4.724471e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>3.793798e+00</td>\n",
              "      <td>9.840992e-01</td>\n",
              "      <td>3.013692e-01</td>\n",
              "      <td>4.795771e-01</td>\n",
              "      <td>3.587026e+01</td>\n",
              "      <td>-8.903109e+01</td>\n",
              "      <td>3.792529e+00</td>\n",
              "      <td>4.925801e+02</td>\n",
              "      <td>7.987709e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>1.391227e+00</td>\n",
              "      <td>2.739150e+00</td>\n",
              "      <td>1.717190e+00</td>\n",
              "      <td>2.159917e+00</td>\n",
              "      <td>5.337905e+00</td>\n",
              "      <td>1.429694e+01</td>\n",
              "      <td>6.258164e-01</td>\n",
              "      <td>8.502007e+02</td>\n",
              "      <td>4.009189e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.000000e+00</td>\n",
              "      <td>-1.000000e+00</td>\n",
              "      <td>-1.000000e+00</td>\n",
              "      <td>-1.000000e+00</td>\n",
              "      <td>2.756446e+01</td>\n",
              "      <td>-1.200837e+02</td>\n",
              "      <td>1.000000e+00</td>\n",
              "      <td>5.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>2.995987e+01</td>\n",
              "      <td>-9.023588e+01</td>\n",
              "      <td>3.500000e+00</td>\n",
              "      <td>9.000000e+01</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>4.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>3.859945e+01</td>\n",
              "      <td>-8.615980e+01</td>\n",
              "      <td>4.000000e+00</td>\n",
              "      <td>2.210000e+02</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>5.000000e+00</td>\n",
              "      <td>1.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>3.994374e+01</td>\n",
              "      <td>-7.545180e+01</td>\n",
              "      <td>4.000000e+00</td>\n",
              "      <td>5.080000e+02</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>5.000000e+00</td>\n",
              "      <td>4.200000e+02</td>\n",
              "      <td>7.920000e+02</td>\n",
              "      <td>4.040000e+02</td>\n",
              "      <td>5.367920e+01</td>\n",
              "      <td>-7.466446e+01</td>\n",
              "      <td>5.000000e+00</td>\n",
              "      <td>7.568000e+03</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-66020141-51cb-4032-a111-5dbd87305069')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-66020141-51cb-4032-a111-5dbd87305069 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-66020141-51cb-4032-a111-5dbd87305069');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# only keep the review text and the usefulness column, which we'll use a proxy for the binary label of 'helpful'\n",
        "review_df = review_df[['text', 'useful']]"
      ],
      "metadata": {
        "id": "otvMuqekVj51"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Take a look at the distribution of usefulness votes\n",
        "review_df.groupby(by='useful').apply(lambda x: (x['text'].count() / review_df.shape[0])*100)"
      ],
      "metadata": {
        "id": "LV0EHztAkLZ_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f4ae9d19-8b83-497a-8752-d38403d4ba3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "useful\n",
              "-1       0.000021\n",
              " 0      58.319291\n",
              " 1      21.973762\n",
              " 2       9.171461\n",
              " 3       4.298449\n",
              "          ...    \n",
              " 392     0.000021\n",
              " 399     0.000042\n",
              " 400     0.000021\n",
              " 404     0.000021\n",
              " 420     0.000021\n",
              "Length: 225, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "review_df.useful.hist(bins=420)\n",
        "# Can see that most of the votes are concentrated in 0-10, not may reviews have a large amount of votes but the tail is very long"
      ],
      "metadata": {
        "id": "hI2gRntNR3go",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 294
        },
        "outputId": "6af474b5-7419-424b-c087-bbe0a4b67a0a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7fde426b1890>"
            ]
          },
          "metadata": {},
          "execution_count": 13
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAPzUlEQVR4nO3cf6zddX3H8edrbf0xqjCtuxKKXtzqjIHp7A3iXMwtiUlFI8nGDMSoGLYuRicuuFlcgpt/uX90GpysmQRZHHUOs3VARhj2Bl2m0iICbYNW50IrWyc/ileNs/O9P85Xcq2355xeTu8593Ofj+Tkfn98vt/z5h36ut/7Od/zTVUhSVr5fmHcBUiSRsNAl6RGGOiS1AgDXZIaYaBLUiMMdElqxFgDPcn1SY4keWDI8W9Ksj/JviR/d6rrk6SVJOO8Dz3Ja4B54MaqOnfA2E3A3wMXVtVjSX65qo4sR52StBKM9Qq9qu4CHl24LcmvJPmXJHuTfCHJS7pdvw98vKoe6441zCVpgUmcQ98B/GFVbQbeC/xVt/3FwIuT/FuSLyXZOrYKJWkCrR13AQslWQ/8JvDZJD/d/PTu51pgEzALbATuSnJeVT2+3HVK0iSaqECn9xfD41X18kX2HQK+XFU/Bv4jydfpBfzdy1mgJE2qiZpyqaon6IX17wKk52Xd7n+kd3VOkg30pmC+NY46JWkSjfu2xZuAfwd+LcmhJFcAbwauSPI1YB9wcTf8duCRJPuB3cAfV9Uj46hbkibRWG9blCSNzkRNuUiSlm5sH4pu2LChpqenl3z897//fU477bTRFdQY+zOYPerP/vQ3rv7s3bv3u1X1vMX2jS3Qp6en2bNnz5KPn5ubY3Z2dnQFNcb+DGaP+rM//Y2rP0n+80T7nHKRpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGrMhAn95+67hLkKSJsyIDXZL08wx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktSIgYGe5Owku5PsT7IvyZWLjJlNcjTJvd3rmlNTriTpRNYOMeYYcFVV3ZPkWcDeJHdU1f7jxn2hqt4w+hIlScMYeIVeVQ9X1T3d8veAA8BZp7owSdLJSVUNPziZBu4Czq2qJxZsnwVuBg4B3wHeW1X7Fjl+G7ANYGpqavPOnTuXVPT9h49yzulrWL9+/ZKOXw3m5+ftzwD2qD/709+4+rNly5a9VTWz6M6qGuoFrAf2Ar+9yL5nA+u75YuAbww63+bNm2upXvi+W2r37t1LPn41sD+D2aP+7E9/4+oPsKdOkKtD3eWSZB29K/BPV9XnFvml8ERVzXfLtwHrkmw4yV88kqSnYJi7XAJ8EjhQVR8+wZjnd+NIcn533kdGWagkqb9h7nJ5NfAW4P4k93bb3g+8AKCqrgMuAd6R5BjwQ+DS7k8DSdIyGRjoVfVFIAPGXAtcO6qiJEknz2+KSlIjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREDAz3J2Ul2J9mfZF+SKxcZkyQfS3IwyX1JXnFqypUkncjaIcYcA66qqnuSPAvYm+SOqtq/YMzrgE3d65XAJ7qfkqRlMvAKvaoerqp7uuXvAQeAs44bdjFwY/V8CTgjyZkjr1aSdEKpquEHJ9PAXcC5VfXEgu23AB+qqi9263cC76uqPccdvw3YBjA1NbV5586dSyr6/sNHOef0Naxfv35Jx68G8/Pz9mcAe9Sf/elvXP3ZsmXL3qqaWWzfMFMuACRZD9wMvGdhmJ+MqtoB7ACYmZmp2dnZpZyGy7ffyg1bT2Opx68Gc3Nz9mcAe9Sf/elvEvsz1F0uSdbRC/NPV9XnFhlyGDh7wfrGbpskaZkMc5dLgE8CB6rqwycYtgt4a3e3ywXA0ap6eIR1SpIGGGbK5dXAW4D7k9zbbXs/8AKAqroOuA24CDgI/AB4++hLlST1MzDQuw86M2BMAe8cVVGSpJPnN0UlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhoxMNCTXJ/kSJIHTrB/NsnRJPd2r2tGX6YkaZC1Q4y5AbgWuLHPmC9U1RtGUpEkaUkGXqFX1V3Ao8tQiyTpKUhVDR6UTAO3VNW5i+ybBW4GDgHfAd5bVftOcJ5twDaAqampzTt37lxS0fcfPso5p69h/fr1Szp+NZifn7c/A9ij/uxPf+Pqz5YtW/ZW1cxi+0YR6M8GflJV80kuAj5aVZsGnXNmZqb27Nkz8L0XM739Vm7Yehqzs7NLOn41mJubsz8D2KP+7E9/4+pPkhMG+lO+y6Wqnqiq+W75NmBdkg1P9bySpJPzlAM9yfOTpFs+vzvnI0/1vJKkkzPwLpckNwGzwIYkh4APAOsAquo64BLgHUmOAT8ELq1h5nEkSSM1MNCr6rIB+6+ld1ujJGmM/KaoJDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGjEw0JNcn+RIkgdOsD9JPpbkYJL7krxi9GVKkgYZ5gr9BmBrn/2vAzZ1r23AJ556WYPdf/jocryNJK0YAwO9qu4CHu0z5GLgxur5EnBGkjNHVaAkaTijmEM/C3howfqhbpskaRmtXc43S7KN3rQMU1NTzM3NLek8V513jKlnsuTjV4P5+Xn7M4A96s/+9DeJ/RlFoB8Gzl6wvrHb9nOqagewA2BmZqZmZ2eX9IaXb7+Vq847xpuWePxqMDc3x1L7u1rYo/7sT3+T2J9RTLnsAt7a3e1yAXC0qh4ewXklSSdh4BV6kpuAWWBDkkPAB4B1AFV1HXAbcBFwEPgB8PZTVawk6cQGBnpVXTZgfwHvHFlFkqQl8ZuiktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNGCrQk2xN8mCSg0m2L7L/8iT/k+Te7vV7oy9VktTP2kEDkqwBPg68FjgE3J1kV1XtP27oZ6rqXaegRknSEIa5Qj8fOFhV36qq/wV2Ahef2rIkSSdr4BU6cBbw0IL1Q8ArFxn3O0leA3wd+KOqeuj4AUm2AdsApqammJubO+mCAa467xhTz2TJx68G8/Pz9mcAe9Sf/elvEvszTKAP45+Bm6rqR0n+APgUcOHxg6pqB7ADYGZmpmZnZ5f0Zpdvv5WrzjvGm5Z4/GowNzfHUvu7Wtij/uxPf5PYn2GmXA4DZy9Y39hte1JVPVJVP+pW/wbYPJryJEnDGibQ7wY2JTknydOAS4FdCwckOXPB6huBA6MrUZI0jIFTLlV1LMm7gNuBNcD1VbUvyQeBPVW1C3h3kjcCx4BHgctPYc2SpEUMNYdeVbcBtx237ZoFy1cDV4+2tMGmt9/Ktz/0+uV+W0maSH5TVJIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1IgVH+jT228ddwmSNBFWfKBLknoMdElqRBOB7rSLJDUS6JKkIQM9ydYkDyY5mGT7IvufnuQz3f4vJ5kedaGSpP4GBnqSNcDHgdcBLwUuS/LS44ZdATxWVb8KfAT4i1EXOojTLpJWu7VDjDkfOFhV3wJIshO4GNi/YMzFwJ91y/8AXJskVVUjrHWgxUL92x96/XKWIEljM0ygnwU8tGD9EPDKE42pqmNJjgLPBb67cFCSbcC2bnU+yYNLKRrg3bDh+PMvJsv+t8LEGKo/q5w96s/+9Deu/rzwRDuGCfSRqaodwI5RnCvJnqqaGcW5WmR/BrNH/dmf/iaxP8N8KHoYOHvB+sZu26JjkqwFTgceGUWBkqThDBPodwObkpyT5GnApcCu48bsAt7WLV8CfH65588labUbOOXSzYm/C7gdWANcX1X7knwQ2FNVu4BPAn+b5CDwKL3QP9VGMnXTMPszmD3qz/70N3H9iRfSktQGvykqSY0w0CWpESsy0Ac9imA1SHJ9kiNJHliw7TlJ7kjyje7nL3Xbk+RjXb/uS/KK8VW+PJKcnWR3kv1J9iW5sttuj4Akz0jylSRf6/rz5932c7rHdxzsHufxtG77qny8R5I1Sb6a5JZufaL7s+ICfchHEawGNwBbj9u2HbizqjYBd3br0OvVpu61DfjEMtU4TseAq6rqpcAFwDu7/0/sUc+PgAur6mXAy4GtSS6g99iOj3SP8XiM3mM9YAIe7zEmVwIHFqxPdn+qakW9gFcBty9Yvxq4etx1jakX08ADC9YfBM7sls8EHuyW/xq4bLFxq+UF/BPwWnu0aG9+EbiH3jfAvwus7bY/+W+N3l1ur+qW13bjMu7aT3FfNtL7pX8hcAuQSe/PirtCZ/FHEZw1plomzVRVPdwt/xcw1S2v6p51f/7+BvBl7NGTuumEe4EjwB3AN4HHq+pYN2RhD37m8R7ATx/v0bK/BP4E+Em3/lwmvD8rMdA1hOpdKqz6e1KTrAduBt5TVU8s3Lfae1RV/1dVL6d3JXo+8JIxlzQxkrwBOFJVe8ddy8lYiYE+zKMIVqv/TnImQPfzSLd9VfYsyTp6Yf7pqvpct9keHaeqHgd205tCOKN7fAf8bA9W2+M9Xg28Mcm3gZ30pl0+yoT3ZyUG+jCPIlitFj6C4W305o1/uv2t3Z0cFwBHF0w7NClJ6H2D+UBVfXjBLnsEJHlekjO65WfS+3zhAL1gv6Qbdnx/Vs3jParq6qraWFXT9DLm81X1Zia9P+P+4GGJH1ZcBHyd3pzfn467njH14CbgYeDH9ObyrqA3Z3cn8A3gX4HndGND786gbwL3AzPjrn8Z+vNb9KZT7gPu7V4X2aMn+/PrwFe7/jwAXNNtfxHwFeAg8Fng6d32Z3TrB7v9Lxr3f8My9moWuGUl9Mev/ktSI1bilIskaREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWrE/wOJtn1xtbID/gAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# If we class 'useful' as anything with 4 of more votes, anything with 1-3 votes we discard and anything with 0 votes is classed as 'unhelpful'\n",
        "# another way to do this could be to look at the total number of helpful votes across all reviews for a business and determine the share of helpful votes a review has, \n",
        "# and then create a cut off for 'helpful' vs. 'unhelpful'\n",
        "# We also may want to remove reviews for business where there have been no reviews marked as helpful, as this could just be to lack of visibility rather than their unhelpfulness.\n",
        "\n",
        "# subset data to our classification of helpful and unhelpful\n",
        "review_df = review_df[(review_df.useful > 3) | (review_df.useful == 0)]\n",
        "# now convert to binary labels\n",
        "review_df['label'] = np.where((review_df.useful > 3), 1, 0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XsAoidYLD8vt",
        "outputId": "273e00bb-0e94-4c43-ef5a-3b8d8a688902"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  if __name__ == '__main__':\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the balance\n",
        "review_df.groupby(by='label').apply(lambda x: (x['text'].count()*100)/review_df.shape[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1iHcQCaVEiRc",
        "outputId": "12f26bba-6931-4197-a992-2cd29fe2a3f2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "label\n",
              "0    90.338642\n",
              "1     9.661358\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Checking the length of the reviews across helpful and unhelpful labels\n",
        "review_df.groupby(by='label').apply(lambda x: x['text'].str.len().mean())\n",
        "# Unhelpful reviews tend to be shorter than helpful ones"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QIx8hRUuE3nS",
        "outputId": "b3964a02-0a6e-4f60-9cdb-56e1043190f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "label\n",
              "0     427.292856\n",
              "1    1092.766427\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Save text length as a variable as it is used in the most recent paper found for this problem space\n",
        "review_df['text_length'] = review_df['text'].str.len()"
      ],
      "metadata": {
        "id": "-Tm8ehSwFP5O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sns.boxplot(x=\"label\", y=\"text_length\", data=review_df);"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "Y4Sxy47KFXsD",
        "outputId": "ac7fa76f-77f9-4e02-d646-d38fa6c4a9a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEGCAYAAAB2EqL0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUSklEQVR4nO3df7BfdZ3f8efLBARUBJZsSm8IwUmWLTqNYgZQd3aidjGw28WdooPrSNbSplZk03ZnV3Q6pVWx605XxazSYskYd1Vk3VroLpWmaOrs1h8kCCK/hltWIHcRIkF+LAsafPeP77nwJebe+4ne74/kPh8z37nnfM6v9818J697zuecz0lVIUnSXJ436gIkSQcGA0OS1MTAkCQ1MTAkSU0MDElSk8WjLmBQjj322FqxYsWoy5CkA8qOHTu+X1VL9rXsoA2MFStWsH379lGXIUkHlCT3zLTMS1KSpCYGhiSpiYEhSWpiYEiSmhy0nd6aP2vXrn1metu2bSOrQ9qb383hGvgZRpLvJrklyU1JtndtxyTZmuSu7ufRXXuSfCzJZJJvJzmlbz/ru/XvSrJ+0HVLkp5rWJekXltVL6+qNd38RcD1VbUKuL6bBzgTWNV9NgCXQS9ggIuB04BTgYunQ0aD1f8X3L7mpVHxuzl8o+rDOBvY0k1vAd7Y1/7p6vk6cFSS44A3AFurandVPQxsBdYNu2hJWsiGERgF/K8kO5Js6NqWVtX93fT3gKXd9ARwX9+2O7u2mdqfI8mGJNuTbN+1a9d8/g6StOANo9P7l6pqKsnPA1uT3NG/sKoqyby8xamqLgcuB1izZo1vhpKkeTTwM4yqmup+Pgh8kV4fxAPdpSa6nw92q08Bx/dtvqxrm6ldkjQkAw2MJC9I8qLpaeAM4DvANcD0nU7rgau76WuA87q7pU4HHukuXV0HnJHk6K6z+4yuTZI0JIO+JLUU+GKS6WN9tqq+lOQG4Kok5wP3AG/u1r8WOAuYBJ4A3g5QVbuTvB+4oVvvfVW1e8C1S5L6DDQwqupuYPU+2h8CXr+P9gIumGFfm4HN812jJKmNQ4NIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpyVACI8miJN9K8ufd/IlJvpFkMsnnkxzatT+/m5/slq/o28d7uvY7k7xhGHVLkp41rDOMjcDtffMfAj5SVSuBh4Hzu/bzgYe79o9065HkZOBc4KXAOuATSRYNqXZJEkMIjCTLgF8F/ms3H+B1wBe6VbYAb+ymz+7m6Za/vlv/bODKqnqqqv4amAROHXTtkqRnDeMM46PA7wE/7uZ/DvhBVe3p5ncCE930BHAfQLf8kW79Z9r3sc0zkmxIsj3J9l27ds337yFJC9pAAyPJrwEPVtWOQR5nWlVdXlVrqmrNkiVLhnFISVowFg94/68Bfj3JWcBhwJHApcBRSRZ3ZxHLgKlu/SngeGBnksXAi4GH+tqn9W8jSRqCgZ5hVNV7qmpZVa2g12n95ap6K/AV4JxutfXA1d30Nd083fIvV1V17ed2d1GdCKwCvjnI2iVJzzXoM4yZvBu4MskHgG8BV3TtVwB/nGQS2E0vZKiqW5NcBdwG7AEuqKqnh1+2JC1cQwuMqtoGbOum72YfdzlV1ZPAm2bY/hLgksFVKEmajU96S5KaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWqyuGWlJL8A/C5wQv82VfW6AdUlSRozTYEB/Cnwn4FPAk8PrhxJ0rhqvSS1p6ouq6pvVtWO6c9cGyU5LMk3k9yc5NYk/6FrPzHJN5JMJvl8kkO79ud385Pd8hV9+3pP135nkjf8FL+rJOlnMGtgJDkmyTHA/0jyziTHTbd17XN5CnhdVa0GXg6sS3I68CHgI1W1EngYOL9b/3zg4a79I916JDkZOBd4KbAO+ESSRfv920qSfmpzXZLaARSQbv53+5YV8JLZNq6qAh7vZg/pPgW8DvjNrn0L8O+By4Czu2mALwB/lCRd+5VV9RTw10kmgVOBr81RvyRpnswaGFV1IvQuLVXVk/3LkhzWcoDuTGAHsBL4OPD/gB9U1Z5ulZ3ARDc9AdzXHXtPkkeAn+vav9632/5t+o+1AdgAsHz58pbyJEmNWvsw/m9j20+oqqer6uXAMnpnBb/YeMz9VlWXV9WaqlqzZMmSQR1GkhakWc8wkvw9en/JH57kFTx7aepI4Ij9OVBV/SDJV4BXAUclWdydZSwDprrVpoDjgZ1JFgMvBh7qa5/Wv40kaQjm6sN4A/Bb9P6D/nBf+2PAe+faeZIlwI+6sDgc+BV6HdlfAc4BrgTWA1d3m1zTzX+tW/7lqqok1wCfTfJh4O8Dq4BvtvyCkqT5MVcfxhZgS5J/UlV/9lPs/7hu+0X0Ln9dVVV/nuQ24MokHwC+BVzRrX8F8Mddp/ZuendGUVW3JrkKuA3YA1xQVT4PIklD1Prg3glJ/s1ebY8AO6rqppk2qqpvA6/YR/vd9Poz9m5/EnjTDPu6BLiksV5J0jxr7fReA7yDXn/GBPAv6D0P8ckkvzeg2iRJY6T1DGMZcEpVPQ6Q5GLgL4BfpnfL7B8MpjxJ0rhoPcP4eXpPbU/7EbC0qv5ur3ZJ0kGq9QzjM8A3kkzfzfSP6d219AJ6HdGSpINcU2BU1fuTfAl4ddf0jqra3k2/dSCVSZLGSusZBsCN9B6WWwyQZHlV3TuQqiSNvU2bNjE5OTnqMp5j48aNIzv2ypUrufDCC0d2/GFofYHShcDFwAP03ocReoMI/sPBlSZJGietZxgbgZOq6qFBFiPpwDHqv6bXrl37E22XXnrp8AtZQFrvkrqP3oN6kjQWtm3bNuu85l/rGcbdwLYkf0HfbbRV9eGZN5EkHUxaA+Pe7nNo95GkkVu9ejXgpahhab2tdvpd3EdU1RODLUmSNI6a+jCSvKobYfaObn51kk8MtDJJ0lhp7fT+KL13YzwEUFU30xtHSpK0QLQGBlV1315Nvo9CkhaQ1k7v+5K8Gqgkh9B7LuP2wZUlSRo3rWcY7wAuoPcujCng5d28JGmBaL1L6vs4yKAkLWizBkaSTfTGjNqnqvrtea9IkjSW5jrD2D7HcknSAjFrYFTVlpadJNlUVQf3uL6StMA131Y7h9fM034kSWNqvgJDknSQMzAkSU3mKzAyT/uRJI2p1sEH3zRHm2MLS9JBrvUM4z2ztVXVp+alGknS2Jrrwb0zgbOAiSQf61t0JLBnkIVJksbLXA/u/Q29h/d+HdjR1/4Y8K8HVZQkafzM9eDezcDNSa6vqp39y5KcBDw8yOIkSeOjtQ/j+iRvnp5J8jvAFwdTkiRpHLUGxlrgbUn+NMlXgV8ATp1royTHJ/lKktuS3JpkY9d+TJKtSe7qfh7dtSfJx5JMJvl2klP69rW+W/+uJOv3+zeVJP1MmgKjqu4HvgS8ClgBbKmqxxs23QP8TlWdDJwOXJDkZOAi4PqqWgVc380DnAms6j4bgMugFzDAxcBp9ILq4umQkSQNR+tzGP+b3n/WLwN+Ffhokv8013ZVdX9V3dhNP0bvLX0TwNnA9MCGW4A3dtNnA5+unq8DRyU5jt77xLdW1e6qehjYCqxr/B0lSfOg9ZLUH1XVeVX1g6q6hd6ZxiP7c6AkK4BXAN8AlnZnLQDfA5Z20xNA/7vDd3ZtM7XvfYwNSbYn2b5r1679KU+SNIfWS1L/PckvJXl713Q08CetB0nyQuDPgH9VVY/ute9ilpc07Y+quryq1lTVmiVLlszHLiVJndZLUhcD7+bZp7sPpTEwkhxCLyw+U1X/rWt+oLvURPfzwa59Cji+b/NlXdtM7ZKkIWm9JPUb9B7e+1uAqvob4EVzbZQkwBXA7VX14b5F1wDTdzqtB67uaz+vu1vqdOCR7tLVdcAZSY7uOrvP6NokSUMy15Pe035YVZWkAJK8oHG71wBvA25JclPX9l7g94GrkpwP3ANMP+NxLb2hSCaBJ4C3A1TV7iTvB27o1ntfVe1urEGSNA9aA+OqJP+F3l1L/xz4p8An59qoqv6SmYc+f/0+1i/gghn2tRnY3FivJGmetQbGEuALwKPAScC/A/7RoIqSJI2f1sD4lap6N73nHwBI8of0OsIlSQvAXMOb/0vgncBLkny7b9GLgL8aZGGSpPEy1xnGZ4H/CfxHnh2+A+AxO50laWGZa3jzR+g90f2W4ZQjSRpXrc9hSJIWOANDktTEwJAkNTEwJElNDAxJUhMDQ5LUxMCQJDUxMCRJTQwMSVITA0OS1MTAkCQ1aR3eXNKY2LRpE5OTk6MuYyxM/zts3LhxxJWMh5UrV3LhhRcObP8GhnSAmZyc5K5bv8XyFz496lJG7tAf9S6SPHXP9hFXMnr3Pr5o4McwMKQD0PIXPs17T3l01GVojHzwxiMHfgz7MCRJTQwMSVITA0OS1MTAkCQ1MTAkSU0MDElSEwNDktTEwJAkNTEwJElNDAxJUhMDQ5LUZKBjSSXZDPwa8GBVvaxrOwb4PLAC+C7w5qp6OEmAS4GzgCeA36qqG7tt1gP/ttvtB6pqyyDrHhfjOirpqEYGHfRInJJmN+gzjE8B6/Zquwi4vqpWAdd38wBnAqu6zwbgMngmYC4GTgNOBS5OcvSA65Yk7WWgZxhV9dUkK/ZqPhtY201vAbYB7+7aP11VBXw9yVFJjuvW3VpVuwGSbKUXQp8bZO3jYBz+ml67du1PtF166aXDL0TSyI2iD2NpVd3fTX8PWNpNTwD39a23s2ubqf0nJNmQZHuS7bt27ZrfqiVpgRtpp3d3NlHzuL/Lq2pNVa1ZsmTJfO12Qdu2bdus85IWjlEExgPdpSa6nw927VPA8X3rLevaZmqXJA3RKALjGmB9N70euLqv/bz0nA480l26ug44I8nRXWf3GV2bhmT16tWsXr3aswtpgRv0bbWfo9dpfWySnfTudvp94Kok5wP3AG/uVr+W3i21k/Ruq307QFXtTvJ+4IZuvfdNd4BLC9HU1BR/+9iiobySUweOex5bxAumBnvxZdB3Sb1lhkWv38e6BVwww342A5vnsTRJ0n4aaGBImn8TExM8ted+3nvKo6MuRWPkgzceyfMn9nkD6bxxaBBJUhMDQ5LUxMCQJDUxMCRJTQwMSVITA0OS1MTAkCQ1MTAkSU0MDElSEwNDktTEoUGkA9C9jzv4IMADT/T+5l16xI9HXMno3fv4IlYN+BgGxgw2bdrE5OTkqMsYC9P/Dhs3bhxxJeNh5cqVI3197sqVK0d27HHzw+67+fwT/DdZxeC/GwbGDCYnJ7npO7fz9BHHjLqUkXveD3svRdxx9wMjrmT0Fj0x+pH1x+Fd7+Ni+o8Y3zM/HAbGLJ4+4hj+7hfPGnUZGiOH33HtqEuQRsZOb0lSEwNDktTEwJAkNTEwJElNDAxJUhMDQ5LUxNtqZzA1NcWiJx7xNko9x6InHmJqas+oy5BGwjMMSVITzzBmMDExwfeeWuyDe3qOw++4lomJpaMuQxoJzzAkSU0MDElSEy9JzWLRE7vt9Aae9+SjAPz4MIfT7g0+6CUpLUwGxgwcQvpZk5OPAbDyJf5HCUv9bmjBMjBm4BDSz3IIaUlgH4YkqZGBIUlqckAFRpJ1Se5MMpnkolHXI0kLyQETGEkWAR8HzgROBt6S5OTRViVJC8eB1Ol9KjBZVXcDJLkSOBu4baRVDdCmTZuY7F5yP0rTNUx3fo/KypUrvRlhjIzD93NcvpuwML6fB1JgTAD39c3vBE7rXyHJBmADwPLly4dX2UHu8MMPH3UJ0j753RyuVNWoa2iS5BxgXVX9s27+bcBpVfWufa2/Zs2a2r59+zBLlKQDXpIdVbVmX8sOmD4MYAo4vm9+WdcmSRqCAykwbgBWJTkxyaHAucA1I65JkhaMA6YPo6r2JHkXcB2wCNhcVbeOuCxJWjAOmMAAqKprAUcDlKQROJAuSUmSRsjAkCQ1MTAkSU0MDElSkwPmwb39lWQXcM+o6ziIHAt8f9RFSPvgd3N+nVBVS/a14KANDM2vJNtnevpTGiW/m8PjJSlJUhMDQ5LUxMBQq8tHXYA0A7+bQ2IfhiSpiWcYkqQmBoYkqYmBoVklWZfkziSTSS4adT3StCSbkzyY5DujrmWhMDA0oySLgI8DZwInA29JcvJoq5Ke8Slg3aiLWEgMDM3mVGCyqu6uqh8CVwJnj7gmCYCq+iqwe9R1LCQGhmYzAdzXN7+za5O0ABkYkqQmBoZmMwUc3ze/rGuTtAAZGJrNDcCqJCcmORQ4F7hmxDVJGhEDQzOqqj3Au4DrgNuBq6rq1tFWJfUk+RzwNeCkJDuTnD/qmg52Dg0iSWriGYYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSHNkySPz7F8xf6OrJrkU0nO+dkqk+aHgSFJamJgSPMsyQuTXJ/kxiS3JOkf4Xdxks8kuT3JF5Ic0W3zyiT/J8mOJNclOW5E5UszMjCk+fck8BtVdQrwWuAPk6RbdhLwiar6B8CjwDuTHAJsAs6pqlcCm4FLRlC3NKvFoy5AOggF+GCSXwZ+TG9I+KXdsvuq6q+66T8Bfhv4EvAyYGuXK4uA+4dasdTAwJDm31uBJcArq+pHSb4LHNYt23ssnqIXMLdW1auGV6K0/7wkJc2/FwMPdmHxWuCEvmXLk0wHw28CfwncCSyZbk9ySJKXDrViqYGBIc2/zwBrktwCnAfc0bfsTuCCJLcDRwOXda+/PQf4UJKbgZuAVw+5ZmlOjlYrSWriGYYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKa/H/By1YDeiC6JQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create train and test data"
      ],
      "metadata": {
        "id": "JVa_1CPEIMv0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# We have a big class imbalance in our dataset, we'll correct this by doing some undersampling on the negative class\n",
        "\n",
        "# Find out how many postive labels we have:\n",
        "positive_samples = review_df[(review_df.label == 1)]\n",
        "n_positive_samples = positive_samples.shape[0]\n",
        "\n",
        "# Sample from negative class so we have the same amount:\n",
        "negative_samples = review_df[(review_df.label == 0)].sample(n=n_positive_samples)\n",
        "\n",
        "balanced_df = positive_samples.append(negative_samples)"
      ],
      "metadata": {
        "id": "a403XrtsrFjz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check balance\n",
        "balanced_df.groupby(by='label').count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "CrH_CrNBsQFO",
        "outputId": "7fe55d53-6d4f-4c3c-ceb9-80e74fbaabfa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         text  useful  text_length\n",
              "label                             \n",
              "0      294666  294666       294666\n",
              "1      294666  294666       294666"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-041ad53b-1e16-4422-962b-6c0dc0e3e433\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>useful</th>\n",
              "      <th>text_length</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>label</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>294666</td>\n",
              "      <td>294666</td>\n",
              "      <td>294666</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>294666</td>\n",
              "      <td>294666</td>\n",
              "      <td>294666</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-041ad53b-1e16-4422-962b-6c0dc0e3e433')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-041ad53b-1e16-4422-962b-6c0dc0e3e433 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-041ad53b-1e16-4422-962b-6c0dc0e3e433');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "# We have ~600k rows, trimming down to ~60k for speed\n",
        "sampled_df = balanced_df.sample(frac=0.1)\n",
        "x = list(sampled_df[['text']].text)\n",
        "y = list(sampled_df[['label']].label)\n",
        "x_train, x_rem, y_train, y_rem = train_test_split(x, y, train_size=0.8)\n",
        "x_valid, x_test, y_valid, y_test = train_test_split(x_rem, y_rem, train_size=0.5)\n",
        "\n",
        "print(f\"Shape X_train: {len(x_train)}\")\n",
        "print(f\"Shape X_valid: {len(x_valid)}\")\n",
        "print(f\"Shape X_test: {len(x_test)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6frmsy86IPTr",
        "outputId": "77870b20-1aa1-47c1-df8d-cc746c2100d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape X_train: 47146\n",
            "Shape X_valid: 5893\n",
            "Shape X_test: 5894\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#pd.DataFrame({'text': x_train, 'label': y_train}).to_csv('/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_train.csv', index = False)\n",
        "#pd.DataFrame({'text': x_test, 'label': y_test}).to_csv('/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_test.csv', index = False)\n",
        "#pd.DataFrame({'text': x_valid, 'label': y_valid}).to_csv('/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_valid.csv', index = False)"
      ],
      "metadata": {
        "id": "IKVAll-UTsuR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Getting a larger data set for comparison:\n",
        "from sklearn.model_selection import train_test_split\n",
        "sampled_df = balanced_df\n",
        "x = list(sampled_df[['text']].text)\n",
        "y = list(sampled_df[['label']].label)\n",
        "x_train_large, x_rem_large, y_train_large, y_rem_large= train_test_split(x, y, train_size=0.8)\n",
        "x_valid_large, x_test_large, y_valid_large, y_test_large = train_test_split(x_rem_large, y_rem_large, train_size=0.5)\n",
        "\n",
        "print(f\"Shape X_train: {len(x_train_large)}\")\n",
        "print(f\"Shape X_valid: {len(x_valid_large)}\")\n",
        "print(f\"Shape X_test: {len(x_test_large)}\")\n",
        "\n",
        "pd.DataFrame({'text': x_train_large, 'label': y_train_large}).to_csv('/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_train_large.csv', index = False)\n",
        "pd.DataFrame({'text': x_test_large, 'label': y_test_large}).to_csv('/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_test_large.csv', index = False)\n",
        "pd.DataFrame({'text': x_valid_large, 'label': y_valid_large}).to_csv('/content/drive/My Drive/Colab Notebooks/W266 Project/data/yelp_valid_large.csv', index = False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TqOPHJrS84T9",
        "outputId": "80fc40b0-baca-4286-972c-8dc508118a82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape X_train: 471465\n",
            "Shape X_valid: 58933\n",
            "Shape X_test: 58934\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "machine_shape": "hm",
      "name": "Yelp_Data_EDA.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "background_execution": "on"
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}