sentiment-analysis-of-imdb-movie-reviews.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "source": [
    "**Sentiment Analysis of IMDB Movie Reviews**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Problem Statement:**\n",
    "\n",
    "In this, we have to predict the number of positive and negative reviews based on sentiments by using different classification models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "1424638f5259100af9f9a5c1b05bd23cf5b71e51"
   },
   "source": [
    "**Import necessary libraries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['IMDB Dataset.csv']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ngame\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     C:\\Users\\ngame\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\ngame\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\ngame\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Load the libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from wordcloud import WordCloud,STOPWORDS\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize,sent_tokenize\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "from nltk.corpus import wordnet\n",
    "from bs4 import BeautifulSoup\n",
    "import spacy\n",
    "import re,string,unicodedata\n",
    "from nltk.tokenize.toktok import ToktokTokenizer\n",
    "from nltk.stem import LancasterStemmer,WordNetLemmatizer\n",
    "from sklearn.linear_model import LogisticRegression,SGDClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import SVC\n",
    "from textblob import TextBlob\n",
    "from textblob import Word\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../Documents/input\"))\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Download necessary corpora\n",
    "nltk.download('wordnet')\n",
    "nltk.download('vader_lexicon')\n",
    "nltk.download('omw-1.4')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "be1b642cce343f7a8f68f8c91f7c50372cdf4381"
   },
   "source": [
    "**Import the training dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "_uuid": "4c593c17588723c0b0b0f19851cb70a8447ced76",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>One of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Probably my all-time favorite movie, a story o...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>I sure would like to see a resurrection of a u...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>This show was an amazing, fresh &amp; innovative i...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Encouraged by the positive comments about this...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>If you like original gut wrenching laughter yo...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  One of the other reviewers has mentioned that ...  positive\n",
       "1  A wonderful little production. <br /><br />The...  positive\n",
       "2  I thought this was a wonderful way to spend ti...  positive\n",
       "3  Basically there's a family where a little boy ...  negative\n",
       "4  Petter Mattei's \"Love in the Time of Money\" is...  positive\n",
       "5  Probably my all-time favorite movie, a story o...  positive\n",
       "6  I sure would like to see a resurrection of a u...  positive\n",
       "7  This show was an amazing, fresh & innovative i...  negative\n",
       "8  Encouraged by the positive comments about this...  negative\n",
       "9  If you like original gut wrenching laughter yo...  positive"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#importing the training data\n",
    "imdb_data=pd.read_csv('../Documents/input/IMDB Dataset.csv')\n",
    "print(imdb_data.shape)\n",
    "imdb_data.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "1ad3773974351ed9bdf389b2847d7475b36c2295"
   },
   "source": [
    "**Exploratery data analysis**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "_uuid": "7f11c83b1320c8982b36889145f7f770563674a8"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>50000</td>\n",
       "      <td>50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>49582</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>Loved today's show!!! It was a variety and not...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>5</td>\n",
       "      <td>25000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   review sentiment\n",
       "count                                               50000     50000\n",
       "unique                                              49582         2\n",
       "top     Loved today's show!!! It was a variety and not...  positive\n",
       "freq                                                    5     25000"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Summary of the dataset\n",
    "imdb_data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "453c3fd238f62ab8f649eb01771817e25bc0c77d"
   },
   "source": [
    "**Sentiment count**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "_uuid": "cb6bb97b0f851947dcf341a1de5708a1f2bc64c1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "positive    25000\n",
       "negative    25000\n",
       "Name: sentiment, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sentiment count\n",
    "imdb_data['sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that the dataset is balanced."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "f61964573faababe1f7897b77d32815a24954d2f"
   },
   "source": [
    "**Spliting the training dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "_uuid": "d3aaabff555e07feb11c72cc3a6e457615975ffe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000,) (40000,)\n",
      "(10000,) (10000,)\n"
     ]
    }
   ],
   "source": [
    "#split the dataset  \n",
    "#train dataset\n",
    "train_reviews=imdb_data.review[:40000]\n",
    "train_sentiments=imdb_data.sentiment[:40000]\n",
    "#test dataset\n",
    "test_reviews=imdb_data.review[40000:]\n",
    "test_sentiments=imdb_data.sentiment[40000:]\n",
    "print(train_reviews.shape,train_sentiments.shape)\n",
    "print(test_reviews.shape,test_sentiments.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "90da29c3b79f46f41d7391a2a116065b616d0fac"
   },
   "source": [
    "**Text normalization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "_uuid": "f000c43d91f68f6668539f089c6a54c5ce3bd819"
   },
   "outputs": [],
   "source": [
    "#Tokenization of text\n",
    "tokenizer=ToktokTokenizer()\n",
    "#Setting English stopwords\n",
    "stopword_list=nltk.corpus.stopwords.words('english')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "328b6e5977da3e055ad4b2e11a31e5e12ccf3b16"
   },
   "source": [
    "**Removing html strips and noise text**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "_uuid": "6f6fcafbdadcdcb0c164e37d71fb9d1623f74d0a"
   },
   "outputs": [],
   "source": [
    "#Removing the html strips\n",
    "def strip_html(text):\n",
    "    soup = BeautifulSoup(text, \"html.parser\")\n",
    "    return soup.get_text()\n",
    "\n",
    "#Removing the square brackets\n",
    "def remove_between_square_brackets(text):\n",
    "    return re.sub('\\[[^]]*\\]', '', text)\n",
    "\n",
    "#Removing the noisy text\n",
    "def denoise_text(text):\n",
    "    text = strip_html(text)\n",
    "    text = remove_between_square_brackets(text)\n",
    "    return text\n",
    "#Apply function on review column\n",
    "imdb_data['review']=imdb_data['review'].apply(denoise_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "88117b74761d1047924d6d70f76642faa0e706ac"
   },
   "source": [
    "**Removing special characters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "_uuid": "219da72b025121fd98081df50ae0fcaace10cc9d"
   },
   "outputs": [],
   "source": [
    "#Define function for removing special characters\n",
    "def remove_special_characters(text, remove_digits=True):\n",
    "    pattern=r'[^a-zA-z0-9\\s]'\n",
    "    text=re.sub(pattern,'',text)\n",
    "    return text\n",
    "#Apply function on review column\n",
    "imdb_data['review']=imdb_data['review'].apply(remove_special_characters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "3b66eeabd5b7b8c251f8b8ddf331140a64bcd514"
   },
   "source": [
    "**Text stemming**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "_uuid": "2295f2946e0ab74c220ad538d0e7adc04d23f697"
   },
   "outputs": [],
   "source": [
    "#Stemming the text\n",
    "def simple_stemmer(text):\n",
    "    ps=nltk.porter.PorterStemmer()\n",
    "    text= ' '.join([ps.stem(word) for word in text.split()])\n",
    "    return text\n",
    "#Apply function on review column\n",
    "imdb_data['review']=imdb_data['review'].apply(simple_stemmer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "e83107e4a281d84d7ae42b4e2c8d81b7ece438e4"
   },
   "source": [
    "**Removing stopwords**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "_uuid": "5dbff82b4d2d188d8777b273a75d8ac714d38885"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'this', 'needn', 'hasn', 'any', 'his', 'too', 'these', 'ain', 'were', 'under', 'had', 'than', \"you're\", 'yourself', 'all', 'while', 'have', 'below', 'ourselves', 'who', \"hadn't\", 'how', 'himself', 'm', \"mightn't\", 'such', 'should', 'i', 'hadn', 'mightn', 'above', 'not', 'shan', 'those', 'd', 'only', 'against', 'that', 'hers', \"don't\", \"you'll\", 'yours', 'being', 'as', 'our', 'up', \"couldn't\", 'some', \"wasn't\", 'be', 'did', 'couldn', 'it', 'won', 'themselves', 're', 'weren', 'same', 'your', 've', 'or', 'from', 'off', 'when', \"aren't\", 'yourselves', 'most', 'few', 'until', 'haven', \"she's\", 'out', 'then', 'can', 'wouldn', 'she', 'again', \"you'd\", 'both', 'didn', \"it's\", 'we', 'am', 's', 'an', 'so', \"you've\", 'but', 'once', 'nor', \"won't\", 'of', 'for', \"doesn't\", 'before', 'isn', 'don', 'by', 'through', \"weren't\", 'herself', 'with', 'during', 'if', 'on', \"shouldn't\", 'no', 'y', 'you', 't', 'was', \"hasn't\", 'is', 'why', 'about', \"wouldn't\", 'aren', 'what', 'o', 'just', 'myself', 'does', 'because', 'itself', 'shouldn', 'him', 'doesn', 'them', 'doing', \"isn't\", 'mustn', 'the', 'are', 'their', 'more', 'other', 'will', 'll', 'further', \"didn't\", 'where', 'in', 'there', 'he', 'each', 'which', \"should've\", 'here', 'has', 'wasn', 'now', \"needn't\", 'its', 'a', 'between', 'ma', 'into', 'her', 'at', 'to', \"haven't\", 'whom', 'been', 'they', 'do', 'after', \"shan't\", 'own', 'very', \"that'll\", 'having', 'me', \"mustn't\", 'theirs', 'over', 'down', 'ours', 'my', 'and'}\n"
     ]
    }
   ],
   "source": [
    "#set stopwords to english\n",
    "stop=set(stopwords.words('english'))\n",
    "print(stop)\n",
    "\n",
    "#removing the stopwords\n",
    "def remove_stopwords(text, is_lower_case=False):\n",
    "    tokens = tokenizer.tokenize(text)\n",
    "    tokens = [token.strip() for token in tokens]\n",
    "    if is_lower_case:\n",
    "        filtered_tokens = [token for token in tokens if token not in stopword_list]\n",
    "    else:\n",
    "        filtered_tokens = [token for token in tokens if token.lower() not in stopword_list]\n",
    "    filtered_text = ' '.join(filtered_tokens)    \n",
    "    return filtered_text\n",
    "#Apply function on review column\n",
    "imdb_data['review']=imdb_data['review'].apply(remove_stopwords)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "b35e7499291173119ed42287deac6f0cd96516e1"
   },
   "source": [
    "**Normalized train reviews**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "_kg_hide-output": true,
    "_uuid": "b20c242bd091929ca896ea2c6e936ca00efe6ecf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'one review ha mention watch 1 oz episod youll hook right thi exactli happen meth first thing struck oz wa brutal unflinch scene violenc set right word go trust thi show faint heart timid thi show pull punch regard drug sex violenc hardcor classic use wordit call oz nicknam given oswald maximum secur state penitentari focus mainli emerald citi experiment section prison cell glass front face inward privaci high agenda em citi home manyaryan muslim gangsta latino christian italian irish moreso scuffl death stare dodgi deal shadi agreement never far awayi would say main appeal show due fact goe show wouldnt dare forget pretti pictur paint mainstream audienc forget charm forget romanceoz doesnt mess around first episod ever saw struck nasti wa surreal couldnt say wa readi watch develop tast oz got accustom high level graphic violenc violenc injustic crook guard wholl sold nickel inmat wholl kill order get away well manner middl class inmat turn prison bitch due lack street skill prison experi watch oz may becom comfort uncomfort viewingthat get touch darker side'"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#normalized train reviews\n",
    "norm_train_reviews=imdb_data.review[:40000]\n",
    "norm_train_reviews[0]\n",
    "#convert dataframe to string\n",
    "#norm_train_string=norm_train_reviews.to_string()\n",
    "#Spelling correction using Textblob\n",
    "#norm_train_spelling=TextBlob(norm_train_string)\n",
    "#norm_train_spelling.correct()\n",
    "#Tokenization using Textblob\n",
    "#norm_train_words=norm_train_spelling.words\n",
    "#norm_train_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "d69462bb209a66cff86376dc8481d0c0140d894d"
   },
   "source": [
    "**Normalized test reviews**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "_kg_hide-output": true,
    "_uuid": "c5d0d38bd9976150367e9d75f3b933774c96a1ab"
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [57]\u001b[0m, in \u001b[0;36m<cell line: 8>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[38;5;66;03m# spelling correction using Textblob\u001b[39;00m\n\u001b[0;32m      7\u001b[0m norm_test_spelling\u001b[38;5;241m=\u001b[39mTextBlob(norm_test_string)\n\u001b[1;32m----> 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43mnorm_test_spelling\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcorrect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m      9\u001b[0m \u001b[38;5;66;03m# Tokenization using Textblob\u001b[39;00m\n\u001b[0;32m     10\u001b[0m norm_test_words\u001b[38;5;241m=\u001b[39mnorm_test_spelling\u001b[38;5;241m.\u001b[39mwords\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\blob.py:549\u001b[0m, in \u001b[0;36mBaseBlob.correct\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    547\u001b[0m tokens \u001b[38;5;241m=\u001b[39m nltk\u001b[38;5;241m.\u001b[39mtokenize\u001b[38;5;241m.\u001b[39mregexp_tokenize(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw, \u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw+|[^\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms]|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    548\u001b[0m corrected \u001b[38;5;241m=\u001b[39m (Word(w)\u001b[38;5;241m.\u001b[39mcorrect() \u001b[38;5;28;01mfor\u001b[39;00m w \u001b[38;5;129;01min\u001b[39;00m tokens)\n\u001b[1;32m--> 549\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcorrected\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    550\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m(ret)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\blob.py:548\u001b[0m, in \u001b[0;36m<genexpr>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    546\u001b[0m \u001b[38;5;66;03m# regex matches: word or punctuation or whitespace\u001b[39;00m\n\u001b[0;32m    547\u001b[0m tokens \u001b[38;5;241m=\u001b[39m nltk\u001b[38;5;241m.\u001b[39mtokenize\u001b[38;5;241m.\u001b[39mregexp_tokenize(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw, \u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw+|[^\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms]|\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124ms\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 548\u001b[0m corrected \u001b[38;5;241m=\u001b[39m (\u001b[43mWord\u001b[49m\u001b[43m(\u001b[49m\u001b[43mw\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcorrect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m w \u001b[38;5;129;01min\u001b[39;00m tokens)\n\u001b[0;32m    549\u001b[0m ret \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(corrected)\n\u001b[0;32m    550\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m(ret)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\blob.py:115\u001b[0m, in \u001b[0;36mWord.correct\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    109\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcorrect\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    110\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Correct the spelling of the word. Returns the word with the highest\u001b[39;00m\n\u001b[0;32m    111\u001b[0m \u001b[38;5;124;03m    confidence using the spelling corrector.\u001b[39;00m\n\u001b[0;32m    112\u001b[0m \n\u001b[0;32m    113\u001b[0m \u001b[38;5;124;03m    .. versionadded:: 0.6.0\u001b[39;00m\n\u001b[0;32m    114\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 115\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m Word(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mspellcheck\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;241m0\u001b[39m])\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\blob.py:107\u001b[0m, in \u001b[0;36mWord.spellcheck\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     98\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mspellcheck\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m     99\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Return a list of (word, confidence) tuples of spelling corrections.\u001b[39;00m\n\u001b[0;32m    100\u001b[0m \n\u001b[0;32m    101\u001b[0m \u001b[38;5;124;03m    Based on: Peter Norvig, \"How to Write a Spelling Corrector\"\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    105\u001b[0m \u001b[38;5;124;03m    .. versionadded:: 0.6.0\u001b[39;00m\n\u001b[0;32m    106\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 107\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msuggest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstring\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\en\\__init__.py:118\u001b[0m, in \u001b[0;36msuggest\u001b[1;34m(w)\u001b[0m\n\u001b[0;32m    116\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msuggest\u001b[39m(w):\n\u001b[0;32m    117\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Returns a list of (word, confidence)-tuples of spelling corrections.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 118\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mspelling\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msuggest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mw\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\_text.py:1692\u001b[0m, in \u001b[0;36mSpelling.suggest\u001b[1;34m(self, w)\u001b[0m\n\u001b[0;32m   1687\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m w\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39misdigit():\n\u001b[0;32m   1688\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m [(w, \u001b[38;5;241m1.0\u001b[39m)]  \u001b[38;5;66;03m# 1.5\u001b[39;00m\n\u001b[0;32m   1689\u001b[0m candidates \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m   1690\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_known([w])\n\u001b[0;32m   1691\u001b[0m     \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_known(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_edit1(w))\n\u001b[1;32m-> 1692\u001b[0m     \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_known(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_edit2\u001b[49m\u001b[43m(\u001b[49m\u001b[43mw\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m   1693\u001b[0m     \u001b[38;5;129;01mor\u001b[39;00m [w]\n\u001b[0;32m   1694\u001b[0m )\n\u001b[0;32m   1695\u001b[0m candidates \u001b[38;5;241m=\u001b[39m [(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget(c, \u001b[38;5;241m0.0\u001b[39m), c) \u001b[38;5;28;01mfor\u001b[39;00m c \u001b[38;5;129;01min\u001b[39;00m candidates]\n\u001b[0;32m   1696\u001b[0m s \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mfloat\u001b[39m(\u001b[38;5;28msum\u001b[39m(p \u001b[38;5;28;01mfor\u001b[39;00m p, word \u001b[38;5;129;01min\u001b[39;00m candidates) \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;241m1\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\_text.py:1667\u001b[0m, in \u001b[0;36mSpelling._edit2\u001b[1;34m(self, w)\u001b[0m\n\u001b[0;32m   1664\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Returns a set of words with edit distance 2 from the given word\"\"\"\u001b[39;00m\n\u001b[0;32m   1665\u001b[0m \u001b[38;5;66;03m# Of all spelling errors, 99% is covered by edit distance 2.\u001b[39;00m\n\u001b[0;32m   1666\u001b[0m \u001b[38;5;66;03m# Only keep candidates that are actually known words (20% speedup).\u001b[39;00m\n\u001b[1;32m-> 1667\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mset\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43me2\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43me1\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_edit1\u001b[49m\u001b[43m(\u001b[49m\u001b[43mw\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43me2\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_edit1\u001b[49m\u001b[43m(\u001b[49m\u001b[43me1\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43me2\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\_text.py:1667\u001b[0m, in \u001b[0;36m<genexpr>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m   1664\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Returns a set of words with edit distance 2 from the given word\"\"\"\u001b[39;00m\n\u001b[0;32m   1665\u001b[0m \u001b[38;5;66;03m# Of all spelling errors, 99% is covered by edit distance 2.\u001b[39;00m\n\u001b[0;32m   1666\u001b[0m \u001b[38;5;66;03m# Only keep candidates that are actually known words (20% speedup).\u001b[39;00m\n\u001b[1;32m-> 1667\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mset\u001b[39m(e2 \u001b[38;5;28;01mfor\u001b[39;00m e1 \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_edit1(w) \u001b[38;5;28;01mfor\u001b[39;00m e2 \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_edit1(e1) \u001b[38;5;28;01mif\u001b[39;00m \u001b[43me2\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\textblob\\_text.py:103\u001b[0m, in \u001b[0;36mlazydict.__contains__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    100\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__iter__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    101\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lazy(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__iter__\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 103\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__contains__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs):\n\u001b[0;32m    104\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lazy(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__contains__\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m*\u001b[39margs)\n\u001b[0;32m    106\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__getitem__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs):\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#Normalized test reviews\n",
    "norm_test_reviews=imdb_data.review[40000:]\n",
    "norm_test_reviews[45005]\n",
    "#convert dataframe to string\n",
    "norm_test_string=norm_test_reviews.to_string()\n",
    "# spelling correction using Textblob\n",
    "norm_test_spelling=TextBlob(norm_test_string)\n",
    "print(norm_test_spelling.correct())\n",
    "# Tokenization using Textblob\n",
    "norm_test_words=norm_test_spelling.words\n",
    "norm_test_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Sentence: The movie is absolutely fantastic and thrilling.\n",
      "Augmented Sentence: The movie constitute absolutely antic and shudder .\n"
     ]
    }
   ],
   "source": [
    "sid = SentimentIntensityAnalyzer()\n",
    "import Levenshtein\n",
    "\n",
    "def most_different_word_lexical(control_word, word_list):\n",
    "    differences = {}\n",
    "    for word in word_list:\n",
    "        distance = Levenshtein.distance(control_word, word)\n",
    "        differences[word] = distance\n",
    "\n",
    "    # Return the word with the highest distance\n",
    "    return max(differences, key=differences.get)\n",
    "def get_synonym(word):\n",
    "    \"\"\"Get a synonym for a word while preserving its sentiment.\"\"\"\n",
    "    synonyms = wordnet.synsets(word)\n",
    "    if not synonyms:\n",
    "        return word \n",
    "\n",
    "    syns = []\n",
    "    for synonym in synonyms:\n",
    "        lemma = synonym.lemmas()[0].name()\n",
    "        if lemma != word:  \n",
    "            syns.append(lemma)\n",
    "    if len(syns) == 0:\n",
    "        return word\n",
    "    return most_different_word_lexical(word, syns)\n",
    "\n",
    "# Test with a sentence\n",
    "sentence = \"The movie is absolutely fantastic and thrilling.\"\n",
    "tokens = tokenizer.tokenize(sentence)\n",
    "\n",
    "augmented_tokens = [get_synonym(token) for token in tokens]\n",
    "augmented_sentence = ' '.join(augmented_tokens)\n",
    "\n",
    "print(\"Original Sentence:\", sentence)\n",
    "print(\"Augmented Sentence:\", augmented_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [59]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m review \u001b[38;5;129;01min\u001b[39;00m norm_train_reviews:\n\u001b[0;32m      3\u001b[0m     tokens \u001b[38;5;241m=\u001b[39m tokenizer\u001b[38;5;241m.\u001b[39mtokenize(review)\n\u001b[1;32m----> 4\u001b[0m     augmented_reviews\u001b[38;5;241m.\u001b[39mappend([get_synonym(token) \u001b[38;5;28;01mfor\u001b[39;00m token \u001b[38;5;129;01min\u001b[39;00m tokens])\n\u001b[0;32m      6\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(augmented_reviews[\u001b[38;5;241m0\u001b[39m])\n",
      "Input \u001b[1;32mIn [59]\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m review \u001b[38;5;129;01min\u001b[39;00m norm_train_reviews:\n\u001b[0;32m      3\u001b[0m     tokens \u001b[38;5;241m=\u001b[39m tokenizer\u001b[38;5;241m.\u001b[39mtokenize(review)\n\u001b[1;32m----> 4\u001b[0m     augmented_reviews\u001b[38;5;241m.\u001b[39mappend([\u001b[43mget_synonym\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtoken\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m token \u001b[38;5;129;01min\u001b[39;00m tokens])\n\u001b[0;32m      6\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(augmented_reviews[\u001b[38;5;241m0\u001b[39m])\n",
      "Input \u001b[1;32mIn [51]\u001b[0m, in \u001b[0;36mget_synonym\u001b[1;34m(word)\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(syns) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m     24\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m word\n\u001b[1;32m---> 25\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmost_different_word_lexical\u001b[49m\u001b[43m(\u001b[49m\u001b[43mword\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msyns\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[1;32mIn [51]\u001b[0m, in \u001b[0;36mmost_different_word_lexical\u001b[1;34m(control_word, word_list)\u001b[0m\n\u001b[0;32m      5\u001b[0m differences \u001b[38;5;241m=\u001b[39m {}\n\u001b[0;32m      6\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m word \u001b[38;5;129;01min\u001b[39;00m word_list:\n\u001b[1;32m----> 7\u001b[0m     distance \u001b[38;5;241m=\u001b[39m \u001b[43mLevenshtein\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdistance\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcontrol_word\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mword\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      8\u001b[0m     differences[word] \u001b[38;5;241m=\u001b[39m distance\n\u001b[0;32m     10\u001b[0m \u001b[38;5;66;03m# Return the word with the highest distance\u001b[39;00m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "augmented_reviews = []\n",
    "for review in norm_train_reviews:\n",
    "    tokens = tokenizer.tokenize(review)\n",
    "    augmented_reviews.append([get_synonym(token) for token in tokens])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'matchless recapitulation hour_angle note determine one oz episod youll overcharge veracious thi exactli find methamphetamine beginning matter fall_upon oz Washington barbarous unflinch picture violenc hardening veracious discussion function confidence thi appearance dim affection diffident thi appearance perpetrate punch attentiveness drug sexual_activity violenc hardcor authoritative manipulation wordit margin_call oz nicknam establish Oswald utmost secur Department_of_State penitentari concentrate mainli emerald citi experiment department prison cellular_telephone methamphetamine battlefront expression inbound privaci senior_high_school agenda em citi dwelling manyaryan Muslim gangsta Latin_American Christian Italian Irish moreso scuffl end gaze dodgi distribute shadi agreement never Army_for_the_Liberation_of_Rwanda awayi would pronounce independent solicitation appearance ascribable fact goe appearance wouldnt make_bold forget pretti pictur key mainstream audienc forget appeal forget romanceoz doesnt batch approximately beginning episod always experience fall_upon nasti Washington phantasmagoric couldnt pronounce Washington readi determine originate tast oz experience habituate senior_high_school horizontal_surface graphic violenc violenc injustic criminal precaution wholl betray nickel inmat wholl stamp_out regulate experience aside good manner middl course inmat change_by_reversal prison backbite ascribable miss street skill prison experi determine oz whitethorn becom consolation uncomfort viewingthat experience partake benighted English'"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(augmented_reviews[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'one review ha mention watch 1 oz episod youll hook right thi exactli happen meth first thing struck oz wa brutal unflinch scene violenc set right word go trust thi show faint heart timid thi show pull punch regard drug sex violenc hardcor classic use wordit call oz nicknam given oswald maximum secur state penitentari focus mainli emerald citi experiment section prison cell glass front face inward privaci high agenda em citi home manyaryan muslim gangsta latino christian italian irish moreso scuffl death stare dodgi deal shadi agreement never far awayi would say main appeal show due fact goe show wouldnt dare forget pretti pictur paint mainstream audienc forget charm forget romanceoz doesnt mess around first episod ever saw struck nasti wa surreal couldnt say wa readi watch develop tast oz got accustom high level graphic violenc violenc injustic crook guard wholl sold nickel inmat wholl kill order get away well manner middl class inmat turn prison bitch due lack street skill prison experi watch oz may becom comfort uncomfort viewingthat get touch darker side'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_train_reviews[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "1c2a872ffcb6b8076fdbbba641af12081b6022ef"
   },
   "source": [
    "**Bags of words model**\n",
    "\n",
    "It is used to convert text documents to numerical vectors or bag of words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "35cf9dcefb40b2dc520c5b0d559695324c46cc04"
   },
   "outputs": [],
   "source": [
    "#Count vectorizer for bag of words\n",
    "cv=CountVectorizer(min_df=0,max_df=1,binary=False,ngram_range=(1,3))\n",
    "#transformed train reviews\n",
    "cv_train_reviews=cv.fit_transform(norm_train_reviews)\n",
    "#transformed test reviews\n",
    "cv_test_reviews=cv.transform(norm_test_reviews)\n",
    "\n",
    "print('BOW_cv_train:',cv_train_reviews.shape)\n",
    "print('BOW_cv_test:',cv_test_reviews.shape)\n",
    "#vocab=cv.get_feature_names()-toget feature names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "52371868f05ff9cf157280c5acf0f5bc71ee176d"
   },
   "source": [
    "**Term Frequency-Inverse Document Frequency model (TFIDF)**\n",
    "\n",
    "It is used to convert text documents to  matrix of  tfidf features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "afe6de957339921e05a6faeaf731f2272fd31946",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Tfidf vectorizer\n",
    "tv=TfidfVectorizer(min_df=0,max_df=1,use_idf=True,ngram_range=(1,3))\n",
    "#transformed train reviews\n",
    "tv_train_reviews=tv.fit_transform(norm_train_reviews)\n",
    "#transformed test reviews\n",
    "tv_test_reviews=tv.transform(norm_test_reviews)\n",
    "print('Tfidf_train:',tv_train_reviews.shape)\n",
    "print('Tfidf_test:',tv_test_reviews.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}