OSNA_FakeNews_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5WgnkPZikMIA"
   },
   "source": [
    "#Project 2\n",
    "\n",
    "- Student 1: \n",
    "\n",
    "Chandni Patel\n",
    "\n",
    "A20455322\n",
    "\n",
    "- Student 2:\n",
    "\n",
    "Omkar Pawar\n",
    "\n",
    "A20448802"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5Io1LWv_S0QE",
    "outputId": "b40ecd9e-2c08-4e16-f28e-6aa9fb694063"
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import seaborn as sns\n",
    "from sklearn.naive_bayes import MultinomialNB,GaussianNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import nltk, re, string, collections\n",
    "from nltk.util import ngrams\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "\n",
    "from num2words import num2words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "K4-3CG-BUjen"
   },
   "source": [
    "## Load Train and Test Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "7vCwNygnTHBS"
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"train.csv\")\n",
    "test = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "id": "FYITtVX_URCZ",
    "outputId": "8badf81e-fae1-40f8-c8eb-1120cdbebdda"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tid1</th>\n",
       "      <th>tid2</th>\n",
       "      <th>title1_en</th>\n",
       "      <th>title2_en</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>195611</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>There are two new old-age insurance benefits f...</td>\n",
       "      <td>Police disprove \"bird's nest congress each per...</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>191474</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>\"If you do not come to Shenzhen, sooner or lat...</td>\n",
       "      <td>Shenzhen's GDP outstrips Hong Kong? Shenzhen S...</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25300</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>\"If you do not come to Shenzhen, sooner or lat...</td>\n",
       "      <td>The GDP overtopped Hong Kong? Shenzhen clarifi...</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>123757</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>\"If you do not come to Shenzhen, sooner or lat...</td>\n",
       "      <td>Shenzhen's GDP overtakes Hong Kong? Bureau of ...</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>141761</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>\"If you do not come to Shenzhen, sooner or lat...</td>\n",
       "      <td>Shenzhen's GDP outpaces Hong Kong? Defending R...</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  tid1  tid2                                          title1_en  \\\n",
       "0  195611     0     1  There are two new old-age insurance benefits f...   \n",
       "1  191474     2     3  \"If you do not come to Shenzhen, sooner or lat...   \n",
       "2   25300     2     4  \"If you do not come to Shenzhen, sooner or lat...   \n",
       "3  123757     2     8  \"If you do not come to Shenzhen, sooner or lat...   \n",
       "4  141761     2    11  \"If you do not come to Shenzhen, sooner or lat...   \n",
       "\n",
       "                                           title2_en      label  \n",
       "0  Police disprove \"bird's nest congress each per...  unrelated  \n",
       "1  Shenzhen's GDP outstrips Hong Kong? Shenzhen S...  unrelated  \n",
       "2  The GDP overtopped Hong Kong? Shenzhen clarifi...  unrelated  \n",
       "3  Shenzhen's GDP overtakes Hong Kong? Bureau of ...  unrelated  \n",
       "4  Shenzhen's GDP outpaces Hong Kong? Defending R...  unrelated  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I3bkkLfzUs9V"
   },
   "source": [
    "## Function for preprocessing\n",
    "- Convert to Lowercase\n",
    "- Remove punctuations\n",
    "- Remove single character if any\n",
    "- Remove stop words\n",
    "- Convert numbers to words\n",
    "- Lemmatization to get root words\n",
    "- Stemming or Lemmatization\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "HSWEJkfzUpw0"
   },
   "outputs": [],
   "source": [
    "# Function for preprocessing\n",
    "def preprocessing(line):\n",
    "    line = line.lower() # Convert to lower case\n",
    "    line = re.sub('[^\\w\\s]',\" \",str(line)) # Remove Extra white space\n",
    "    line = re.sub('[^a-zA-Z0-9]',\" \",str(line)) # Remove Special Characters\n",
    "    line = word_tokenize(line) # Tokenize words\n",
    "\n",
    "    lemmatizer = WordNetLemmatizer() # For lemmatization\n",
    "    final = \" \"\n",
    "\n",
    "    # Remove stopwords\n",
    "    for i in line:\n",
    "        if(i not in stopwords.words(\"english\")):\n",
    "\n",
    "          temp = lemmatizer.lemmatize(i) #Lemmatize\n",
    "          if temp.isnumeric():\n",
    "            temp = num2words(temp) # Convert number values to string\n",
    "        final += i +\" \"\n",
    "    return final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QhcKLwLOZW-F"
   },
   "source": [
    "## Preprocessing for each column title1 and title2 Training Data\n",
    "Here we create two parameters and preprocess each of them to get the preprocessed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "0_cY2cMaUT7r"
   },
   "outputs": [],
   "source": [
    "paramter_1 = train[\"title1_en\"]\n",
    "paramter_2 = train[\"title2_en\"]\n",
    "\n",
    "# Column: title_1\n",
    "preprocessed_1 = []\n",
    "for i in (range(len(paramter_1))):\n",
    "    preprocessed_1.append(preprocessing(paramter_1[i]))\n",
    "\n",
    "# Column: title_2\n",
    "preprocessed_2 = []\n",
    "for j in (range(len(paramter_2))):\n",
    "    preprocessed_2.append(preprocessing(paramter_2[j]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "m5QZ-zIBfT1N"
   },
   "source": [
    "Convert the preprocessed data to dataframe and store it in csv format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "jCcHtVlYbE8E"
   },
   "outputs": [],
   "source": [
    "train_preprocessed = pd.DataFrame()\n",
    "train_preprocessed[\"id\"] = train[\"id\"]\n",
    "train_preprocessed[\"title1_en\"] = preprocessed_1\n",
    "train_preprocessed[\"title2_en\"] = preprocessed_2\n",
    "train_preprocessed[\"label\"] = train[\"label\"]\n",
    "\n",
    "train_preprocessed.to_csv('train_preprocessed.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3x2HTt_ugQbP"
   },
   "source": [
    "## Preprocessing for testing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "nr_NbSeffxYE"
   },
   "outputs": [],
   "source": [
    "test_paramter_1 = test[\"title1_en\"]\n",
    "test_paramter_2 = test[\"title2_en\"]\n",
    "\n",
    "# Column: title_1\n",
    "test_preprocessed_1 = []\n",
    "for m in (range(len(test_paramter_1))):\n",
    "    test_preprocessed_1.append(preprocessing(test_paramter_1[m]))\n",
    "\n",
    "# Column: title_2\n",
    "test_preprocessed_2 = []\n",
    "for n in (range(len(test_paramter_2))):\n",
    "    test_preprocessed_2.append(preprocessing(test_paramter_2[n]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "5sJjaryRhfL3"
   },
   "outputs": [],
   "source": [
    "test_preprocessed = pd.DataFrame()\n",
    "test_preprocessed[\"id\"] = test[\"id\"]\n",
    "test_preprocessed[\"title1_en\"] = test_preprocessed_1\n",
    "test_preprocessed[\"title2_en\"] = test_preprocessed_2\n",
    "\n",
    "test_preprocessed.to_csv('test_preprocessed.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "OSNA_FakeNews_Preprocessing",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}