Feature_Engg_TestData_v6.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "ae0794d8"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ae0794d8"
      },
      "source": [
        "### Importing all the required libraries"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PZsjSal4gCyO",
        "outputId": "a4b4c4d8-0416-44fc-c7e1-5e41a526e3b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "!pip install Afinn"
      ],
      "metadata": {
        "id": "ZNT5r2DMns_e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "305262f3",
        "outputId": "cd417ded-eaf6-489f-cce3-bc84b53c724c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"
          ]
        }
      ],
      "source": [
        "import math\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import nltk\n",
        "import string\n",
        "import re\n",
        "import scipy\n",
        "from nltk.tokenize import word_tokenize, sent_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from collections import Counter\n",
        "from transformers import pipeline\n",
        "from textblob import TextBlob\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_selection import SelectKBest, f_regression\n",
        "from afinn import Afinn\n",
        "import matplotlib.pyplot as plt\n",
        "import pylab as plot\n",
        "import seaborn as sn\n",
        "import joblib\n",
        "import warnings\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "import seaborn as sns\n",
        "from tqdm import tqdm_notebook, tqdm\n",
        "warnings.filterwarnings('ignore')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('vader_lexicon')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('omw-1.4')\n",
        "import gc\n",
        "from scipy.sparse import hstack\n",
        "from nltk.corpus import stopwords"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Dja4k5jCAvog"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data vectorization and other pre processing for Test Data"
      ],
      "metadata": {
        "id": "nThYQyuEogPi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_test= pd.read_csv('/content/gdrive/MyDrive/test_stg2.tsv', sep='\\t')"
      ],
      "metadata": {
        "id": "kqnmUWBsorBl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test.fillna('', inplace=True)\n",
        "df_test['item_description']  = df_test['item_description'].str.replace('^no description yet$', '', regex=True)"
      ],
      "metadata": {
        "id": "azZAqFonorGg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test['name'] = df_test['name'] + \" \" + df_test['brand_name']\n",
        "df_test['text'] = df_test['item_description'] + \" \" + df_test['name'] + \" \" + df_test['category_name']"
      ],
      "metadata": {
        "id": "EI29tEI6orVM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_train = joblib.load('/content/gdrive/MyDrive/binary_files/df_train_26Jan23_v1.joblib')"
      ],
      "metadata": {
        "id": "PeZjvCGVCsL7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Ref: AAIC Notebook for Donors' Choose\n",
        "def decontracted(phrase):\n",
        "    '''\n",
        "    Description -> Replaces the short form words to their decontracted form such as won't to will not, \n",
        "                   this is done to make the text data uniform using regex commands.\n",
        "    '''\n",
        "    phrase = re.sub(r\"aren\\'t\", \"are not\", phrase)\n",
        "    phrase = re.sub(r\"didn\\'t\", \"did not\", phrase)\n",
        "    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n",
        "    phrase = re.sub(r\"couldn\\'t\", \"could not\", phrase)\n",
        "    phrase = re.sub(r\"won\\'t\", \"would not\", phrase)\n",
        "    phrase = re.sub(r\"wouldn\\'t\", \"would not\", phrase)\n",
        "    phrase = re.sub(r\"haven\\'t\", \"have not\", phrase)\n",
        "    phrase = re.sub(r\"shouldn\\'t\", \"should not\", phrase)\n",
        "    phrase = re.sub(r\"doesn\\'t\", \"does not\", phrase)\n",
        "    phrase = re.sub(r\"don\\'t\", \"do not\", phrase)\n",
        "    phrase = re.sub(r\"didn\\'t\", \"did not\", phrase)\n",
        "    phrase = re.sub(r\"mustn\\'t\", \"must not\", phrase)\n",
        "    phrase = re.sub(r\"needn\\'t\", \"need not\", phrase)\n",
        "    \n",
        "    return phrase"
      ],
      "metadata": {
        "id": "TjH73lzzoraE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test['name'] = df_test['name'].apply(lambda x : decontracted(x))\n",
        "df_test['text'] = df_test['text'].apply(lambda x : decontracted(x))"
      ],
      "metadata": {
        "id": "utjM4vHqorel"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Reference: https://www.geeksforgeeks.org/python-lemmatization-with-nltk/\n",
        "#Lemmatization is the process of grouping together the different inflected forms of a word so they can be analyzed as a single item\n",
        "def perform_lemma(sent,all_stopwords):\n",
        "    '''\n",
        "    Description -> Applying wordnet lemmatizer on the input sentence and returning the nearest base word/sentence.\n",
        "    '''    \n",
        "    sent_list = sent.split()\n",
        "    lem = WordNetLemmatizer()\n",
        "    text = [lem.lemmatize(word) for word in sent_list if word not in all_stopwords]\n",
        "    sent = \" \".join(text)\n",
        "    return sent"
      ],
      "metadata": {
        "id": "eLPKQk-1pfOu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "regex_special_chars = re.compile('[^A-Za-z0-9.]+')\n",
        "regex_decimal_digits = re.compile('(?<!\\d)\\.(?!\\d)')\n",
        "regex_white_space = re.compile(r'\\s+')\n",
        "\n",
        "# Since considering positive and negative emotion of buyer, so considering negative words also\n",
        "# Reference : https://stackabuse.com/removing-stop-words-from-strings-in-python/\n",
        "# all_stopwords = stopwords.words('english')\n",
        "# all_stopwords.remove('not')\n",
        "# all_stopwords.remove('nor')\n",
        "# all_stopwords.remove('no')\n",
        "#optimized way\n",
        "all_stopwords = set(stopwords.words(\"english\")) - {\"no\", \"nor\", \"not\"} \n",
        "\n",
        "\n",
        "def process_text_data(sent):\n",
        "    '''\n",
        "    Description -> Calling the demojify and lemmatization functions one by one with data as the input\n",
        "                  and returning the final preprocessed data.\n",
        "    '''\n",
        "    \n",
        "    #Removing new line, carriage return, double quotes\n",
        "    sent = sent.replace('\\\\r', ' ')\n",
        "    #sent = sent.replace('\\\\\"', ' ')\n",
        "    sent = sent.replace('\\\\n', ' ')\n",
        "    #remove enojis  https://stackoverflow.com/questions/33404752/removing-emojis-from-a-string-in-python\n",
        "    #sent = deEmojify(sent)\n",
        "\n",
        "    #removing all special charecter except mentioned in regex strings    \n",
        "    sent = regex_special_chars.sub(' ', sent)\n",
        "\n",
        "    #removing all digit mentioned in regex strings\n",
        "    sent = regex_decimal_digits.sub(' ', sent)\n",
        "    \n",
        "    # removing all white spaces  https://bobbyhadz.com/blog/python-remove-whitespace-regex      \n",
        "    sent = regex_white_space.sub(' ', sent)\n",
        "    #sent = re.sub(r'^\\s+', '', sent, flags=re.MULTILINE)\n",
        "\n",
        "    #removing end space and converting to lower\n",
        "    sent = sent.strip().lower()\n",
        "\n",
        "    # only take the words which are not stop words\n",
        "    sent = perform_lemma(sent, all_stopwords)    \n",
        "    return sent"
      ],
      "metadata": {
        "id": "TTIaaQFYpfYg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_test['name'] = df_test['name'].apply(lambda x : process_text_data(x))\n",
        "df_test['text'] = df_test['text'].apply(lambda x : process_text_data(x))"
      ],
      "metadata": {
        "id": "LjUlqrgypfci"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def text_encoder_testdata(train_data,test_data,type,params):\n",
        "    '''\n",
        "    Description -> Encoding different types of input text data according to its requirements using Countvectorizer\n",
        "                   & Tfidfvectorizer and returning the transformed data as output\n",
        "    '''\n",
        "    if(type == \"BOW\"):\n",
        "        vectorizer = CountVectorizer(ngram_range = params[0],min_df = params[1],max_df = params[2],max_features = params[3])\n",
        "    elif(type == \"TFIDF\"):\n",
        "        N_GRAMS =params\n",
        "        vectorizer = vectorizer = TfidfVectorizer(max_features = 100000,\n",
        "                                 ngram_range = (1, N_GRAMS),\n",
        "                                 strip_accents = 'unicode',\n",
        "                                 analyzer = 'word',\n",
        "                                 token_pattern = r'\\w+')\n",
        "    elif(type==\"CNTVECT\"):\n",
        "        vectorizer = CountVectorizer(vocabulary=params, lowercase=False, binary=True)\n",
        "\n",
        "    #Vectorize on train data and transform on test data\n",
        "    vectorizer.fit(train_data)    \n",
        "    test_transform = vectorizer.transform(test_data)\n",
        "\n",
        "    if (type == \"BOW\"):\n",
        "        return test_transform,  ''\n",
        "    elif (type == \"CNTVECT\"):\n",
        "        return test_transform, ''\n",
        "    elif (type == \"TFIDF\"):\n",
        "        feat_names = vectorizer.get_feature_names_out()\n",
        "        del vectorizer\n",
        "        gc.collect()\n",
        "        return test_transform, feat_names"
      ],
      "metadata": {
        "id": "ZCZOYhxNpfgy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_test_name,_ = text_encoder_testdata(df_train['name'],df_test['name'],\"TFIDF\", 1)\n",
        "X_test_text,_ = text_encoder_testdata(df_train['text'],df_test['text'],\"TFIDF\", 2)"
      ],
      "metadata": {
        "id": "ThxTg31WpflJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_test_name.shape,X_test_text.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5iFgGCUIpfoy",
        "outputId": "1759e1f7-8c93-420e-8a96-3da1d94711f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((3460725, 85394), (3460725, 100000))"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def one_hot_encoder_testdata(train_data,test_data):\n",
        "    ohe_encoder = OneHotEncoder()\n",
        "    #Vectorize on train data and transform on test data\n",
        "    ohe_encoder.fit(train_data)\n",
        "    test_ohe = ohe_encoder.transform(test_data)\n",
        "    return test_ohe"
      ],
      "metadata": {
        "id": "4tMhhL17pfsw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_test_shipping = one_hot_encoder_testdata(np.reshape(df_train['shipping'].values, (-1, 1)),np.reshape(df_test['shipping'].values, (-1, 1)))\n",
        "X_test_item_condition = one_hot_encoder_testdata(np.reshape(df_train['item_condition_id'].values, (-1, 1)),np.reshape(df_test['item_condition_id'].values, (-1, 1)))"
      ],
      "metadata": {
        "id": "vGuvD0KGor4p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "testframe = hstack((X_test_name,\n",
        "                 X_test_text,\n",
        "                 X_test_shipping,\n",
        "                 X_test_item_condition)).tocsr().astype('float32')"
      ],
      "metadata": {
        "id": "AwS7pJCmuuci"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "testframe.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fg3fkG_quuiG",
        "outputId": "213f2a09-1e6a-4422-a0ca-8781a03ad2eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3460725, 185401)"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "joblib.dump(testframe, '/content/gdrive/MyDrive/binary_files/testframe_26Jan23_v1.joblib')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x5Gg_eFWAxAA",
        "outputId": "1b026cba-2e14-494d-96b4-81cbf351f0a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/content/gdrive/MyDrive/binary_files/testframe_26Jan23_v1.joblib']"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "joblib.dump(df_test, '/content/gdrive/MyDrive/binary_files/df_test_26Jan23_v1.joblib')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "im_SHSkAGbdA",
        "outputId": "e1c680d2-dce6-4b6d-b5c1-a933acf5c1bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/content/gdrive/MyDrive/binary_files/df_test_26Jan23_v1.joblib']"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    }
  ]
}