advanced_features_with_other_algorithms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea453808",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "79179af9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>...</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>token_set_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>95241</td>\n",
       "      <td>189061</td>\n",
       "      <td>189062</td>\n",
       "      <td>is michelle obama a transgender person</td>\n",
       "      <td>what are the most attractive photos of michell...</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>53</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0.222220</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>7.5</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>37</td>\n",
       "      <td>55</td>\n",
       "      <td>53</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>185512</td>\n",
       "      <td>366337</td>\n",
       "      <td>366338</td>\n",
       "      <td>what is the main reason behind 500   1k rs not...</td>\n",
       "      <td>why are 500 and 1k notes being banned in india</td>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>46</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0.208332</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.234043</td>\n",
       "      <td>43</td>\n",
       "      <td>67</td>\n",
       "      <td>44</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>178179</td>\n",
       "      <td>351959</td>\n",
       "      <td>351960</td>\n",
       "      <td>why england is not returning kohinoor diamond ...</td>\n",
       "      <td>why will not britain return the koh i noor dia...</td>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0.416663</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.216667</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>73</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>366934</td>\n",
       "      <td>718195</td>\n",
       "      <td>718196</td>\n",
       "      <td>what will a career in nabard be like  how is i...</td>\n",
       "      <td>is an engineer eligible for the nabard assista...</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>61</td>\n",
       "      <td>25</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086956</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>16.5</td>\n",
       "      <td>0.145161</td>\n",
       "      <td>36</td>\n",
       "      <td>43</td>\n",
       "      <td>34</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>215887</td>\n",
       "      <td>425692</td>\n",
       "      <td>425693</td>\n",
       "      <td>which mba college should i opt for in asia wit...</td>\n",
       "      <td>which colleges can i get into with 770 gmat score</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>49</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333331</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.220000</td>\n",
       "      <td>66</td>\n",
       "      <td>63</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>185158</td>\n",
       "      <td>365646</td>\n",
       "      <td>365647</td>\n",
       "      <td>what is the best place to visit for summer</td>\n",
       "      <td>what is the best place to visit in summer</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>41</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0.888879</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.761905</td>\n",
       "      <td>94</td>\n",
       "      <td>93</td>\n",
       "      <td>94</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>277866</td>\n",
       "      <td>546311</td>\n",
       "      <td>546312</td>\n",
       "      <td>how can i get over this girl</td>\n",
       "      <td>how do i get over a girl i cannot be with</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0.454541</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>61</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>387648</td>\n",
       "      <td>757859</td>\n",
       "      <td>757860</td>\n",
       "      <td>how do i become a graphics designer</td>\n",
       "      <td>should i become a graphic designer</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0.542857</td>\n",
       "      <td>90</td>\n",
       "      <td>88</td>\n",
       "      <td>81</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>280197</td>\n",
       "      <td>550837</td>\n",
       "      <td>550838</td>\n",
       "      <td>how do i bake a cake in a microwave oven</td>\n",
       "      <td>how do you bake cakes in a convection oven</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>42</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0.599994</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9.5</td>\n",
       "      <td>0.170732</td>\n",
       "      <td>78</td>\n",
       "      <td>75</td>\n",
       "      <td>63</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>293568</td>\n",
       "      <td>576715</td>\n",
       "      <td>576716</td>\n",
       "      <td>what can i do professionally to help people an...</td>\n",
       "      <td>why do people want to live a very long life</td>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>43</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>42</td>\n",
       "      <td>51</td>\n",
       "      <td>46</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30000 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id    qid1    qid2  \\\n",
       "0       95241  189061  189062   \n",
       "1      185512  366337  366338   \n",
       "2      178179  351959  351960   \n",
       "3      366934  718195  718196   \n",
       "4      215887  425692  425693   \n",
       "...       ...     ...     ...   \n",
       "29995  185158  365646  365647   \n",
       "29996  277866  546311  546312   \n",
       "29997  387648  757859  757860   \n",
       "29998  280197  550837  550838   \n",
       "29999  293568  576715  576716   \n",
       "\n",
       "                                               question1  \\\n",
       "0                 is michelle obama a transgender person   \n",
       "1      what is the main reason behind 500   1k rs not...   \n",
       "2      why england is not returning kohinoor diamond ...   \n",
       "3      what will a career in nabard be like  how is i...   \n",
       "4      which mba college should i opt for in asia wit...   \n",
       "...                                                  ...   \n",
       "29995         what is the best place to visit for summer   \n",
       "29996                       how can i get over this girl   \n",
       "29997                how do i become a graphics designer   \n",
       "29998           how do i bake a cake in a microwave oven   \n",
       "29999  what can i do professionally to help people an...   \n",
       "\n",
       "                                               question2  is_duplicate  \\\n",
       "0      what are the most attractive photos of michell...             0   \n",
       "1         why are 500 and 1k notes being banned in india             1   \n",
       "2      why will not britain return the koh i noor dia...             1   \n",
       "3      is an engineer eligible for the nabard assista...             0   \n",
       "4      which colleges can i get into with 770 gmat score             0   \n",
       "...                                                  ...           ...   \n",
       "29995          what is the best place to visit in summer             1   \n",
       "29996          how do i get over a girl i cannot be with             1   \n",
       "29997                 should i become a graphic designer             0   \n",
       "29998         how do you bake cakes in a convection oven             0   \n",
       "29999        why do people want to live a very long life             0   \n",
       "\n",
       "       q1_len  q2_len  q1_num_words  q2_num_words  ...   ctc_max  \\\n",
       "0          38      53             6             9  ...  0.222220   \n",
       "1         132      46            28            10  ...  0.208332   \n",
       "2          59      59            10            12  ...  0.416663   \n",
       "3         120      61            25            10  ...  0.086956   \n",
       "4          67      49            15            10  ...  0.333331   \n",
       "...       ...     ...           ...           ...  ...       ...   \n",
       "29995      42      41             9             9  ...  0.888879   \n",
       "29996      28      41             7            11  ...  0.454541   \n",
       "29997      35      34             7             6  ...  0.571420   \n",
       "29998      40      42            10             9  ...  0.599994   \n",
       "29999      70      43            14            10  ...  0.285712   \n",
       "\n",
       "       last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "0                 0              1             3       7.5   \n",
       "1                 1              1            14      17.0   \n",
       "2                 1              1             2      11.0   \n",
       "3                 0              1            13      16.5   \n",
       "4                 0              1             5      12.5   \n",
       "...             ...            ...           ...       ...   \n",
       "29995             1              1             0       9.0   \n",
       "29996             0              1             4       9.0   \n",
       "29997             1              1             1       6.5   \n",
       "29998             1              1             1       9.5   \n",
       "29999             0              1             4      12.0   \n",
       "\n",
       "       longest_substr_ratio  fuzz_ratio  fuzz_partial_ratio  token_sort_ratio  \\\n",
       "0                  0.384615          37                  55                53   \n",
       "1                  0.234043          43                  67                44   \n",
       "2                  0.216667          71                  71                73   \n",
       "3                  0.145161          36                  43                34   \n",
       "4                  0.220000          66                  63                71   \n",
       "...                     ...         ...                 ...               ...   \n",
       "29995              0.761905          94                  93                94   \n",
       "29996              0.413793          61                  75                75   \n",
       "29997              0.542857          90                  88                81   \n",
       "29998              0.170732          78                  75                63   \n",
       "29999              0.181818          42                  51                46   \n",
       "\n",
       "       token_set_ratio  \n",
       "0                   54  \n",
       "1                   63  \n",
       "2                   78  \n",
       "3                   35  \n",
       "4                   71  \n",
       "...                ...  \n",
       "29995               96  \n",
       "29996               81  \n",
       "29997               87  \n",
       "29998               78  \n",
       "29999               51  \n",
       "\n",
       "[30000 rows x 28 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('30000_advanced_features_dataset.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "65acdcc2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Here ,was number 111'"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['question1'][3090]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "f8f4d012",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                      0\n",
       "qid1                    0\n",
       "qid2                    0\n",
       "question1               0\n",
       "question2               1\n",
       "is_duplicate            0\n",
       "q1_len                  0\n",
       "q2_len                  0\n",
       "q1_num_words            0\n",
       "q2_num_words            0\n",
       "word_common             0\n",
       "word_total              0\n",
       "word_share              0\n",
       "cwc_min                 0\n",
       "cwc_max                 0\n",
       "csc_min                 0\n",
       "csc_max                 0\n",
       "ctc_min                 0\n",
       "ctc_max                 0\n",
       "last_word_eq            0\n",
       "first_word_eq           0\n",
       "abs_len_diff            0\n",
       "mean_len                0\n",
       "longest_substr_ratio    0\n",
       "fuzz_ratio              0\n",
       "fuzz_partial_ratio      0\n",
       "token_sort_ratio        0\n",
       "token_set_ratio         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "9e990b1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df= df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "20d0549b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ques_df = df[['question1','question2']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "b31a26b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>token_set_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>53</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>...</td>\n",
       "      <td>0.222220</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>7.5</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>37</td>\n",
       "      <td>55</td>\n",
       "      <td>53</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>46</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.208332</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.234043</td>\n",
       "      <td>43</td>\n",
       "      <td>67</td>\n",
       "      <td>44</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>...</td>\n",
       "      <td>0.416663</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.216667</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>73</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>61</td>\n",
       "      <td>25</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.099999</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086956</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>16.5</td>\n",
       "      <td>0.145161</td>\n",
       "      <td>36</td>\n",
       "      <td>43</td>\n",
       "      <td>34</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>49</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333331</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.220000</td>\n",
       "      <td>66</td>\n",
       "      <td>63</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>41</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>...</td>\n",
       "      <td>0.888879</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.761905</td>\n",
       "      <td>94</td>\n",
       "      <td>93</td>\n",
       "      <td>94</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.454541</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>61</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0.542857</td>\n",
       "      <td>90</td>\n",
       "      <td>88</td>\n",
       "      <td>81</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>42</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>...</td>\n",
       "      <td>0.599994</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9.5</td>\n",
       "      <td>0.170732</td>\n",
       "      <td>78</td>\n",
       "      <td>75</td>\n",
       "      <td>63</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>43</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>...</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>42</td>\n",
       "      <td>51</td>\n",
       "      <td>46</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "0                 0      38      53             6             9            2   \n",
       "1                 1     132      46            28            10            5   \n",
       "2                 1      59      59            10            12            5   \n",
       "3                 0     120      61            25            10            2   \n",
       "4                 0      67      49            15            10            5   \n",
       "...             ...     ...     ...           ...           ...          ...   \n",
       "29995             1      42      41             9             9            8   \n",
       "29996             1      28      41             7            11            5   \n",
       "29997             0      35      34             7             6            4   \n",
       "29998             0      40      42            10             9            6   \n",
       "29999             0      70      43            14            10            4   \n",
       "\n",
       "       word_total  word_share   cwc_min   cwc_max  ...   ctc_max  \\\n",
       "0              13        0.15  0.499987  0.499987  ...  0.222220   \n",
       "1              28        0.18  0.799984  0.285712  ...  0.208332   \n",
       "2              17        0.29  0.333328  0.333328  ...  0.416663   \n",
       "3              29        0.07  0.166664  0.099999  ...  0.086956   \n",
       "4              20        0.25  0.399992  0.285710  ...  0.333331   \n",
       "...           ...         ...       ...       ...  ...       ...   \n",
       "29995          10        0.80  0.999975  0.999975  ...  0.888879   \n",
       "29996          12        0.42  0.999950  0.666644  ...  0.454541   \n",
       "29997           9        0.44  0.666644  0.666644  ...  0.571420   \n",
       "29998          12        0.50  0.499987  0.499987  ...  0.599994   \n",
       "29999          20        0.20  0.199996  0.199996  ...  0.285712   \n",
       "\n",
       "       last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "0                 0              1             3       7.5   \n",
       "1                 1              1            14      17.0   \n",
       "2                 1              1             2      11.0   \n",
       "3                 0              1            13      16.5   \n",
       "4                 0              1             5      12.5   \n",
       "...             ...            ...           ...       ...   \n",
       "29995             1              1             0       9.0   \n",
       "29996             0              1             4       9.0   \n",
       "29997             1              1             1       6.5   \n",
       "29998             1              1             1       9.5   \n",
       "29999             0              1             4      12.0   \n",
       "\n",
       "       longest_substr_ratio  fuzz_ratio  fuzz_partial_ratio  token_sort_ratio  \\\n",
       "0                  0.384615          37                  55                53   \n",
       "1                  0.234043          43                  67                44   \n",
       "2                  0.216667          71                  71                73   \n",
       "3                  0.145161          36                  43                34   \n",
       "4                  0.220000          66                  63                71   \n",
       "...                     ...         ...                 ...               ...   \n",
       "29995              0.761905          94                  93                94   \n",
       "29996              0.413793          61                  75                75   \n",
       "29997              0.542857          90                  88                81   \n",
       "29998              0.170732          78                  75                63   \n",
       "29999              0.181818          42                  51                46   \n",
       "\n",
       "       token_set_ratio  \n",
       "0                   54  \n",
       "1                   63  \n",
       "2                   78  \n",
       "3                   35  \n",
       "4                   71  \n",
       "...                ...  \n",
       "29995               96  \n",
       "29996               81  \n",
       "29997               87  \n",
       "29998               78  \n",
       "29999               51  \n",
       "\n",
       "[29999 rows x 23 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = df.drop(columns=['id','qid1','qid2','question1','question2'])\n",
    "final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cef3eb56",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "questions = list(ques_df['question1']) + list(ques_df['question2'])\n",
    "cv = CountVectorizer(max_features=2000, ngram_range =(1,2))\n",
    "q1_arr ,q2_arr = np.vsplit(cv.fit_transform(questions).toarray(),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0867c44",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f7035a81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1990</th>\n",
       "      <th>1991</th>\n",
       "      <th>1992</th>\n",
       "      <th>1993</th>\n",
       "      <th>1994</th>\n",
       "      <th>1995</th>\n",
       "      <th>1996</th>\n",
       "      <th>1997</th>\n",
       "      <th>1998</th>\n",
       "      <th>1999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 4000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0     1     2     3     4     5     6     7     8     9     ...  1990  \\\n",
       "0         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "1         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "2         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "3         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "4         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "...     ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  ...   ...   \n",
       "29995     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "29996     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "29997     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "29998     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "29999     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "\n",
       "       1991  1992  1993  1994  1995  1996  1997  1998  1999  \n",
       "0         0     0     0     0     0     0     0     0     0  \n",
       "1         0     0     0     0     0     0     0     0     0  \n",
       "2         0     0     0     0     0     0     0     0     0  \n",
       "3         0     0     0     0     0     0     0     0     0  \n",
       "4         0     0     0     0     0     0     0     0     0  \n",
       "...     ...   ...   ...   ...   ...   ...   ...   ...   ...  \n",
       "29995     0     0     0     0     0     0     0     0     0  \n",
       "29996     0     0     0     0     0     0     0     0     0  \n",
       "29997     0     0     0     0     0     0     0     0     0  \n",
       "29998     0     0     0     0     0     0     0     0     0  \n",
       "29999     0     0     0     0     0     0     0     0     0  \n",
       "\n",
       "[29999 rows x 4000 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df1 = pd.DataFrame(q1_arr, index= ques_df.index)\n",
    "temp_df2 = pd.DataFrame(q2_arr, index= ques_df.index)\n",
    "temp_df = pd.concat([temp_df1,temp_df2] , axis=1)\n",
    "temp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6648252c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>1990</th>\n",
       "      <th>1991</th>\n",
       "      <th>1992</th>\n",
       "      <th>1993</th>\n",
       "      <th>1994</th>\n",
       "      <th>1995</th>\n",
       "      <th>1996</th>\n",
       "      <th>1997</th>\n",
       "      <th>1998</th>\n",
       "      <th>1999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>53</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>46</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>61</td>\n",
       "      <td>25</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.099999</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>49</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>41</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>42</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>43</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 4023 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "0                 0      38      53             6             9            2   \n",
       "1                 1     132      46            28            10            5   \n",
       "2                 1      59      59            10            12            5   \n",
       "3                 0     120      61            25            10            2   \n",
       "4                 0      67      49            15            10            5   \n",
       "...             ...     ...     ...           ...           ...          ...   \n",
       "29995             1      42      41             9             9            8   \n",
       "29996             1      28      41             7            11            5   \n",
       "29997             0      35      34             7             6            4   \n",
       "29998             0      40      42            10             9            6   \n",
       "29999             0      70      43            14            10            4   \n",
       "\n",
       "       word_total  word_share   cwc_min   cwc_max  ...  1990  1991  1992  \\\n",
       "0              13        0.15  0.499988  0.499988  ...     0     0     0   \n",
       "1              28        0.18  0.799984  0.285712  ...     0     0     0   \n",
       "2              17        0.29  0.333328  0.333328  ...     0     0     0   \n",
       "3              29        0.07  0.166664  0.099999  ...     0     0     0   \n",
       "4              20        0.25  0.399992  0.285710  ...     0     0     0   \n",
       "...           ...         ...       ...       ...  ...   ...   ...   ...   \n",
       "29995          10        0.80  0.999975  0.999975  ...     0     0     0   \n",
       "29996          12        0.42  0.999950  0.666644  ...     0     0     0   \n",
       "29997           9        0.44  0.666644  0.666644  ...     0     0     0   \n",
       "29998          12        0.50  0.499988  0.499988  ...     0     0     0   \n",
       "29999          20        0.20  0.199996  0.199996  ...     0     0     0   \n",
       "\n",
       "       1993  1994  1995  1996  1997  1998  1999  \n",
       "0         0     0     0     0     0     0     0  \n",
       "1         0     0     0     0     0     0     0  \n",
       "2         0     0     0     0     0     0     0  \n",
       "3         0     0     0     0     0     0     0  \n",
       "4         0     0     0     0     0     0     0  \n",
       "...     ...   ...   ...   ...   ...   ...   ...  \n",
       "29995     0     0     0     0     0     0     0  \n",
       "29996     0     0     0     0     0     0     0  \n",
       "29997     0     0     0     0     0     0     0  \n",
       "29998     0     0     0     0     0     0     0  \n",
       "29999     0     0     0     0     0     0     0  \n",
       "\n",
       "[29999 rows x 4023 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = pd.concat([final_df,temp_df],axis=1)\n",
    "final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "50678818",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(final_df.iloc[:,1:].values,final_df.iloc[:,0].values,test_size=0.2,random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bf735099",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7825"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train,y_train)\n",
    "y_pred = rf.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "de4face1",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv2 = CountVectorizer(max_features=3000, ngram_range =(1,3))\n",
    "q11_arr ,q22_arr = np.vsplit(cv2.fit_transform(questions).toarray(),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f670c67b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>2990</th>\n",
       "      <th>2991</th>\n",
       "      <th>2992</th>\n",
       "      <th>2993</th>\n",
       "      <th>2994</th>\n",
       "      <th>2995</th>\n",
       "      <th>2996</th>\n",
       "      <th>2997</th>\n",
       "      <th>2998</th>\n",
       "      <th>2999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 6000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0     1     2     3     4     5     6     7     8     9     ...  2990  \\\n",
       "0         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "1         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "2         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "3         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "4         0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "...     ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  ...   ...   \n",
       "29995     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "29996     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "29997     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "29998     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "29999     0     0     0     0     0     0     0     0     0     0  ...     0   \n",
       "\n",
       "       2991  2992  2993  2994  2995  2996  2997  2998  2999  \n",
       "0         0     0     0     0     0     0     0     0     0  \n",
       "1         0     0     0     0     0     0     0     0     0  \n",
       "2         0     0     0     0     0     0     0     0     0  \n",
       "3         0     0     0     0     0     0     0     0     0  \n",
       "4         0     0     0     0     0     0     0     0     0  \n",
       "...     ...   ...   ...   ...   ...   ...   ...   ...   ...  \n",
       "29995     0     0     0     0     0     0     0     0     0  \n",
       "29996     0     0     0     0     0     0     0     0     0  \n",
       "29997     0     0     0     0     0     0     0     0     0  \n",
       "29998     0     0     0     0     0     0     0     0     0  \n",
       "29999     0     0     0     0     0     0     0     0     0  \n",
       "\n",
       "[29999 rows x 6000 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_df11 = pd.DataFrame(q11_arr, index= ques_df.index)\n",
    "temp_df22 = pd.DataFrame(q22_arr, index= ques_df.index)\n",
    "temp_df2 = pd.concat([temp_df11,temp_df22] , axis=1)\n",
    "temp_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "293145e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>1990</th>\n",
       "      <th>1991</th>\n",
       "      <th>1992</th>\n",
       "      <th>1993</th>\n",
       "      <th>1994</th>\n",
       "      <th>1995</th>\n",
       "      <th>1996</th>\n",
       "      <th>1997</th>\n",
       "      <th>1998</th>\n",
       "      <th>1999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>53</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>46</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>61</td>\n",
       "      <td>25</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.099999</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>49</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>41</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>42</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>43</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 8023 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "0                 0      38      53             6             9            2   \n",
       "1                 1     132      46            28            10            5   \n",
       "2                 1      59      59            10            12            5   \n",
       "3                 0     120      61            25            10            2   \n",
       "4                 0      67      49            15            10            5   \n",
       "...             ...     ...     ...           ...           ...          ...   \n",
       "29995             1      42      41             9             9            8   \n",
       "29996             1      28      41             7            11            5   \n",
       "29997             0      35      34             7             6            4   \n",
       "29998             0      40      42            10             9            6   \n",
       "29999             0      70      43            14            10            4   \n",
       "\n",
       "       word_total  word_share   cwc_min   cwc_max  ...  1990  1991  1992  \\\n",
       "0              13        0.15  0.499988  0.499988  ...     0     0     0   \n",
       "1              28        0.18  0.799984  0.285712  ...     0     0     0   \n",
       "2              17        0.29  0.333328  0.333328  ...     0     0     0   \n",
       "3              29        0.07  0.166664  0.099999  ...     0     0     0   \n",
       "4              20        0.25  0.399992  0.285710  ...     0     0     0   \n",
       "...           ...         ...       ...       ...  ...   ...   ...   ...   \n",
       "29995          10        0.80  0.999975  0.999975  ...     0     0     0   \n",
       "29996          12        0.42  0.999950  0.666644  ...     0     0     0   \n",
       "29997           9        0.44  0.666644  0.666644  ...     0     0     0   \n",
       "29998          12        0.50  0.499988  0.499988  ...     0     0     0   \n",
       "29999          20        0.20  0.199996  0.199996  ...     0     0     0   \n",
       "\n",
       "       1993  1994  1995  1996  1997  1998  1999  \n",
       "0         0     0     0     0     0     0     0  \n",
       "1         0     0     0     0     0     0     0  \n",
       "2         0     0     0     0     0     0     0  \n",
       "3         0     0     0     0     0     0     0  \n",
       "4         0     0     0     0     0     0     0  \n",
       "...     ...   ...   ...   ...   ...   ...   ...  \n",
       "29995     0     0     0     0     0     0     0  \n",
       "29996     0     0     0     0     0     0     0  \n",
       "29997     0     0     0     0     0     0     0  \n",
       "29998     0     0     0     0     0     0     0  \n",
       "29999     0     0     0     0     0     0     0  \n",
       "\n",
       "[29999 rows x 8023 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df2 = pd.concat([final_df,temp_df],axis=1)\n",
    "final_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "63c674f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(final_df.iloc[:,1:].values,final_df.iloc[:,0].values,test_size=0.2,random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ed7781b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.785"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train,y_train)\n",
    "y_pred = rf.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1d50617d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23999, 22)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "05aee3d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf = TfidfVectorizer(max_features =2000)\n",
    "q1t_arr ,q2t_arr = np.vsplit(tfidf.fit_transform(questions).toarray(),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a0de6923",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1990</th>\n",
       "      <th>1991</th>\n",
       "      <th>1992</th>\n",
       "      <th>1993</th>\n",
       "      <th>1994</th>\n",
       "      <th>1995</th>\n",
       "      <th>1996</th>\n",
       "      <th>1997</th>\n",
       "      <th>1998</th>\n",
       "      <th>1999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.606174</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 4000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0     1     2     3     4     5     6     7     8     9     ...  1990  \\\n",
       "0       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "1       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "2       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "3       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "4       0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "...     ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  ...   ...   \n",
       "29995   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "29996   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "29997   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "29998   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "29999   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  ...   0.0   \n",
       "\n",
       "       1991  1992      1993  1994  1995  1996  1997  1998  1999  \n",
       "0       0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "1       0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2       0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "3       0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4       0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "...     ...   ...       ...   ...   ...   ...   ...   ...   ...  \n",
       "29995   0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "29996   0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "29997   0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "29998   0.0   0.0  0.606174   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "29999   0.0   0.0  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[29999 rows x 4000 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_dft1 = pd.DataFrame(q1t_arr, index= ques_df.index)\n",
    "temp_dft2 = pd.DataFrame(q2t_arr, index= ques_df.index)\n",
    "temp_dft = pd.concat([temp_dft1,temp_dft2] , axis=1)\n",
    "temp_dft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "450553df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>1990</th>\n",
       "      <th>1991</th>\n",
       "      <th>1992</th>\n",
       "      <th>1993</th>\n",
       "      <th>1994</th>\n",
       "      <th>1995</th>\n",
       "      <th>1996</th>\n",
       "      <th>1997</th>\n",
       "      <th>1998</th>\n",
       "      <th>1999</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>53</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>46</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>61</td>\n",
       "      <td>25</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.099999</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>49</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>41</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>42</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>0.499988</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.606174</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>43</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 4023 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "0                 0      38      53             6             9            2   \n",
       "1                 1     132      46            28            10            5   \n",
       "2                 1      59      59            10            12            5   \n",
       "3                 0     120      61            25            10            2   \n",
       "4                 0      67      49            15            10            5   \n",
       "...             ...     ...     ...           ...           ...          ...   \n",
       "29995             1      42      41             9             9            8   \n",
       "29996             1      28      41             7            11            5   \n",
       "29997             0      35      34             7             6            4   \n",
       "29998             0      40      42            10             9            6   \n",
       "29999             0      70      43            14            10            4   \n",
       "\n",
       "       word_total  word_share   cwc_min   cwc_max  ...  1990  1991  1992  \\\n",
       "0              13        0.15  0.499988  0.499988  ...   0.0   0.0   0.0   \n",
       "1              28        0.18  0.799984  0.285712  ...   0.0   0.0   0.0   \n",
       "2              17        0.29  0.333328  0.333328  ...   0.0   0.0   0.0   \n",
       "3              29        0.07  0.166664  0.099999  ...   0.0   0.0   0.0   \n",
       "4              20        0.25  0.399992  0.285710  ...   0.0   0.0   0.0   \n",
       "...           ...         ...       ...       ...  ...   ...   ...   ...   \n",
       "29995          10        0.80  0.999975  0.999975  ...   0.0   0.0   0.0   \n",
       "29996          12        0.42  0.999950  0.666644  ...   0.0   0.0   0.0   \n",
       "29997           9        0.44  0.666644  0.666644  ...   0.0   0.0   0.0   \n",
       "29998          12        0.50  0.499988  0.499988  ...   0.0   0.0   0.0   \n",
       "29999          20        0.20  0.199996  0.199996  ...   0.0   0.0   0.0   \n",
       "\n",
       "           1993  1994  1995  1996  1997  1998  1999  \n",
       "0      0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "1      0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2      0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "3      0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4      0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "...         ...   ...   ...   ...   ...   ...   ...  \n",
       "29995  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "29996  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "29997  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "29998  0.606174   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "29999  0.000000   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[29999 rows x 4023 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dft = pd.concat([final_df, temp_dft],axis=1)\n",
    "final_dft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "74d7ebe5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(final_dft.iloc[:,1:].values,final_dft.iloc[:,0].values,test_size=0.2,random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "33a323d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7808333333333334"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train,y_train)\n",
    "y_pred = rf.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "690b4922",
   "metadata": {},
   "source": [
    "# word2vec\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7525dbb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "from gensim.utils import simple_preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "c4b5256a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import sent_tokenize,word_tokenize\n",
    "story =[]\n",
    "questions = list(ques_df['question1']) + list(ques_df['question2'])\n",
    "for doc in questions:\n",
    "    raw_sent = sent_tokenize(doc)\n",
    "    for sent in raw_sent:\n",
    "        story.append(simple_preprocess(sent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "ac1920dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59998"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(story)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "64a38773",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = gensim.models.Word2Vec(window=10,min_count=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "91fb145e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.build_vocab(story)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "f429ca74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2206011, 3134090)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.train(story, total_examples=model.corpus_count,epochs=model.epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "069afb2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def document_vector(doc):\n",
    "    doc = [word for word in doc.split() if word in model.wv.index_to_key]\n",
    "    return np.mean(model.wv[doc],axis=0)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "cea9b7cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.53036714, -0.00831612,  0.224935  ,  0.68127894, -0.5272065 ,\n",
       "       -0.46875465,  0.25847828,  0.4745493 , -0.52870524, -0.44407606,\n",
       "        0.11851392, -1.019911  , -0.25301647,  0.18891788, -0.30394572,\n",
       "       -0.37159848,  0.09993097, -0.2449909 , -0.25924554, -0.84090537,\n",
       "        0.16450483, -0.02330386,  0.36032605, -0.00967271, -0.25259045,\n",
       "        0.38972867, -0.36863887,  0.12974192, -0.07306664, -0.21945363,\n",
       "        0.15148488,  0.49293563,  0.17886607, -0.41513625,  0.10831378,\n",
       "        0.56523055, -0.01677633, -0.6359643 , -0.3772881 , -1.0657269 ,\n",
       "        0.5214727 , -0.77377707, -0.37750646, -0.16166316,  0.46024156,\n",
       "       -0.4174148 , -0.42098084, -0.65245646,  0.46530032,  0.16680218,\n",
       "       -0.11668835, -0.29543188,  0.10639019, -0.15502839, -0.08731142,\n",
       "        0.07736634,  0.1234901 , -0.6113042 , -0.17926818,  0.06657968,\n",
       "       -0.06332939, -0.10804226,  0.3752245 ,  0.00769631, -0.5144415 ,\n",
       "       -0.14394169, -0.19705369,  0.6828351 , -0.5115636 ,  0.3968042 ,\n",
       "       -0.299499  ,  0.269716  , -0.05509895,  0.47267935,  0.7789815 ,\n",
       "        0.36614552, -0.12012101, -0.02864951, -0.03731629, -0.06033126,\n",
       "       -0.53805846, -0.04415793, -0.35280278,  0.85077715, -0.90773004,\n",
       "       -0.31722316,  0.58514535,  0.1882265 ,  0.4803339 ,  0.75597507,\n",
       "       -0.13359596,  0.22780927,  0.13665244,  0.26386115,  0.6936385 ,\n",
       "        0.5968791 ,  0.34104672, -0.22298284, -0.24004741,  0.3845602 ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_vector(ques_df['question1'].values[29998])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "aa6f2fa9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29999/29999 [00:29<00:00, 1014.86it/s]\n"
     ]
    }
   ],
   "source": [
    "x=[]\n",
    "from tqdm import tqdm\n",
    "for doc in tqdm(ques_df['question1'].values):\n",
    "    x.append(document_vector(doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "5849f0e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29999/29999 [00:32<00:00, 911.16it/s] \n"
     ]
    }
   ],
   "source": [
    "y=[]\n",
    "from tqdm import tqdm\n",
    "for doc in tqdm(ques_df['question2'].values):\n",
    "    y.append(document_vector(doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "36da7ffa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'111'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ques_df['question1'][3090]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "861b0e3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array(x)\n",
    "y= np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "834adf95",
   "metadata": {},
   "outputs": [],
   "source": [
    "x= x.astype(np.float64)\n",
    "y= y.astype(np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "4e9810c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>is michelle obama a transgender person</td>\n",
       "      <td>what are the most attractive photos of michell...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>what is the main reason behind 500   1k rs not...</td>\n",
       "      <td>why are 500 and 1k notes being banned in india</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>why england is not returning kohinoor diamond ...</td>\n",
       "      <td>why will not britain return the koh i noor dia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>what will a career in nabard be like  how is i...</td>\n",
       "      <td>is an engineer eligible for the nabard assista...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>which mba college should i opt for in asia wit...</td>\n",
       "      <td>which colleges can i get into with 770 gmat score</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>what is the best place to visit for summer</td>\n",
       "      <td>what is the best place to visit in summer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>how can i get over this girl</td>\n",
       "      <td>how do i get over a girl i cannot be with</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>how do i become a graphics designer</td>\n",
       "      <td>should i become a graphic designer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>how do i bake a cake in a microwave oven</td>\n",
       "      <td>how do you bake cakes in a convection oven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>what can i do professionally to help people an...</td>\n",
       "      <td>why do people want to live a very long life</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               question1  \\\n",
       "0                 is michelle obama a transgender person   \n",
       "1      what is the main reason behind 500   1k rs not...   \n",
       "2      why england is not returning kohinoor diamond ...   \n",
       "3      what will a career in nabard be like  how is i...   \n",
       "4      which mba college should i opt for in asia wit...   \n",
       "...                                                  ...   \n",
       "29995         what is the best place to visit for summer   \n",
       "29996                       how can i get over this girl   \n",
       "29997                how do i become a graphics designer   \n",
       "29998           how do i bake a cake in a microwave oven   \n",
       "29999  what can i do professionally to help people an...   \n",
       "\n",
       "                                               question2  \n",
       "0      what are the most attractive photos of michell...  \n",
       "1         why are 500 and 1k notes being banned in india  \n",
       "2      why will not britain return the koh i noor dia...  \n",
       "3      is an engineer eligible for the nabard assista...  \n",
       "4      which colleges can i get into with 770 gmat score  \n",
       "...                                                  ...  \n",
       "29995          what is the best place to visit in summer  \n",
       "29996          how do i get over a girl i cannot be with  \n",
       "29997                 should i become a graphic designer  \n",
       "29998         how do you bake cakes in a convection oven  \n",
       "29999        why do people want to live a very long life  \n",
       "\n",
       "[29999 rows x 2 columns]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ques_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "cbbaa193",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>ctc_max</th>\n",
       "      <th>last_word_eq</th>\n",
       "      <th>first_word_eq</th>\n",
       "      <th>abs_len_diff</th>\n",
       "      <th>mean_len</th>\n",
       "      <th>longest_substr_ratio</th>\n",
       "      <th>fuzz_ratio</th>\n",
       "      <th>fuzz_partial_ratio</th>\n",
       "      <th>token_sort_ratio</th>\n",
       "      <th>token_set_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>53</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>...</td>\n",
       "      <td>0.222220</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>7.5</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>37</td>\n",
       "      <td>55</td>\n",
       "      <td>53</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>46</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>28</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.208332</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.234043</td>\n",
       "      <td>43</td>\n",
       "      <td>67</td>\n",
       "      <td>44</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>17</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>...</td>\n",
       "      <td>0.416663</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.216667</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "      <td>73</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>61</td>\n",
       "      <td>25</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.099999</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086956</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>16.5</td>\n",
       "      <td>0.145161</td>\n",
       "      <td>36</td>\n",
       "      <td>43</td>\n",
       "      <td>34</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>49</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333331</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>12.5</td>\n",
       "      <td>0.220000</td>\n",
       "      <td>66</td>\n",
       "      <td>63</td>\n",
       "      <td>71</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>41</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>...</td>\n",
       "      <td>0.888879</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.761905</td>\n",
       "      <td>94</td>\n",
       "      <td>93</td>\n",
       "      <td>94</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.454541</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>61</td>\n",
       "      <td>75</td>\n",
       "      <td>75</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.571420</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0.542857</td>\n",
       "      <td>90</td>\n",
       "      <td>88</td>\n",
       "      <td>81</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>42</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>...</td>\n",
       "      <td>0.599994</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9.5</td>\n",
       "      <td>0.170732</td>\n",
       "      <td>78</td>\n",
       "      <td>75</td>\n",
       "      <td>63</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>43</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>...</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>42</td>\n",
       "      <td>51</td>\n",
       "      <td>46</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29999 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "0                 0      38      53             6             9            2   \n",
       "1                 1     132      46            28            10            5   \n",
       "2                 1      59      59            10            12            5   \n",
       "3                 0     120      61            25            10            2   \n",
       "4                 0      67      49            15            10            5   \n",
       "...             ...     ...     ...           ...           ...          ...   \n",
       "29995             1      42      41             9             9            8   \n",
       "29996             1      28      41             7            11            5   \n",
       "29997             0      35      34             7             6            4   \n",
       "29998             0      40      42            10             9            6   \n",
       "29999             0      70      43            14            10            4   \n",
       "\n",
       "       word_total  word_share   cwc_min   cwc_max  ...   ctc_max  \\\n",
       "0              13        0.15  0.499987  0.499987  ...  0.222220   \n",
       "1              28        0.18  0.799984  0.285712  ...  0.208332   \n",
       "2              17        0.29  0.333328  0.333328  ...  0.416663   \n",
       "3              29        0.07  0.166664  0.099999  ...  0.086956   \n",
       "4              20        0.25  0.399992  0.285710  ...  0.333331   \n",
       "...           ...         ...       ...       ...  ...       ...   \n",
       "29995          10        0.80  0.999975  0.999975  ...  0.888879   \n",
       "29996          12        0.42  0.999950  0.666644  ...  0.454541   \n",
       "29997           9        0.44  0.666644  0.666644  ...  0.571420   \n",
       "29998          12        0.50  0.499987  0.499987  ...  0.599994   \n",
       "29999          20        0.20  0.199996  0.199996  ...  0.285712   \n",
       "\n",
       "       last_word_eq  first_word_eq  abs_len_diff  mean_len  \\\n",
       "0                 0              1             3       7.5   \n",
       "1                 1              1            14      17.0   \n",
       "2                 1              1             2      11.0   \n",
       "3                 0              1            13      16.5   \n",
       "4                 0              1             5      12.5   \n",
       "...             ...            ...           ...       ...   \n",
       "29995             1              1             0       9.0   \n",
       "29996             0              1             4       9.0   \n",
       "29997             1              1             1       6.5   \n",
       "29998             1              1             1       9.5   \n",
       "29999             0              1             4      12.0   \n",
       "\n",
       "       longest_substr_ratio  fuzz_ratio  fuzz_partial_ratio  token_sort_ratio  \\\n",
       "0                  0.384615          37                  55                53   \n",
       "1                  0.234043          43                  67                44   \n",
       "2                  0.216667          71                  71                73   \n",
       "3                  0.145161          36                  43                34   \n",
       "4                  0.220000          66                  63                71   \n",
       "...                     ...         ...                 ...               ...   \n",
       "29995              0.761905          94                  93                94   \n",
       "29996              0.413793          61                  75                75   \n",
       "29997              0.542857          90                  88                81   \n",
       "29998              0.170732          78                  75                63   \n",
       "29999              0.181818          42                  51                46   \n",
       "\n",
       "       token_set_ratio  \n",
       "0                   54  \n",
       "1                   63  \n",
       "2                   78  \n",
       "3                   35  \n",
       "4                   71  \n",
       "...                ...  \n",
       "29995               96  \n",
       "29996               81  \n",
       "29997               87  \n",
       "29998               78  \n",
       "29999               51  \n",
       "\n",
       "[29999 rows x 23 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "367f94b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = pd.DataFrame(x)\n",
    "y_data = pd.DataFrame(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "b0fa58e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.081008</td>\n",
       "      <td>-0.181846</td>\n",
       "      <td>0.043656</td>\n",
       "      <td>-0.284470</td>\n",
       "      <td>0.911652</td>\n",
       "      <td>0.134902</td>\n",
       "      <td>-0.161196</td>\n",
       "      <td>0.086004</td>\n",
       "      <td>-0.066971</td>\n",
       "      <td>0.125468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.216604</td>\n",
       "      <td>0.528166</td>\n",
       "      <td>-0.140012</td>\n",
       "      <td>0.085546</td>\n",
       "      <td>0.737972</td>\n",
       "      <td>0.560770</td>\n",
       "      <td>0.479416</td>\n",
       "      <td>-0.249945</td>\n",
       "      <td>-0.383590</td>\n",
       "      <td>-0.103985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>...</td>\n",
       "      <td>0.261018</td>\n",
       "      <td>0.536788</td>\n",
       "      <td>0.459042</td>\n",
       "      <td>0.177220</td>\n",
       "      <td>0.707681</td>\n",
       "      <td>1.108182</td>\n",
       "      <td>0.201348</td>\n",
       "      <td>-0.544829</td>\n",
       "      <td>-0.785933</td>\n",
       "      <td>0.320921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.099999</td>\n",
       "      <td>...</td>\n",
       "      <td>0.422205</td>\n",
       "      <td>0.293958</td>\n",
       "      <td>-0.082359</td>\n",
       "      <td>-0.168592</td>\n",
       "      <td>0.749170</td>\n",
       "      <td>0.755016</td>\n",
       "      <td>-0.289738</td>\n",
       "      <td>-0.575909</td>\n",
       "      <td>-0.137944</td>\n",
       "      <td>0.169422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>1.045250</td>\n",
       "      <td>0.381819</td>\n",
       "      <td>-0.561544</td>\n",
       "      <td>0.207188</td>\n",
       "      <td>0.252208</td>\n",
       "      <td>-0.123082</td>\n",
       "      <td>-0.323588</td>\n",
       "      <td>-0.564712</td>\n",
       "      <td>-0.078848</td>\n",
       "      <td>0.371620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>1.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>...</td>\n",
       "      <td>0.514043</td>\n",
       "      <td>0.479171</td>\n",
       "      <td>0.025565</td>\n",
       "      <td>0.262315</td>\n",
       "      <td>0.411220</td>\n",
       "      <td>0.730304</td>\n",
       "      <td>0.128723</td>\n",
       "      <td>-0.488675</td>\n",
       "      <td>-0.078426</td>\n",
       "      <td>0.894383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>1.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.303090</td>\n",
       "      <td>0.692995</td>\n",
       "      <td>0.078118</td>\n",
       "      <td>0.703030</td>\n",
       "      <td>0.959234</td>\n",
       "      <td>0.124965</td>\n",
       "      <td>0.052928</td>\n",
       "      <td>-0.480969</td>\n",
       "      <td>-0.189443</td>\n",
       "      <td>0.291008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.362742</td>\n",
       "      <td>0.194083</td>\n",
       "      <td>0.098163</td>\n",
       "      <td>0.109113</td>\n",
       "      <td>0.463521</td>\n",
       "      <td>0.636916</td>\n",
       "      <td>0.282300</td>\n",
       "      <td>-0.471562</td>\n",
       "      <td>-0.080146</td>\n",
       "      <td>0.306487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>...</td>\n",
       "      <td>0.580127</td>\n",
       "      <td>0.441762</td>\n",
       "      <td>0.612558</td>\n",
       "      <td>0.580019</td>\n",
       "      <td>0.535653</td>\n",
       "      <td>1.090930</td>\n",
       "      <td>0.836119</td>\n",
       "      <td>-0.232894</td>\n",
       "      <td>-0.774645</td>\n",
       "      <td>0.807484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30000 rows × 223 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "0               0.0    38.0    53.0           6.0           9.0          2.0   \n",
       "1               1.0   132.0    46.0          28.0          10.0          5.0   \n",
       "2               1.0    59.0    59.0          10.0          12.0          5.0   \n",
       "3               0.0   120.0    61.0          25.0          10.0          2.0   \n",
       "4               0.0    67.0    49.0          15.0          10.0          5.0   \n",
       "...             ...     ...     ...           ...           ...          ...   \n",
       "29995           1.0    42.0    41.0           9.0           9.0          8.0   \n",
       "29996           1.0    28.0    41.0           7.0          11.0          5.0   \n",
       "29997           0.0    35.0    34.0           7.0           6.0          4.0   \n",
       "29998           0.0    40.0    42.0          10.0           9.0          6.0   \n",
       "29999           0.0    70.0    43.0          14.0          10.0          4.0   \n",
       "\n",
       "       word_total  word_share   cwc_min   cwc_max  ...        90        91  \\\n",
       "0            13.0        0.15  0.499987  0.499987  ... -0.081008 -0.181846   \n",
       "1            28.0        0.18  0.799984  0.285712  ...  0.216604  0.528166   \n",
       "2            17.0        0.29  0.333328  0.333328  ...  0.261018  0.536788   \n",
       "3            29.0        0.07  0.166664  0.099999  ...  0.422205  0.293958   \n",
       "4            20.0        0.25  0.399992  0.285710  ...  1.045250  0.381819   \n",
       "...           ...         ...       ...       ...  ...       ...       ...   \n",
       "29995        10.0        0.80  0.999975  0.999975  ...  0.514043  0.479171   \n",
       "29996        12.0        0.42  0.999950  0.666644  ...  0.303090  0.692995   \n",
       "29997         9.0        0.44  0.666644  0.666644  ...  0.362742  0.194083   \n",
       "29998        12.0        0.50  0.499987  0.499987  ...  0.580127  0.441762   \n",
       "29999        20.0        0.20  0.199996  0.199996  ...       NaN       NaN   \n",
       "\n",
       "             92        93        94        95        96        97        98  \\\n",
       "0      0.043656 -0.284470  0.911652  0.134902 -0.161196  0.086004 -0.066971   \n",
       "1     -0.140012  0.085546  0.737972  0.560770  0.479416 -0.249945 -0.383590   \n",
       "2      0.459042  0.177220  0.707681  1.108182  0.201348 -0.544829 -0.785933   \n",
       "3     -0.082359 -0.168592  0.749170  0.755016 -0.289738 -0.575909 -0.137944   \n",
       "4     -0.561544  0.207188  0.252208 -0.123082 -0.323588 -0.564712 -0.078848   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "29995  0.025565  0.262315  0.411220  0.730304  0.128723 -0.488675 -0.078426   \n",
       "29996  0.078118  0.703030  0.959234  0.124965  0.052928 -0.480969 -0.189443   \n",
       "29997  0.098163  0.109113  0.463521  0.636916  0.282300 -0.471562 -0.080146   \n",
       "29998  0.612558  0.580019  0.535653  1.090930  0.836119 -0.232894 -0.774645   \n",
       "29999       NaN       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "\n",
       "             99  \n",
       "0      0.125468  \n",
       "1     -0.103985  \n",
       "2      0.320921  \n",
       "3      0.169422  \n",
       "4      0.371620  \n",
       "...         ...  \n",
       "29995  0.894383  \n",
       "29996  0.291008  \n",
       "29997  0.306487  \n",
       "29998  0.807484  \n",
       "29999       NaN  \n",
       "\n",
       "[30000 rows x 223 columns]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finalw_df = pd.concat([final_df,x_data,y_data], axis=1)\n",
    "finalw_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "5fb8e960",
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_rows = (np.abs(finalw_df) > np.finfo(np.float32).max).any(axis=1) | np.isinf(finalw_df).any(axis=1) | np.isnan(finalw_df).any(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "3b7e3f16",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_df = finalw_df[~bad_rows]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "e298dce6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_duplicate</th>\n",
       "      <th>q1_len</th>\n",
       "      <th>q2_len</th>\n",
       "      <th>q1_num_words</th>\n",
       "      <th>q2_num_words</th>\n",
       "      <th>word_common</th>\n",
       "      <th>word_total</th>\n",
       "      <th>word_share</th>\n",
       "      <th>cwc_min</th>\n",
       "      <th>cwc_max</th>\n",
       "      <th>...</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.081008</td>\n",
       "      <td>-0.181846</td>\n",
       "      <td>0.043656</td>\n",
       "      <td>-0.284470</td>\n",
       "      <td>0.911652</td>\n",
       "      <td>0.134902</td>\n",
       "      <td>-0.161196</td>\n",
       "      <td>0.086004</td>\n",
       "      <td>-0.066971</td>\n",
       "      <td>0.125468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.799984</td>\n",
       "      <td>0.285712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.216604</td>\n",
       "      <td>0.528166</td>\n",
       "      <td>-0.140012</td>\n",
       "      <td>0.085546</td>\n",
       "      <td>0.737972</td>\n",
       "      <td>0.560770</td>\n",
       "      <td>0.479416</td>\n",
       "      <td>-0.249945</td>\n",
       "      <td>-0.383590</td>\n",
       "      <td>-0.103985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>0.333328</td>\n",
       "      <td>...</td>\n",
       "      <td>0.261018</td>\n",
       "      <td>0.536788</td>\n",
       "      <td>0.459042</td>\n",
       "      <td>0.177220</td>\n",
       "      <td>0.707681</td>\n",
       "      <td>1.108182</td>\n",
       "      <td>0.201348</td>\n",
       "      <td>-0.544829</td>\n",
       "      <td>-0.785933</td>\n",
       "      <td>0.320921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.166664</td>\n",
       "      <td>0.099999</td>\n",
       "      <td>...</td>\n",
       "      <td>0.422205</td>\n",
       "      <td>0.293958</td>\n",
       "      <td>-0.082359</td>\n",
       "      <td>-0.168592</td>\n",
       "      <td>0.749170</td>\n",
       "      <td>0.755016</td>\n",
       "      <td>-0.289738</td>\n",
       "      <td>-0.575909</td>\n",
       "      <td>-0.137944</td>\n",
       "      <td>0.169422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.399992</td>\n",
       "      <td>0.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>1.045250</td>\n",
       "      <td>0.381819</td>\n",
       "      <td>-0.561544</td>\n",
       "      <td>0.207188</td>\n",
       "      <td>0.252208</td>\n",
       "      <td>-0.123082</td>\n",
       "      <td>-0.323588</td>\n",
       "      <td>-0.564712</td>\n",
       "      <td>-0.078848</td>\n",
       "      <td>0.371620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29994</th>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.249994</td>\n",
       "      <td>0.199996</td>\n",
       "      <td>...</td>\n",
       "      <td>0.342219</td>\n",
       "      <td>0.372661</td>\n",
       "      <td>0.409757</td>\n",
       "      <td>-0.160408</td>\n",
       "      <td>1.125393</td>\n",
       "      <td>0.881159</td>\n",
       "      <td>-0.180660</td>\n",
       "      <td>0.033588</td>\n",
       "      <td>-0.055709</td>\n",
       "      <td>0.615418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29995</th>\n",
       "      <td>1.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>...</td>\n",
       "      <td>0.514043</td>\n",
       "      <td>0.479171</td>\n",
       "      <td>0.025565</td>\n",
       "      <td>0.262315</td>\n",
       "      <td>0.411220</td>\n",
       "      <td>0.730304</td>\n",
       "      <td>0.128723</td>\n",
       "      <td>-0.488675</td>\n",
       "      <td>-0.078426</td>\n",
       "      <td>0.894383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29996</th>\n",
       "      <td>1.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.303090</td>\n",
       "      <td>0.692995</td>\n",
       "      <td>0.078118</td>\n",
       "      <td>0.703030</td>\n",
       "      <td>0.959234</td>\n",
       "      <td>0.124965</td>\n",
       "      <td>0.052928</td>\n",
       "      <td>-0.480969</td>\n",
       "      <td>-0.189443</td>\n",
       "      <td>0.291008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>0.666644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.362742</td>\n",
       "      <td>0.194083</td>\n",
       "      <td>0.098163</td>\n",
       "      <td>0.109113</td>\n",
       "      <td>0.463521</td>\n",
       "      <td>0.636916</td>\n",
       "      <td>0.282300</td>\n",
       "      <td>-0.471562</td>\n",
       "      <td>-0.080146</td>\n",
       "      <td>0.306487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>0.499987</td>\n",
       "      <td>...</td>\n",
       "      <td>0.580127</td>\n",
       "      <td>0.441762</td>\n",
       "      <td>0.612558</td>\n",
       "      <td>0.580019</td>\n",
       "      <td>0.535653</td>\n",
       "      <td>1.090930</td>\n",
       "      <td>0.836119</td>\n",
       "      <td>-0.232894</td>\n",
       "      <td>-0.774645</td>\n",
       "      <td>0.807484</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29998 rows × 223 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       is_duplicate  q1_len  q2_len  q1_num_words  q2_num_words  word_common  \\\n",
       "0               0.0    38.0    53.0           6.0           9.0          2.0   \n",
       "1               1.0   132.0    46.0          28.0          10.0          5.0   \n",
       "2               1.0    59.0    59.0          10.0          12.0          5.0   \n",
       "3               0.0   120.0    61.0          25.0          10.0          2.0   \n",
       "4               0.0    67.0    49.0          15.0          10.0          5.0   \n",
       "...             ...     ...     ...           ...           ...          ...   \n",
       "29994           0.0    53.0    34.0          11.0           8.0          3.0   \n",
       "29995           1.0    42.0    41.0           9.0           9.0          8.0   \n",
       "29996           1.0    28.0    41.0           7.0          11.0          5.0   \n",
       "29997           0.0    35.0    34.0           7.0           6.0          4.0   \n",
       "29998           0.0    40.0    42.0          10.0           9.0          6.0   \n",
       "\n",
       "       word_total  word_share   cwc_min   cwc_max  ...        90        91  \\\n",
       "0            13.0        0.15  0.499987  0.499987  ... -0.081008 -0.181846   \n",
       "1            28.0        0.18  0.799984  0.285712  ...  0.216604  0.528166   \n",
       "2            17.0        0.29  0.333328  0.333328  ...  0.261018  0.536788   \n",
       "3            29.0        0.07  0.166664  0.099999  ...  0.422205  0.293958   \n",
       "4            20.0        0.25  0.399992  0.285710  ...  1.045250  0.381819   \n",
       "...           ...         ...       ...       ...  ...       ...       ...   \n",
       "29994        15.0        0.20  0.249994  0.199996  ...  0.342219  0.372661   \n",
       "29995        10.0        0.80  0.999975  0.999975  ...  0.514043  0.479171   \n",
       "29996        12.0        0.42  0.999950  0.666644  ...  0.303090  0.692995   \n",
       "29997         9.0        0.44  0.666644  0.666644  ...  0.362742  0.194083   \n",
       "29998        12.0        0.50  0.499987  0.499987  ...  0.580127  0.441762   \n",
       "\n",
       "             92        93        94        95        96        97        98  \\\n",
       "0      0.043656 -0.284470  0.911652  0.134902 -0.161196  0.086004 -0.066971   \n",
       "1     -0.140012  0.085546  0.737972  0.560770  0.479416 -0.249945 -0.383590   \n",
       "2      0.459042  0.177220  0.707681  1.108182  0.201348 -0.544829 -0.785933   \n",
       "3     -0.082359 -0.168592  0.749170  0.755016 -0.289738 -0.575909 -0.137944   \n",
       "4     -0.561544  0.207188  0.252208 -0.123082 -0.323588 -0.564712 -0.078848   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "29994  0.409757 -0.160408  1.125393  0.881159 -0.180660  0.033588 -0.055709   \n",
       "29995  0.025565  0.262315  0.411220  0.730304  0.128723 -0.488675 -0.078426   \n",
       "29996  0.078118  0.703030  0.959234  0.124965  0.052928 -0.480969 -0.189443   \n",
       "29997  0.098163  0.109113  0.463521  0.636916  0.282300 -0.471562 -0.080146   \n",
       "29998  0.612558  0.580019  0.535653  1.090930  0.836119 -0.232894 -0.774645   \n",
       "\n",
       "             99  \n",
       "0      0.125468  \n",
       "1     -0.103985  \n",
       "2      0.320921  \n",
       "3      0.169422  \n",
       "4      0.371620  \n",
       "...         ...  \n",
       "29994  0.615418  \n",
       "29995  0.894383  \n",
       "29996  0.291008  \n",
       "29997  0.306487  \n",
       "29998  0.807484  \n",
       "\n",
       "[29998 rows x 223 columns]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "dd09e9fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(cleaned_df.iloc[:,1:].values,cleaned_df.iloc[:,0].values,test_size=0.2,random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "c9cdf6fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23998, 222)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "7f53d851",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7418333333333333"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "rf = RandomForestClassifier()\n",
    "rf.fit(X_train,y_train)\n",
    "y_pred = rf.predict(X_test)\n",
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0b8d2be",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}