EnsembleTechniques1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/anaconda/envs/py35/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_realestate = pd.read_csv('RealEstate.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(414, 8)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_realestate.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No</th>\n",
       "      <th>X1 transaction date</th>\n",
       "      <th>X2 house age</th>\n",
       "      <th>X3 distance to the nearest MRT station</th>\n",
       "      <th>X4 number of convenience stores</th>\n",
       "      <th>X5 latitude</th>\n",
       "      <th>X6 longitude</th>\n",
       "      <th>Y house price of unit area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2012.917</td>\n",
       "      <td>32.0</td>\n",
       "      <td>84.87882</td>\n",
       "      <td>10</td>\n",
       "      <td>24.98298</td>\n",
       "      <td>121.54024</td>\n",
       "      <td>37.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2012.917</td>\n",
       "      <td>19.5</td>\n",
       "      <td>306.59470</td>\n",
       "      <td>9</td>\n",
       "      <td>24.98034</td>\n",
       "      <td>121.53951</td>\n",
       "      <td>42.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2013.583</td>\n",
       "      <td>13.3</td>\n",
       "      <td>561.98450</td>\n",
       "      <td>5</td>\n",
       "      <td>24.98746</td>\n",
       "      <td>121.54391</td>\n",
       "      <td>47.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2013.500</td>\n",
       "      <td>13.3</td>\n",
       "      <td>561.98450</td>\n",
       "      <td>5</td>\n",
       "      <td>24.98746</td>\n",
       "      <td>121.54391</td>\n",
       "      <td>54.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2012.833</td>\n",
       "      <td>5.0</td>\n",
       "      <td>390.56840</td>\n",
       "      <td>5</td>\n",
       "      <td>24.97937</td>\n",
       "      <td>121.54245</td>\n",
       "      <td>43.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   No  X1 transaction date  X2 house age  \\\n",
       "0   1             2012.917          32.0   \n",
       "1   2             2012.917          19.5   \n",
       "2   3             2013.583          13.3   \n",
       "3   4             2013.500          13.3   \n",
       "4   5             2012.833           5.0   \n",
       "\n",
       "   X3 distance to the nearest MRT station  X4 number of convenience stores  \\\n",
       "0                                84.87882                               10   \n",
       "1                               306.59470                                9   \n",
       "2                               561.98450                                5   \n",
       "3                               561.98450                                5   \n",
       "4                               390.56840                                5   \n",
       "\n",
       "   X5 latitude  X6 longitude  Y house price of unit area  \n",
       "0     24.98298     121.54024                        37.9  \n",
       "1     24.98034     121.53951                        42.2  \n",
       "2     24.98746     121.54391                        47.3  \n",
       "3     24.98746     121.54391                        54.8  \n",
       "4     24.97937     121.54245                        43.1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_realestate.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No                                          int64\n",
       "X1 transaction date                       float64\n",
       "X2 house age                              float64\n",
       "X3 distance to the nearest MRT station    float64\n",
       "X4 number of convenience stores             int64\n",
       "X5 latitude                               float64\n",
       "X6 longitude                              float64\n",
       "Y house price of unit area                float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_realestate.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No                                        0\n",
       "X1 transaction date                       0\n",
       "X2 house age                              0\n",
       "X3 distance to the nearest MRT station    0\n",
       "X4 number of convenience stores           0\n",
       "X5 latitude                               0\n",
       "X6 longitude                              0\n",
       "Y house price of unit area                0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_realestate.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_realestate.drop(['No'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X1 transaction date</th>\n",
       "      <th>X2 house age</th>\n",
       "      <th>X3 distance to the nearest MRT station</th>\n",
       "      <th>X4 number of convenience stores</th>\n",
       "      <th>X5 latitude</th>\n",
       "      <th>X6 longitude</th>\n",
       "      <th>Y house price of unit area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>X1 transaction date</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.017549</td>\n",
       "      <td>0.060880</td>\n",
       "      <td>0.009635</td>\n",
       "      <td>0.035058</td>\n",
       "      <td>-0.041082</td>\n",
       "      <td>0.087491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X2 house age</th>\n",
       "      <td>0.017549</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.025622</td>\n",
       "      <td>0.049593</td>\n",
       "      <td>0.054420</td>\n",
       "      <td>-0.048520</td>\n",
       "      <td>-0.210567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X3 distance to the nearest MRT station</th>\n",
       "      <td>0.060880</td>\n",
       "      <td>0.025622</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.602519</td>\n",
       "      <td>-0.591067</td>\n",
       "      <td>-0.806317</td>\n",
       "      <td>-0.673613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X4 number of convenience stores</th>\n",
       "      <td>0.009635</td>\n",
       "      <td>0.049593</td>\n",
       "      <td>-0.602519</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.444143</td>\n",
       "      <td>0.449099</td>\n",
       "      <td>0.571005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X5 latitude</th>\n",
       "      <td>0.035058</td>\n",
       "      <td>0.054420</td>\n",
       "      <td>-0.591067</td>\n",
       "      <td>0.444143</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.412924</td>\n",
       "      <td>0.546307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X6 longitude</th>\n",
       "      <td>-0.041082</td>\n",
       "      <td>-0.048520</td>\n",
       "      <td>-0.806317</td>\n",
       "      <td>0.449099</td>\n",
       "      <td>0.412924</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.523287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y house price of unit area</th>\n",
       "      <td>0.087491</td>\n",
       "      <td>-0.210567</td>\n",
       "      <td>-0.673613</td>\n",
       "      <td>0.571005</td>\n",
       "      <td>0.546307</td>\n",
       "      <td>0.523287</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        X1 transaction date  X2 house age  \\\n",
       "X1 transaction date                                1.000000      0.017549   \n",
       "X2 house age                                       0.017549      1.000000   \n",
       "X3 distance to the nearest MRT station             0.060880      0.025622   \n",
       "X4 number of convenience stores                    0.009635      0.049593   \n",
       "X5 latitude                                        0.035058      0.054420   \n",
       "X6 longitude                                      -0.041082     -0.048520   \n",
       "Y house price of unit area                         0.087491     -0.210567   \n",
       "\n",
       "                                        X3 distance to the nearest MRT station  \\\n",
       "X1 transaction date                                                   0.060880   \n",
       "X2 house age                                                          0.025622   \n",
       "X3 distance to the nearest MRT station                                1.000000   \n",
       "X4 number of convenience stores                                      -0.602519   \n",
       "X5 latitude                                                          -0.591067   \n",
       "X6 longitude                                                         -0.806317   \n",
       "Y house price of unit area                                           -0.673613   \n",
       "\n",
       "                                        X4 number of convenience stores  \\\n",
       "X1 transaction date                                            0.009635   \n",
       "X2 house age                                                   0.049593   \n",
       "X3 distance to the nearest MRT station                        -0.602519   \n",
       "X4 number of convenience stores                                1.000000   \n",
       "X5 latitude                                                    0.444143   \n",
       "X6 longitude                                                   0.449099   \n",
       "Y house price of unit area                                     0.571005   \n",
       "\n",
       "                                        X5 latitude  X6 longitude  \\\n",
       "X1 transaction date                        0.035058     -0.041082   \n",
       "X2 house age                               0.054420     -0.048520   \n",
       "X3 distance to the nearest MRT station    -0.591067     -0.806317   \n",
       "X4 number of convenience stores            0.444143      0.449099   \n",
       "X5 latitude                                1.000000      0.412924   \n",
       "X6 longitude                               0.412924      1.000000   \n",
       "Y house price of unit area                 0.546307      0.523287   \n",
       "\n",
       "                                        Y house price of unit area  \n",
       "X1 transaction date                                       0.087491  \n",
       "X2 house age                                             -0.210567  \n",
       "X3 distance to the nearest MRT station                   -0.673613  \n",
       "X4 number of convenience stores                           0.571005  \n",
       "X5 latitude                                               0.546307  \n",
       "X6 longitude                                              0.523287  \n",
       "Y house price of unit area                                1.000000  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_realestate.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_realestate_X = df_realestate.drop(['Y house price of unit area'], axis=1)\n",
    "df_realestate_y = df_realestate[['Y house price of unit area']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(414, 6)\n",
      "(414, 1)\n"
     ]
    }
   ],
   "source": [
    "print(df_realestate_X.shape)\n",
    "print(df_realestate_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/preprocessing/data.py:617: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/base.py:462: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "sc_X=StandardScaler()\n",
    "df_realestate_X1 = sc_X.fit_transform(df_realestate_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc_y=StandardScaler()\n",
    "df_realestate_y1 = sc_y.fit_transform(df_realestate_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.formula.api as sm\n",
    "X=np.append(arr=np.ones((414,1)).astype(int), values=df_realestate_X1, axis=1)\n",
    "X_opt=X[:,1:]\n",
    "X_opt.shape\n",
    "y=df_realestate_y1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>            <td>y</td>        <th>  R-squared:         </th> <td>   0.582</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.576</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   94.82</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Tue, 29 Sep 2020</td> <th>  Prob (F-statistic):</th> <td>3.16e-74</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>16:57:54</td>     <th>  Log-Likelihood:    </th> <td> -406.70</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   414</td>      <th>  AIC:               </th> <td>   825.4</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   408</td>      <th>  BIC:               </th> <td>   849.5</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     6</td>      <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "   <td></td>     <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x1</th> <td>    0.1066</td> <td>    0.032</td> <td>    3.309</td> <td> 0.001</td> <td>    0.043</td> <td>    0.170</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x2</th> <td>   -0.2258</td> <td>    0.032</td> <td>   -7.008</td> <td> 0.000</td> <td>   -0.289</td> <td>   -0.162</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x3</th> <td>   -0.4162</td> <td>    0.067</td> <td>   -6.257</td> <td> 0.000</td> <td>   -0.547</td> <td>   -0.285</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x4</th> <td>    0.2453</td> <td>    0.041</td> <td>    6.030</td> <td> 0.000</td> <td>    0.165</td> <td>    0.325</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x5</th> <td>    0.2056</td> <td>    0.041</td> <td>    5.065</td> <td> 0.000</td> <td>    0.126</td> <td>    0.285</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x6</th> <td>   -0.0140</td> <td>    0.055</td> <td>   -0.256</td> <td> 0.798</td> <td>   -0.122</td> <td>    0.094</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>231.634</td> <th>  Durbin-Watson:     </th> <td>   2.153</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>3599.138</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 2.026</td>  <th>  Prob(JB):          </th> <td>    0.00</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td>16.865</td>  <th>  Cond. No.          </th> <td>    4.22</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:                      y   R-squared:                       0.582\n",
       "Model:                            OLS   Adj. R-squared:                  0.576\n",
       "Method:                 Least Squares   F-statistic:                     94.82\n",
       "Date:                Tue, 29 Sep 2020   Prob (F-statistic):           3.16e-74\n",
       "Time:                        16:57:54   Log-Likelihood:                -406.70\n",
       "No. Observations:                 414   AIC:                             825.4\n",
       "Df Residuals:                     408   BIC:                             849.5\n",
       "Df Model:                           6                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "x1             0.1066      0.032      3.309      0.001       0.043       0.170\n",
       "x2            -0.2258      0.032     -7.008      0.000      -0.289      -0.162\n",
       "x3            -0.4162      0.067     -6.257      0.000      -0.547      -0.285\n",
       "x4             0.2453      0.041      6.030      0.000       0.165       0.325\n",
       "x5             0.2056      0.041      5.065      0.000       0.126       0.285\n",
       "x6            -0.0140      0.055     -0.256      0.798      -0.122       0.094\n",
       "==============================================================================\n",
       "Omnibus:                      231.634   Durbin-Watson:                   2.153\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             3599.138\n",
       "Skew:                           2.026   Prob(JB):                         0.00\n",
       "Kurtosis:                      16.865   Cond. No.                         4.22\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regressor_OLS = sm.OLS(endog=y, exog=X_opt).fit()\n",
    "regressor_OLS.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(414, 5)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_opt=X[:,1:6]\n",
    "X_opt.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>            <td>y</td>        <th>  R-squared:         </th> <td>   0.582</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.577</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   114.0</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Tue, 29 Sep 2020</td> <th>  Prob (F-statistic):</th> <td>2.90e-75</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>16:57:55</td>     <th>  Log-Likelihood:    </th> <td> -406.73</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   414</td>      <th>  AIC:               </th> <td>   823.5</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   409</td>      <th>  BIC:               </th> <td>   843.6</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     5</td>      <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "   <td></td>     <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x1</th> <td>    0.1064</td> <td>    0.032</td> <td>    3.307</td> <td> 0.001</td> <td>    0.043</td> <td>    0.170</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x2</th> <td>   -0.2255</td> <td>    0.032</td> <td>   -7.012</td> <td> 0.000</td> <td>   -0.289</td> <td>   -0.162</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x3</th> <td>   -0.4038</td> <td>    0.045</td> <td>   -8.898</td> <td> 0.000</td> <td>   -0.493</td> <td>   -0.315</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x4</th> <td>    0.2460</td> <td>    0.041</td> <td>    6.063</td> <td> 0.000</td> <td>    0.166</td> <td>    0.326</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>x5</th> <td>    0.2069</td> <td>    0.040</td> <td>    5.142</td> <td> 0.000</td> <td>    0.128</td> <td>    0.286</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>232.810</td> <th>  Durbin-Watson:     </th> <td>   2.154</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>3644.713</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 2.038</td>  <th>  Prob(JB):          </th> <td>    0.00</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td>16.953</td>  <th>  Cond. No.          </th> <td>    2.50</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:                      y   R-squared:                       0.582\n",
       "Model:                            OLS   Adj. R-squared:                  0.577\n",
       "Method:                 Least Squares   F-statistic:                     114.0\n",
       "Date:                Tue, 29 Sep 2020   Prob (F-statistic):           2.90e-75\n",
       "Time:                        16:57:55   Log-Likelihood:                -406.73\n",
       "No. Observations:                 414   AIC:                             823.5\n",
       "Df Residuals:                     409   BIC:                             843.6\n",
       "Df Model:                           5                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "x1             0.1064      0.032      3.307      0.001       0.043       0.170\n",
       "x2            -0.2255      0.032     -7.012      0.000      -0.289      -0.162\n",
       "x3            -0.4038      0.045     -8.898      0.000      -0.493      -0.315\n",
       "x4             0.2460      0.041      6.063      0.000       0.166       0.326\n",
       "x5             0.2069      0.040      5.142      0.000       0.128       0.286\n",
       "==============================================================================\n",
       "Omnibus:                      232.810   Durbin-Watson:                   2.154\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             3644.713\n",
       "Skew:                           2.038   Prob(JB):                         0.00\n",
       "Kurtosis:                      16.953   Cond. No.                         2.50\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regressor_OLS = sm.OLS(endog=y, exog=X_opt).fit()\n",
    "regressor_OLS.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.25562833, -0.79249504,  2.00740743,  1.12542986,  0.44876222],\n",
       "       [ 0.15708622, -0.61661163,  1.66750286,  0.91244418,  0.40113894],\n",
       "       [-0.38779067, -0.41401527,  0.30788458,  1.48686011,  0.68818339],\n",
       "       ...,\n",
       "       [ 0.09556786, -0.54967846,  0.98769372,  0.82289338,  0.42397202],\n",
       "       [-0.84478419, -0.77668389,  0.30788458, -0.18475479,  0.4768143 ],\n",
       "       [-0.98539758, -0.78807071,  1.66750286,  0.42757905,  0.63534112]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_realestate_X1[:,1:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_realestate_X1, df_realestate_y1, test_size=0.3, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression, LogisticRegression, SGDRegressor, Lasso, Ridge\n",
    "from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier\n",
    "from sklearn.svm import SVR, SVC\n",
    "from sklearn.metrics import mean_absolute_error, accuracy_score\n",
    "from scipy.stats import mode"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average Ensembling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/utils/validation.py:752: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "knn_regressor = KNeighborsRegressor(1)\n",
    "lasso_regressor = Lasso()\n",
    "svr_regressor = SVR(kernel=\"poly\", degree=5)\n",
    "\n",
    "knn_regressor.fit(X_train, y_train)\n",
    "lasso_regressor.fit(X_train, y_train)\n",
    "svr_regressor.fit(X_train, y_train)\n",
    "\n",
    "y_pred1 = knn_regressor.predict(X_test)\n",
    "y_pred2 = lasso_regressor.predict(X_test)\n",
    "y_pred3 = svr_regressor.predict(X_test)\n",
    "\n",
    "y_pred1 = y_pred1.reshape(y_test.shape[0], 1)\n",
    "y_pred2 = y_pred2.reshape(125, 1)\n",
    "y_pred3 = y_pred3.reshape(125, 1)\n",
    "\n",
    "avg_pred = np.mean([y_pred1,y_pred2,y_pred3], axis =0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Ensembler Mean Absolute Error: 0.48709255488962744\n",
      "KNN Mean Absolute Error: 0.5220880505643672\n",
      "Lasso Mean Absolute Error: 0.7568088178180192\n",
      "SVR Mean Absolute Error: 0.5015218832952784\n"
     ]
    }
   ],
   "source": [
    "print(\"Average Ensembler Mean Absolute Error:\", mean_absolute_error(y_test, avg_pred))\n",
    "print(\"KNN Mean Absolute Error:\", mean_absolute_error(y_test, y_pred1))\n",
    "print(\"Lasso Mean Absolute Error:\", mean_absolute_error(y_test, y_pred2))\n",
    "print(\"SVR Mean Absolute Error:\", mean_absolute_error(y_test, y_pred3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weighted Average Ensembling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/utils/validation.py:752: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "knn_regressor = KNeighborsRegressor(1)\n",
    "lasso_regressor = Lasso()\n",
    "svr_regressor = SVR(kernel=\"poly\", degree=5)\n",
    "\n",
    "knn_regressor.fit(X_train, y_train)\n",
    "lasso_regressor.fit(X_train, y_train)\n",
    "svr_regressor.fit(X_train, y_train)\n",
    "\n",
    "y_pred1 = knn_regressor.predict(X_test)\n",
    "y_pred2 = lasso_regressor.predict(X_test)\n",
    "y_pred3 = svr_regressor.predict(X_test)\n",
    "\n",
    "y_pred1 = y_pred1.reshape(y_test.shape[0], 1)\n",
    "y_pred2 = y_pred2.reshape(125, 1)\n",
    "y_pred3 = y_pred3.reshape(125, 1)\n",
    "\n",
    "weighted_pred = (y_pred1*0.40+y_pred2*0.20+y_pred3*0.40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weightage Average Ensembler Mean Absolute Error: 0.46186097145642674\n",
      "KNN Mean Absolute Error: 0.5220880505643672\n",
      "Lasso Mean Absolute Error: 0.7568088178180192\n",
      "SVR Mean Absolute Error: 0.5015218832952784\n"
     ]
    }
   ],
   "source": [
    "print(\"Weightage Average Ensembler Mean Absolute Error:\", mean_absolute_error(y_test, weighted_pred))\n",
    "print(\"KNN Mean Absolute Error:\", mean_absolute_error(y_test, y_pred1))\n",
    "print(\"Lasso Mean Absolute Error:\", mean_absolute_error(y_test, y_pred2))\n",
    "print(\"SVR Mean Absolute Error:\", mean_absolute_error(y_test, y_pred3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Max Voting Ensembling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['40', '20', '40', '20', '40', '40', '40', '40', '60', '60', '60', '40', '40', '40', '60', '40', '40', '60', '60', '60', '60', '40', '40', '60', '60', '60', '60', '40', '60', '40', '60', '40', '60', '40', '60', '40', '60', '40', '60', '20', '60', '40', '40', '60', '20', '60', '40', '20', '40', '60', '60', '40', '60', '20', '40', '40', '60', '40', '60', '40', '40', '60', '40', '60', '40', '40', '20', '40', '60', '40', '40', '60', '40', '40', '40', '40', '40', '20', '60', '60', '60', '40', '40', '40', '40', '40', '60', '60', '60', '60', '60', '60', '40', '40', '20', '40', '40', '60', '20', '60', '60', '60', '60', '40', '40', '40', '60', '20', '40', '20', '60', '40', '40', '60', '60', '60', '60', '40', '40', '40', '40', '40', '40', '40', '60']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/linear_model/logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/scipy/stats/stats.py:245: RuntimeWarning: The input array could not be properly checked for nan values. nan values will be ignored.\n",
      "  \"values. nan values will be ignored.\", RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "category = pd.cut(df_realestate['Y house price of unit area'],bins=[0,20,40,60,100,120],labels=['20','40','60','100','120'])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(df_realestate_X1, category, test_size=0.3, random_state=0)\n",
    "\n",
    "knn_classifier = KNeighborsClassifier(1)\n",
    "logistic_regressor = LogisticRegression()\n",
    "svc_classifier = SVC(kernel=\"poly\", degree=5)\n",
    "\n",
    "knn_classifier.fit(X_train, y_train)\n",
    "logistic_regressor.fit(X_train, y_train)\n",
    "svc_classifier.fit(X_train, y_train)\n",
    "\n",
    "y_pred1 = knn_classifier.predict(X_test)\n",
    "y_pred2 = logistic_regressor.predict(X_test)\n",
    "y_pred3 = svc_classifier.predict(X_test)\n",
    "\n",
    "maxvoted_pred = []\n",
    "for i in range(y_test.shape[0]):\n",
    "    maxvoted_pred.append(mode([y_pred1[i], y_pred2[i], y_pred3[i]])[0][0])\n",
    "\n",
    "print(maxvoted_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Voted Ensembler Accuracy: 72.0\n",
      "KNN Accuracy: 67.2\n",
      "Logistic Regression Accuracy: 74.4\n",
      "SVC Accuracy: 70.39999999999999\n"
     ]
    }
   ],
   "source": [
    "print(\"Max Voted Ensembler Accuracy:\", accuracy_score(y_test, maxvoted_pred)*100)\n",
    "print(\"KNN Accuracy:\", accuracy_score(y_test, y_pred1)*100)\n",
    "print(\"Logistic Regression Accuracy:\", accuracy_score(y_test, y_pred2)*100)\n",
    "print(\"SVC Accuracy:\", accuracy_score(y_test, y_pred3)*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### sklearn implementation for MaxVoting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sklearn Max Voting Classifier Accuracy: 72.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/linear_model/logistic.py:459: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/anaconda/envs/py35/lib/python3.5/site-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "maxVotingClassifier = VotingClassifier(estimators=[('KNN', knn_classifier), ('Logistic Regression', logistic_regressor), ('SVC', svc_classifier)], voting='hard')\n",
    "maxVotingClassifier.fit(X_train, y_train)\n",
    "\n",
    "y_pred = maxVotingClassifier.predict(X_test)\n",
    "print(\"Sklearn Max Voting Classifier Accuracy:\", accuracy_score(y_test, y_pred)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}