IBM intro to deep learning - regression with neural nets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prelims\n",
    "\n",
    "Let's get the data downloaded and converted into a dataframe first."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('concrete_data.csv', <http.client.HTTPMessage at 0x1d9a4f02b48>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import urllib.request\n",
    "\n",
    "url = 'https://cocl.us/concrete_data'\n",
    "\n",
    "urllib.request.urlretrieve(url, 'concrete_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cement</th>\n",
       "      <th>Blast Furnace Slag</th>\n",
       "      <th>Fly Ash</th>\n",
       "      <th>Water</th>\n",
       "      <th>Superplasticizer</th>\n",
       "      <th>Coarse Aggregate</th>\n",
       "      <th>Fine Aggregate</th>\n",
       "      <th>Age</th>\n",
       "      <th>Strength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1040.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "      <td>79.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1055.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "      <td>61.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>270</td>\n",
       "      <td>40.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>365</td>\n",
       "      <td>41.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>198.6</td>\n",
       "      <td>132.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978.4</td>\n",
       "      <td>825.5</td>\n",
       "      <td>360</td>\n",
       "      <td>44.30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Cement  Blast Furnace Slag  Fly Ash  Water  Superplasticizer  \\\n",
       "0   540.0                 0.0      0.0  162.0               2.5   \n",
       "1   540.0                 0.0      0.0  162.0               2.5   \n",
       "2   332.5               142.5      0.0  228.0               0.0   \n",
       "3   332.5               142.5      0.0  228.0               0.0   \n",
       "4   198.6               132.4      0.0  192.0               0.0   \n",
       "\n",
       "   Coarse Aggregate  Fine Aggregate  Age  Strength  \n",
       "0            1040.0           676.0   28     79.99  \n",
       "1            1055.0           676.0   28     61.89  \n",
       "2             932.0           594.0  270     40.27  \n",
       "3             932.0           594.0  365     41.05  \n",
       "4             978.4           825.5  360     44.30  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_csv('concrete_data.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part A\n",
    "\n",
    "Baseline model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "pred_cols = [col for col in df.columns if col != 'Strength']\n",
    "X = df[pred_cols].to_numpy()\n",
    "y = df['Strength'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting run 0...\n",
      "starting run 1...\n",
      "starting run 2...\n",
      "starting run 3...\n",
      "starting run 4...\n",
      "starting run 5...\n",
      "starting run 6...\n",
      "starting run 7...\n",
      "starting run 8...\n",
      "starting run 9...\n",
      "starting run 10...\n",
      "starting run 11...\n",
      "starting run 12...\n",
      "starting run 13...\n",
      "starting run 14...\n",
      "starting run 15...\n",
      "starting run 16...\n",
      "starting run 17...\n",
      "starting run 18...\n",
      "starting run 19...\n",
      "starting run 20...\n",
      "starting run 21...\n",
      "starting run 22...\n",
      "starting run 23...\n",
      "starting run 24...\n",
      "starting run 25...\n",
      "starting run 26...\n",
      "starting run 27...\n",
      "starting run 28...\n",
      "starting run 29...\n",
      "starting run 30...\n",
      "starting run 31...\n",
      "starting run 32...\n",
      "starting run 33...\n",
      "starting run 34...\n",
      "starting run 35...\n",
      "starting run 36...\n",
      "starting run 37...\n",
      "starting run 38...\n",
      "starting run 39...\n",
      "starting run 40...\n",
      "starting run 41...\n",
      "starting run 42...\n",
      "starting run 43...\n",
      "starting run 44...\n",
      "starting run 45...\n",
      "starting run 46...\n",
      "starting run 47...\n",
      "starting run 48...\n",
      "starting run 49...\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "resultsA = []\n",
    "\n",
    "for i in range(50):\n",
    "    print(\"starting run {}...\".format(i))\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.3)\n",
    "    modelA = Sequential()\n",
    "    modelA.add(Dense(10, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "    modelA.add(Dense(1))\n",
    "    modelA.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "    modelA.fit(X_train, y_train, epochs=50, verbose=0)\n",
    "    resultsA.append(modelA.evaluate(X_test, y_test, verbose=0))\n",
    "resultsA = np.array(resultsA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of MSEs: 400.47796578317786\n",
      "Stddev of MSEs: 423.0974670369794\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('Mean of MSEs:', resultsA.mean())\n",
    "print('Stddev of MSEs:', resultsA.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part B\n",
    "\n",
    "Normalized data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting run 0...\n",
      "starting run 1...\n",
      "starting run 2...\n",
      "starting run 3...\n",
      "starting run 4...\n",
      "starting run 5...\n",
      "starting run 6...\n",
      "starting run 7...\n",
      "starting run 8...\n",
      "starting run 9...\n",
      "starting run 10...\n",
      "starting run 11...\n",
      "starting run 12...\n",
      "starting run 13...\n",
      "starting run 14...\n",
      "starting run 15...\n",
      "starting run 16...\n",
      "starting run 17...\n",
      "starting run 18...\n",
      "starting run 19...\n",
      "starting run 20...\n",
      "starting run 21...\n",
      "starting run 22...\n",
      "starting run 23...\n",
      "starting run 24...\n",
      "starting run 25...\n",
      "starting run 26...\n",
      "starting run 27...\n",
      "starting run 28...\n",
      "starting run 29...\n",
      "starting run 30...\n",
      "starting run 31...\n",
      "starting run 32...\n",
      "starting run 33...\n",
      "starting run 34...\n",
      "starting run 35...\n",
      "starting run 36...\n",
      "starting run 37...\n",
      "starting run 38...\n",
      "starting run 39...\n",
      "starting run 40...\n",
      "starting run 41...\n",
      "starting run 42...\n",
      "starting run 43...\n",
      "starting run 44...\n",
      "starting run 45...\n",
      "starting run 46...\n",
      "starting run 47...\n",
      "starting run 48...\n",
      "starting run 49...\n"
     ]
    }
   ],
   "source": [
    "X_normalized = (X - X.mean(axis=0))/X.std(axis=0)\n",
    "\n",
    "resultsB = []\n",
    "\n",
    "for i in range(50):\n",
    "    print(\"starting run {}...\".format(i))\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_normalized,y, test_size = 0.3)\n",
    "    modelA = Sequential()\n",
    "    modelA.add(Dense(10, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "    modelA.add(Dense(1))\n",
    "    modelA.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "    modelA.fit(X_train, y_train, epochs=50, verbose=0)\n",
    "    resultsB.append(modelA.evaluate(X_test, y_test, verbose=0))\n",
    "resultsB = np.array(resultsB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of MSEs: 327.9287613053306\n",
      "Stddev of MSEs: 75.50634621907868\n",
      "Compare to...\n",
      "Part A MSE mean: 400.47796578317786\n",
      "Part A MSE std: 423.0974670369794\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('Mean of MSEs:', resultsB.mean())\n",
    "print('Stddev of MSEs:', resultsB.std())\n",
    "\n",
    "print('Compare to...')\n",
    "print('Part A MSE mean:', resultsA.mean())\n",
    "print('Part A MSE std:', resultsA.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalizing helped decrease the MSE on average and enormously reduced the standard deviation of the prediction error."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting run 0...\n",
      "starting run 1...\n",
      "starting run 2...\n",
      "starting run 3...\n",
      "starting run 4...\n",
      "starting run 5...\n",
      "starting run 6...\n",
      "starting run 7...\n",
      "starting run 8...\n",
      "starting run 9...\n",
      "starting run 10...\n",
      "starting run 11...\n",
      "starting run 12...\n",
      "starting run 13...\n",
      "starting run 14...\n",
      "starting run 15...\n",
      "starting run 16...\n",
      "starting run 17...\n",
      "starting run 18...\n",
      "starting run 19...\n",
      "starting run 20...\n",
      "starting run 21...\n",
      "starting run 22...\n",
      "starting run 23...\n",
      "starting run 24...\n",
      "starting run 25...\n",
      "starting run 26...\n",
      "starting run 27...\n",
      "starting run 28...\n",
      "starting run 29...\n",
      "starting run 30...\n",
      "starting run 31...\n",
      "starting run 32...\n",
      "starting run 33...\n",
      "starting run 34...\n",
      "starting run 35...\n",
      "starting run 36...\n",
      "starting run 37...\n",
      "starting run 38...\n",
      "starting run 39...\n",
      "starting run 40...\n",
      "starting run 41...\n",
      "starting run 42...\n",
      "starting run 43...\n",
      "starting run 44...\n",
      "starting run 45...\n",
      "starting run 46...\n",
      "starting run 47...\n",
      "starting run 48...\n",
      "starting run 49...\n"
     ]
    }
   ],
   "source": [
    "resultsC = []\n",
    "\n",
    "for i in range(50):\n",
    "    print(\"starting run {}...\".format(i))\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_normalized,y, test_size = 0.3)\n",
    "    modelA = Sequential()\n",
    "    modelA.add(Dense(10, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "    modelA.add(Dense(1))\n",
    "    modelA.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "    modelA.fit(X_train, y_train, epochs=100, verbose=0)\n",
    "    resultsC.append(modelA.evaluate(X_test, y_test, verbose=0))\n",
    "resultsC = np.array(resultsC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of MSEs: 163.45172728948992\n",
      "Stddev of MSEs: 9.99178536813674\n",
      "Compare to...\n",
      "Part B MSE mean: 327.9287613053306\n",
      "Part B MSE std: 75.50634621907868\n"
     ]
    }
   ],
   "source": [
    "print('Mean of MSEs:', resultsC.mean())\n",
    "print('Stddev of MSEs:', resultsC.std())\n",
    "print('Compare to...')\n",
    "print('Part B MSE mean:', resultsB.mean())\n",
    "print('Part B MSE std:', resultsB.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Additional runtime halved the MSE and dramatically reduced the standard deviation of the errors."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting run 0...\n",
      "starting run 1...\n",
      "starting run 2...\n",
      "starting run 3...\n",
      "starting run 4...\n",
      "starting run 5...\n",
      "starting run 6...\n",
      "starting run 7...\n",
      "starting run 8...\n",
      "starting run 9...\n",
      "starting run 10...\n",
      "starting run 11...\n",
      "starting run 12...\n",
      "starting run 13...\n",
      "starting run 14...\n",
      "starting run 15...\n",
      "starting run 16...\n",
      "starting run 17...\n",
      "starting run 18...\n",
      "starting run 19...\n",
      "starting run 20...\n",
      "starting run 21...\n",
      "starting run 22...\n",
      "starting run 23...\n",
      "starting run 24...\n",
      "starting run 25...\n",
      "starting run 26...\n",
      "starting run 27...\n",
      "starting run 28...\n",
      "starting run 29...\n",
      "starting run 30...\n",
      "starting run 31...\n",
      "starting run 32...\n",
      "starting run 33...\n",
      "starting run 34...\n",
      "starting run 35...\n",
      "starting run 36...\n",
      "starting run 37...\n",
      "starting run 38...\n",
      "starting run 39...\n",
      "starting run 40...\n",
      "starting run 41...\n",
      "starting run 42...\n",
      "starting run 43...\n",
      "starting run 44...\n",
      "starting run 45...\n",
      "starting run 46...\n",
      "starting run 47...\n",
      "starting run 48...\n",
      "starting run 49...\n"
     ]
    }
   ],
   "source": [
    "resultsD = []\n",
    "\n",
    "for i in range(50):\n",
    "    print(\"starting run {}...\".format(i))\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_normalized,y, test_size = 0.3)\n",
    "    modelD = Sequential()\n",
    "    modelD.add(Dense(10, activation='relu', input_shape=(X_train.shape[1],)))\n",
    "    modelD.add(Dense(10, activation='relu'))\n",
    "    modelD.add(Dense(10, activation='relu'))\n",
    "    modelD.add(Dense(1))\n",
    "    modelD.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "    modelD.fit(X_train, y_train, epochs=50, verbose=0)\n",
    "    resultsD.append(modelA.evaluate(X_test, y_test, verbose=0))\n",
    "resultsD = np.array(resultsD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of MSEs: 156.10441887321596\n",
      "Stddev of MSEs: 9.878436398598264\n",
      "Compare to...\n",
      "Part B MSE mean: 327.9287613053306\n",
      "Part B MSE std: 75.50634621907868\n"
     ]
    }
   ],
   "source": [
    "print('Mean of MSEs:', resultsD.mean())\n",
    "print('Stddev of MSEs:', resultsD.std())\n",
    "print('Compare to...')\n",
    "print('Part B MSE mean:', resultsB.mean())\n",
    "print('Part B MSE std:', resultsB.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding depth to the neural net has a similar effect as adding training time: the mean MSE is halved and the standard deviation of the error is much smaller."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}