codes/02_perceptron.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "📝 **Author:** Amirhossein Heydari - 📧 **Email:** amirhosseinheydari78@gmail.com - 📍 **Linktree:** [linktr.ee/mr_pylin](https://linktr.ee/mr_pylin)\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib.colors import ListedColormap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utility Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_decision_regions(X: np.ndarray, y: np.ndarray, classifier, resolution: float = 0.01) -> None:\n",
    "    # setup marker generator and color map\n",
    "    markers = ('o', 's')\n",
    "    colors = ('red', 'blue')\n",
    "    cmap = ListedColormap(colors)\n",
    "\n",
    "    # plot the decision surface\n",
    "    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1\n",
    "    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1\n",
    "    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution), np.arange(x2_min, x2_max, resolution))\n",
    "    y_pred = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T).reshape(xx1.shape)\n",
    "    plt.contourf(xx1, xx2, y_pred, alpha=0.2, cmap=cmap)\n",
    "    plt.xlim(xx1.min(), xx1.max())\n",
    "    plt.ylim(xx2.min(), xx2.max())\n",
    "\n",
    "    # plot class examples\n",
    "    for idx, cl in enumerate(np.unique(y)):\n",
    "        plt.scatter(x=X[y == cl, 0], y=X[y == cl, 1], alpha=0.8, c=colors[idx], marker=markers[idx], label=f'Class {cl}', edgecolor='black')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Iris Dataset\n",
    "\n",
    "<figure style=\"text-align: center;\">\n",
    "    <img src=\"../resources/images/others/01_08.png\" alt=\"01_08.png\" style=\"width: 50%;\">\n",
    "    <figcaption style=\"text-align: center;\">©️ Image: <a href= \"https://github.com/rasbt/machine-learning-book/blob/main/ch01/figures/01_08.png\">Machine Learning with PyTorch and Scikit-Learn</a></figcaption>\n",
    "</figure>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2    3            4\n",
       "0  5.1  3.5  1.4  0.2  Iris-setosa\n",
       "1  4.9  3.0  1.4  0.2  Iris-setosa\n",
       "2  4.7  3.2  1.3  0.2  Iris-setosa\n",
       "3  4.6  3.1  1.5  0.2  Iris-setosa\n",
       "4  5.0  3.6  1.4  0.2  Iris-setosa"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris_dataset_url = r\"https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data\"\n",
    "\n",
    "# pandas data-frame\n",
    "df = pd.read_csv(iris_dataset_url, header=None, encoding='utf-8')\n",
    "\n",
    "# log\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique labels: ['Iris-setosa' 'Iris-versicolor' 'Iris-virginica']\n",
      "Number of data per label: 4\n",
      "Iris-setosa        50\n",
      "Iris-versicolor    50\n",
      "Iris-virginica     50\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# number of unique labels\n",
    "unique_classes = df.iloc[:, -1].unique()\n",
    "\n",
    "# number of data per label\n",
    "num_data_per_class = df.iloc[:, -1].value_counts()\n",
    "\n",
    "# log\n",
    "print(f\"Unique labels: {unique_classes}\")\n",
    "print(f\"Number of data per label: {num_data_per_class}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X.shape : (100, 2)\n",
      "X.dtype : float64\n",
      "y.shape : (100,)\n",
      "y.dtype : int32\n"
     ]
    }
   ],
   "source": [
    "# we only need species: {'Iris-setosa', 'Iris-versicolor'}\n",
    "filtered_df = df[df.iloc[:, -1].isin(['Iris-setosa', 'Iris-versicolor'])]\n",
    "\n",
    "# select only the sepal length(first column) and petal length(third column)\n",
    "filtered_df = filtered_df.iloc[:, [0, 2, -1]]\n",
    "\n",
    "# split features and labels\n",
    "X = filtered_df.iloc[:, [0, 1]].values\n",
    "y = filtered_df.iloc[:, [2]].values.squeeze()\n",
    "\n",
    "# convert labels into numbers : {'Iris-setosa':0, 'Iris-versicolor':1}\n",
    "y = np.where(y == 'Iris-setosa', 0, 1)\n",
    "\n",
    "# log\n",
    "print(f\"X.shape : {X.shape}\")\n",
    "print(f\"X.dtype : {X.dtype}\")\n",
    "print(f\"y.shape : {y.shape}\")\n",
    "print(f\"y.dtype : {y.dtype}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot data\n",
    "plt.scatter(X[:50, 0], X[:50, 1], color='red', marker='o', label='Iris-setosa:0')\n",
    "plt.scatter(X[50:100, 0], X[50:100, 1], color='blue', marker='s', label='Iris-versicolor:1')\n",
    "plt.xlabel('Sepal length [cm]')\n",
    "plt.ylabel('Petal length [cm]')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perceptron\n",
    "   - The Perceptron is a type of binary linear classifier introduced by [Frank Rosenblatt](https://en.wikipedia.org/wiki/Frank_Rosenblatt) in 1958.\n",
    "   - The simplest form of a neural network, consisting of a single layer of input neurons connected to a single output neuron.\n",
    "   - Mathematical Form:\n",
    "      $$\n",
    "      y = \\begin{cases} \n",
    "      1 & \\text{if} \\; \\sum w_i x_i + b \\ge 0 \\\\\n",
    "      0 & \\text{otherwise}\n",
    "      \\end{cases}\n",
    "      $$\n",
    "      - $w_i$: Weights for input $x_i$\n",
    "      - $b$: Bias term\n",
    "   - 📝 Paper: [THE PERCEPTRON: A PROBABILISTIC MODEL FOR INFORMATION STORAGE AND ORGANIZATION IN THE BRAIN](https://www.ling.upenn.edu/courses/cogs501/Rosenblatt1958.pdf)\n",
    "\n",
    "\n",
    "\n",
    "<figure style=\"text-align: center;\">\n",
    "    <img src=\"../resources/images/SVGs/perceptron-2.svg\" alt=\"perceptron-2.png\" style=\"width: 80%;\">\n",
    "    <figcaption style=\"text-align: center;\">A Perceptron</figcaption>\n",
    "</figure>\n",
    "\n",
    "---\n",
    "\n",
    "**Training Rule (Rosenblatt's Rule)**:\n",
    "$$\n",
    "w_i \\leftarrow w_i + \\eta \\cdot (y_{\\text{true}} - y_{\\text{pred}}) \\cdot x_i\n",
    "$$\n",
    "   - $\\eta$: Learning rate\n",
    "   - $y_{true}$: Actual class label\n",
    "   - $y_{pred}$: Predicted class label\n",
    "\n",
    "✍️ Note:\n",
    "   - The following code is adapted from the book *[Machine Learning with PyTorch and Scikit-Learn](https://github.com/rasbt/machine-learning-book)* with **modifications** made to fit the requirements of this analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Perceptron:\n",
    "    def __init__(self, eta: int = 0.01, epochs: int = 50, seed: int = 42) -> None:\n",
    "        self.eta = eta\n",
    "        self.epochs = epochs\n",
    "        self.seed = seed\n",
    "\n",
    "    def fit(self, X: np.ndarray, y: np.ndarray) -> None:\n",
    "        # initialize weights and bias following a normal distribution with a deterministic seed\n",
    "        rng = np.random.default_rng(seed=self.seed)\n",
    "        self.w_ = rng.normal(loc=0, scale=0.01, size=X.shape[1])\n",
    "        self.b_ = rng.normal(loc=0, scale=0.01, size=1)\n",
    "\n",
    "        # to collect errors per epoch\n",
    "        self.errors = []\n",
    "\n",
    "        # train loop\n",
    "        for epoch in range(self.epochs):\n",
    "            errors = 0\n",
    "            for x, y_true in zip(X, y):\n",
    "\n",
    "                # output of the perceptron\n",
    "                y_pred = self.predict(x)\n",
    "\n",
    "                # update w_ and b_\n",
    "                update_step = self.eta * (y_true - y_pred)\n",
    "                self.w_ += update_step * x  # equivalent to self.w_ = self.w_ + update_step * x\n",
    "                self.b_ += update_step      # equivalent to self.b_ = self.b_ + update_step\n",
    "\n",
    "                # count number of updates in the current epoch\n",
    "                if update_step != 0:\n",
    "                    errors += 1\n",
    "\n",
    "            self.errors.append(errors)\n",
    "\n",
    "    def predict(self, x: np.ndarray) -> np.ndarray:\n",
    "        output = np.dot(x, self.w_) + self.b_\n",
    "        unit_function = np.where(output >= 0, 1, 0)\n",
    "        return unit_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize a perceptron\n",
    "perceptron = Perceptron(eta=0.1, epochs=10)\n",
    "\n",
    "# fit dataset to the model\n",
    "perceptron.fit(X, y)\n",
    "\n",
    "# plot errors per epoch\n",
    "plt.plot(perceptron.errors, marker='o')\n",
    "plt.xticks(range(10))\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Number of updates')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot decision boundary\n",
    "plot_decision_regions(X, y, classifier=perceptron)\n",
    "plt.xlabel('Sepal length [cm]')\n",
    "plt.ylabel('Petal length [cm]')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adaptive Linear Neurons (AdaLiNe)\n",
    "   - The Adaline, developed by [Bernard Widrow](https://en.wikipedia.org/wiki/Bernard_Widrow) and [Ted Hoff](https://en.wikipedia.org/wiki/Marcian_Hoff) in 1960.\n",
    "   - It is a refinement of the perceptron, but it uses a different cost function and activation.\n",
    "\n",
    "<figure style=\"text-align: center;\">\n",
    "    <img src=\"../resources/images/SVGs/adaline.svg\" alt=\"adaline.png\" style=\"width: 80%;\">\n",
    "    <figcaption style=\"text-align: center;\">Adaptive Linear Neurons</figcaption>\n",
    "</figure>\n",
    "\n",
    "---\n",
    "\n",
    "**Training Rule (Widrow-Hoff Rule)**:\n",
    "   - Mean Squared Error (MSE) Loss:\n",
    "   $$\n",
    "   L(\\mathbf{w}, b) = \\frac{1}{2n} \\sum_{i=1}^{n} \\left( y^{(i)} - \\mathbf{w}^T \\mathbf{x}^{(i)} - b \\right)^2\n",
    "   $$\n",
    "   - Weights update:\n",
    "   $$\n",
    "   \\Delta \\mathbf{w} = -\\eta \\nabla_{w}L(\\mathbf{w}, b) = \\eta \\frac{1}{n} \\sum_{i=1}^{n} \\left( y^{(i)} - \\mathbf{w}^T \\mathbf{x}^{(i)} - b \\right) \\mathbf{x}^{(i)}\n",
    "   $$\n",
    "   $$\n",
    "   \\mathbf{w} := \\mathbf{w} + \\Delta \\mathbf{w}\n",
    "   $$\n",
    "   - Bias update:\n",
    "   $$\n",
    "   \\Delta b = -\\eta \\nabla_{b}L(\\mathbf{w}, b) = \\eta \\frac{1}{n} \\sum_{i=1}^{n} \\left( y^{(i)} - \\mathbf{w}^T \\mathbf{x}^{(i)} - b \\right)\n",
    "   $$\n",
    "   $$\n",
    "   b := b + \\Delta b\n",
    "   $$\n",
    "\n",
    "✍️ Note:\n",
    "   - The following code is adapted from the book *[Machine Learning with PyTorch and Scikit-Learn](https://github.com/rasbt/machine-learning-book)* with **modifications** made to fit the requirements of this analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdaLiNe:\n",
    "    def __init__(self, eta: int = 0.01, epochs: int = 50, seed: int = 42) -> None:\n",
    "        self.eta = eta\n",
    "        self.epochs = epochs\n",
    "        self.seed = seed\n",
    "\n",
    "    def fit(self, X: np.ndarray, y: np.ndarray) -> None:\n",
    "        # initialize weights and bias following a normal distribution with a deterministic seed\n",
    "        rng = np.random.default_rng(seed=self.seed)\n",
    "        self.w_ = rng.normal(loc=0, scale=0.01, size=X.shape[1])\n",
    "        self.b_ = rng.normal(loc=0, scale=0.01, size=1)\n",
    "\n",
    "        # to collect losses per epoch\n",
    "        self.losses_ = []\n",
    "\n",
    "        # training loop\n",
    "        for epoch in range(self.epochs):\n",
    "            \n",
    "            # output of the adaline (before passing to threshold function)\n",
    "            net_input = self.net_input(X)\n",
    "            output = self.activation(net_input)\n",
    "\n",
    "            # update w_ and b_\n",
    "            errors = (y - output)\n",
    "            self.w_ += self.eta * X.T.dot(errors) / X.shape[0]\n",
    "            self.b_ += self.eta * errors.mean()\n",
    "\n",
    "            # calculate loss function (MSE in this case)\n",
    "            loss = (errors**2).mean() / 2\n",
    "            self.losses_.append(loss)\n",
    "        return self\n",
    "\n",
    "    def net_input(self, X):\n",
    "        \"\"\"Calculate net input\"\"\"\n",
    "        return np.dot(X, self.w_) + self.b_\n",
    "    \n",
    "    def activation(self, X):\n",
    "        \"\"\"Compute linear activation\"\"\"\n",
    "        return X\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \"\"\"Return class label after unit step\"\"\"\n",
    "        return np.where(self.activation(self.net_input(X)) >= 0.5, 1, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize several adaline using different learning rates and standardizing inputs\n",
    "ada1 = AdaLiNe(epochs=20, eta=0.1).fit(X, y)\n",
    "ada2 = AdaLiNe(epochs=20, eta=0.0001).fit(X, y)\n",
    "\n",
    "# standardize inputs to have mean=0 and std=1\n",
    "X_std = np.copy(X)\n",
    "X_std[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std()\n",
    "X_std[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std()\n",
    "ada3 = AdaLiNe(epochs=20, eta=0.5).fit(X_std, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(20, 4))\n",
    "ax[0].plot(range(1, len(ada1.losses_) + 1), np.log10(ada1.losses_), marker='o')\n",
    "ax[0].set_xlabel('Epochs')\n",
    "ax[0].set_ylabel('log(Mean squared error)')\n",
    "ax[0].set_title('Adaline - Learning rate 0.1')\n",
    "ax[1].plot(range(1, len(ada2.losses_) + 1), ada2.losses_, marker='o')\n",
    "ax[1].set_xlabel('Epochs')\n",
    "ax[1].set_ylabel('Mean squared error')\n",
    "ax[1].set_title('Adaline - Learning rate 0.0001')\n",
    "ax[2].plot(range(1, len(ada3.losses_) + 1), ada3.losses_, marker='o')\n",
    "ax[2].set_xlabel('Epochs')\n",
    "ax[2].set_ylabel('Mean squared error')\n",
    "ax[2].set_title('Adaline - Learning rate 0.5 + Standardize')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot decision region\n",
    "fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(18, 6), layout='compressed')\n",
    "plt.sca(axes[0])\n",
    "plt.title('Adaline - Learning rate 0.1')\n",
    "plot_decision_regions(X, y, ada1)\n",
    "plt.sca(axes[1])\n",
    "plt.title('Adaline - Learning rate 0.0001')\n",
    "plot_decision_regions(X, y, ada2)\n",
    "plt.sca(axes[2])\n",
    "plt.title('Adaline - Learning rate 0.5 + Standardize')\n",
    "plot_decision_regions(X_std, y, ada3)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}