codes/06_logistic-regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "📝 **Author:** Amirhossein Heydari - 📧 **Email:** amirhosseinheydari78@gmail.com - 📍 **Linktree:** [linktr.ee/mr_pylin](https://linktr.ee/mr_pylin)\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression\n",
    "<div style=\"display: flex; margin-top: 50px;\">\n",
    "    <div style=\"width: 20%;\">\n",
    "        <table style=\"margin-left: auto; margin-right: auto;\">\n",
    "            <caption style=\"text-align: center;\">Dataset</caption>\n",
    "            <tr>\n",
    "                <th>#</th>\n",
    "                <th><span style=\"color: cyan;\">x<sub>1</span></th>\n",
    "                <th><span style=\"color: #FF9999;\">y</span></th>\n",
    "            </tr>\n",
    "            <tr>\n",
    "                <th><span style=\"color: #99DD00\">1</span></th>\n",
    "                <td>1</td>\n",
    "                <td>0</td>\n",
    "            </tr>\n",
    "            <tr>\n",
    "                <th><span style=\"color: #99DD00\">2</span></th>\n",
    "                <td>2</td>\n",
    "                <td>0</td>\n",
    "            </tr>\n",
    "            <tr>\n",
    "                <th><span style=\"color: #99DD00\">3</span></th>\n",
    "                <td>3</td>\n",
    "                <td>0</td>\n",
    "            </tr>\n",
    "            <tr>\n",
    "                <th><span style=\"color: #99DD00\">4</span></th>\n",
    "                <td>4</td>\n",
    "                <td>1</td>\n",
    "            </tr>\n",
    "            <tr>\n",
    "                <th><span style=\"color: #99DD00\">5</span></th>\n",
    "                <td>5</td>\n",
    "                <td>1</td>\n",
    "            </tr>\n",
    "            <tr>\n",
    "                <th><span style=\"color: #99DD00\">6</span></th>\n",
    "                <td>6</td>\n",
    "                <td>1</td>\n",
    "            </tr>\n",
    "        </table>\n",
    "    </div>\n",
    "    <div style=\"width: 80%;\">\n",
    "        <figure style=\"text-align: center;\">\n",
    "            <img src=\"../resources/images/SVGs/logistic-regression.svg\" alt=\"logistic-regression.svg\" style=\"width: 100%;\">\n",
    "            <figcaption style=\"text-align: center;\">Logistic Regression Model</figcaption>\n",
    "        </figure>\n",
    "    </div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate artificial data\n",
    "n_samples, n_features = 10, 2\n",
    "\n",
    "x, y = datasets.make_classification(\n",
    "    n_samples=n_samples,\n",
    "    n_features=n_features,\n",
    "    n_informative=2,\n",
    "    n_redundant=0,\n",
    "    n_clusters_per_class=1,\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "# convert numpy.ndarray to torch.Tensor\n",
    "x_train = torch.from_numpy(x.astype(np.float32))\n",
    "y_train = torch.from_numpy(y.astype(np.float32)).view(-1, 1)\n",
    "\n",
    "# plot\n",
    "plt.scatter(x[y == 0][:, 0], x[y == 0][:, 1], color='b', label='Class 0')\n",
    "plt.scatter(x[y == 1][:, 0], x[y == 1][:, 1], color='r', label='Class 1')\n",
    "plt.xlabel('x_1')\n",
    "plt.ylabel('x_2')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=2, out_features=1, bias=True)\n",
       "  (1): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# logistic regression model\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(n_features, 1),\n",
    "    torch.nn.Sigmoid()\n",
    ")\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot stuff\n",
    "state = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0 -> loss: 0.38442\n",
      "epoch: 1 -> loss: 0.34141\n",
      "epoch: 2 -> loss: 0.30671\n",
      "epoch: 3 -> loss: 0.27825\n",
      "epoch: 4 -> loss: 0.25456\n",
      "epoch: 5 -> loss: 0.23458\n"
     ]
    }
   ],
   "source": [
    "# initial weights [educational purpose]\n",
    "with torch.no_grad():\n",
    "    # b [w_0] = +1\n",
    "    model[0].bias[0].fill_(1)\n",
    "\n",
    "    # w_1 = -1\n",
    "    model[0].weight[0, 0].fill_(-1)\n",
    "\n",
    "    # w_2 = +1\n",
    "    model[0].weight[0, 1].fill_(1)\n",
    "\n",
    "# hyper parameters\n",
    "epoch = 6\n",
    "lr = .5\n",
    "criterion = torch.nn.BCELoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n",
    "\n",
    "# training loop\n",
    "model.train()\n",
    "for i in range(epoch):\n",
    "\n",
    "    # forward\n",
    "    y_pred = model(x_train)\n",
    "\n",
    "    # backward\n",
    "    loss = criterion(y_pred, y_train)\n",
    "    loss.backward()\n",
    "\n",
    "    # save new y_pred every 5 epochs [plot stuff]\n",
    "    state.append([model[0].weight.clone().detach().numpy(), model[0].bias.clone().detach().numpy()])\n",
    "\n",
    "    # update parameters\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    # log\n",
    "    print(f\"epoch: {i} -> loss: {loss.item():>7.5f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot\n",
    "fig, axs = plt.subplots(nrows=3, ncols=2, figsize=(12, 16), layout='compressed')\n",
    "\n",
    "for row in range(3):\n",
    "    for col in range(2):\n",
    "        axs[row, col].scatter(x[y == 0][:, 0], x[y == 0][:, 1], color='b', label='Class 0')\n",
    "        axs[row, col].scatter(x[y == 1][:, 0], x[y == 1][:, 1], color='r', label='Class 1')\n",
    "        axs[row, col].set(title=f\"epoch {row * 2 + col}, W: {state[row * 2 + col][0].squeeze()}, b: {state[row * 2 + col][1].squeeze():.3f}\",\n",
    "                          xlim=(x[:, 0].min() - 1, x[:, 0].max() + 1), ylim=(x[:, 1].min() - 1, x[:, 1].max() + 1))\n",
    "\n",
    "        # decision boundary\n",
    "        w, b = state[row * 2 + col]\n",
    "        slope = -w[0][0] / w[0][1]\n",
    "        intercept = -b[0] / w[0][1]\n",
    "        x_plot = np.array([np.min(x[:, 0]), np.max(x[:, 0])])\n",
    "        y_plot = slope * x_plot + intercept\n",
    "\n",
    "        axs[row, col].plot(x_plot, y_plot, color='g', linestyle='--', label='Decision Boundary')\n",
    "        axs[row, col].legend()\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}