unit04-multilayer-nets/4.4-dataloaders/4.4-dataloaders-part4-define-and-run.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4ea39cec-0612-4dda-89ec-70755fef67e6",
   "metadata": {},
   "source": [
    "# Unit 4.4: Defining Efficient Data Loaders\n",
    "\n",
    "## Part 4 (Code part 2 of 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2f2cbd7b-7193-4596-9b34-dde48cf29a2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.9.15\n",
      "IPython version      : 8.6.0\n",
      "\n",
      "matplotlib : 3.6.2\n",
      "numpy      : 1.23.4\n",
      "pandas     : 1.5.2\n",
      "torchvision: 0.14.0\n",
      "torch      : 1.13.0\n",
      "\n",
      "conda environment: dl-fundamentals\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p matplotlib,numpy,pandas,torchvision,torch --conda"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33ce53b7-e04e-4023-9860-9785b9b3b901",
   "metadata": {},
   "source": [
    "## 1) Defining the Dataset Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e7dae215-f3ff-45a5-bf0e-84d983626a48",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "\n",
    "\n",
    "class MyDataset(Dataset):\n",
    "    def __init__(self, csv_path, img_dir, transform=None):\n",
    "\n",
    "        df = pd.read_csv(csv_path)\n",
    "        self.img_dir = img_dir\n",
    "        self.transform = transform\n",
    "\n",
    "        # based on DataFrame columns\n",
    "        self.img_names = df[\"filepath\"]\n",
    "        self.labels = df[\"label\"]\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        img = Image.open(os.path.join(self.img_dir, self.img_names[index]))\n",
    "\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "\n",
    "        label = self.labels[index]\n",
    "        return img, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.labels.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dea3d05-638b-444e-8b53-756ff19e160f",
   "metadata": {},
   "source": [
    "## 2) Defining an optional batch visualization function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fca921eb-ca85-44fc-97b6-e6ac907db82a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def viz_batch_images(batch):\n",
    "\n",
    "    plt.figure(figsize=(8, 8))\n",
    "    plt.axis(\"off\")\n",
    "    plt.title(\"Training images\")\n",
    "    plt.imshow(\n",
    "        np.transpose(\n",
    "            vutils.make_grid(batch[0][:64], padding=2, normalize=True), (1, 2, 0)\n",
    "        )\n",
    "    )\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58d62288-d414-473d-b281-a287df0a71fa",
   "metadata": {},
   "source": [
    "## 3) Defining optional image transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "66a6ae1b-b549-4b8a-8e50-bedba63d6e19",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "\n",
    "data_transforms = {\n",
    "    \"train\": transforms.Compose(\n",
    "        [\n",
    "            transforms.Resize(32),\n",
    "            transforms.RandomCrop((28, 28)),\n",
    "            transforms.ToTensor(),\n",
    "            # normalize images to [-1, 1] range\n",
    "            transforms.Normalize((0.5,), (0.5,)),\n",
    "        ]\n",
    "    ),\n",
    "    \"test\": transforms.Compose(\n",
    "        [\n",
    "            transforms.Resize(32),\n",
    "            transforms.CenterCrop((28, 28)),\n",
    "            transforms.ToTensor(),\n",
    "            # normalize images to [-1, 1] range\n",
    "            transforms.Normalize((0.5,), (0.5,)),\n",
    "        ]\n",
    "    ),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa872422-2650-4639-91eb-b70e02acaac9",
   "metadata": {},
   "source": [
    "## 4) Defining the data loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7760203f-a60e-4f6f-9483-4b78587fe93a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_dataset = MyDataset(\n",
    "    csv_path=\"mnist-pngs/new_train.csv\",\n",
    "    img_dir=\"mnist-pngs/\",\n",
    "    transform=data_transforms[\"train\"],\n",
    ")\n",
    "\n",
    "train_loader = DataLoader(\n",
    "    dataset=train_dataset,\n",
    "    batch_size=32,\n",
    "    shuffle=True,  # want to shuffle the dataset\n",
    "    num_workers=0,  # number processes/CPUs to use\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3736c925-cb06-4bd1-b34b-a8b8e857fb0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dataset = MyDataset(\n",
    "    csv_path=\"mnist-pngs/new_val.csv\",\n",
    "    img_dir=\"mnist-pngs/\",\n",
    "    transform=data_transforms[\"test\"],\n",
    ")\n",
    "\n",
    "val_loader = DataLoader(\n",
    "    dataset=val_dataset,\n",
    "    batch_size=32,\n",
    "    shuffle=False,\n",
    "    num_workers=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "04ea739e-ba4e-4b33-8113-16c8bee73687",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = MyDataset(\n",
    "    csv_path=\"mnist-pngs/test.csv\",\n",
    "    img_dir=\"mnist-pngs/\",\n",
    "    transform=data_transforms[\"test\"],\n",
    ")\n",
    "\n",
    "test_loader = DataLoader(\n",
    "    dataset=test_dataset,\n",
    "    batch_size=32,\n",
    "    shuffle=False,\n",
    "    num_workers=0\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d378ee04-d824-4591-bcee-8f7a68b53cad",
   "metadata": {},
   "source": [
    "## 5) Testing the data loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d1a5518a-0d15-4ee8-8ec7-04b52922df1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Batch index: 0 | Batch size: 32 | x shape: torch.Size([32, 1, 28, 28]) | y shape: torch.Size([32])\n",
      " Batch index: 1 | Batch size: 32 | x shape: torch.Size([32, 1, 28, 28]) | y shape: torch.Size([32])\n",
      " Batch index: 2 | Batch size: 32 | x shape: torch.Size([32, 1, 28, 28]) | y shape: torch.Size([32])\n",
      "Labels from current batch: tensor([9, 1, 3, 2, 8, 4, 8, 3, 1, 3, 6, 5, 4, 9, 6, 2, 6, 0, 5, 8, 0, 5, 1, 0,\n",
      "        4, 2, 3, 8, 5, 5, 4, 6])\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "num_epochs = 1\n",
    "for epoch in range(num_epochs):\n",
    "\n",
    "    for batch_idx, (x, y) in enumerate(train_loader):\n",
    "        time.sleep(1)\n",
    "        if batch_idx >= 3:\n",
    "            break\n",
    "        print(\" Batch index:\", batch_idx, end=\"\")\n",
    "        print(\" | Batch size:\", y.shape[0], end=\"\")\n",
    "        print(\" | x shape:\", x.shape, end=\"\")\n",
    "        print(\" | y shape:\", y.shape)\n",
    "\n",
    "print(\"Labels from current batch:\", y)\n",
    "\n",
    "# Uncomment to visualize a data batch:\n",
    "# batch = next(iter(train_loader))\n",
    "# viz_batch_images(batch[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}