result.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9cxJcUVeLukX",
        "outputId": "8e9e7ba8-9bb2-4bdb-d965-fe3d6b872a7e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mon Jun  6 00:23:29 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   38C    P0    30W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kAAfJ3oVYd5G",
        "outputId": "0d9fb118-0f27-4897-cfc7-dbc65c5a8dfe"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "processor\t: 0\n",
            "vendor_id\t: GenuineIntel\n",
            "cpu family\t: 6\n",
            "model\t\t: 63\n",
            "model name\t: Intel(R) Xeon(R) CPU @ 2.30GHz\n",
            "stepping\t: 0\n",
            "microcode\t: 0x1\n",
            "cpu MHz\t\t: 2299.998\n",
            "cache size\t: 46080 KB\n",
            "physical id\t: 0\n"
          ]
        }
      ],
      "source": [
        "!head /proc/cpuinfo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hcHuyPrzYjn4",
        "outputId": "ea130978-10ee-4c92-c0d4-ea341d095049"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "MemTotal:       26692024 kB\n",
            "MemFree:        23537484 kB\n",
            "MemAvailable:   25416984 kB\n"
          ]
        }
      ],
      "source": [
        "!head -n 3 /proc/meminfo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oaf7YwhyZbzt",
        "outputId": "0d606383-10a6-49ff-97b2-5ef67421effa"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.11.0+cu113\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "\n",
        "print(torch.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EkLv89vZaSmH",
        "outputId": "dba48ce5-195a-4774-abd4-ab47a426da20"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Ubuntu 18.04.5 LTS\n"
          ]
        }
      ],
      "source": [
        "!cat /etc/issue.net"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fWQAsfrtidxN",
        "outputId": "90a56720-0574-434b-a78b-82a16e3a5bc6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "\n",
        "# Accessing My Google Drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xA9CDuubSMyK"
      },
      "source": [
        "# Hyperparameter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lp6xyiuI8MD1"
      },
      "outputs": [],
      "source": [
        "from datetime import datetime\n",
        "import torch\n",
        "import numpy as np\n",
        "import time\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "EPOCHS = 300\n",
        "BATCHSIZE = 256\n",
        "SEED = 128\n",
        "WEIGHTDECAY = 1e-4\n",
        "LEARNINGRATE = 1e-3\n",
        "MINLR=1e-7\n",
        "STEPSIZEUP = 20\n",
        "GAMMA_CYCLE=0.9\n",
        "PRINTFREQ = 20\n",
        "IMGSIZE = (25,18)\n",
        "DEFAULTPATH='/content/drive/MyDrive/CD1'\n",
        "MODELPATH='/content/drive/MyDrive/CD1/model'\n",
        "IMGPATH='/content/drive/MyDrive/CD1/img'\n",
        "\n",
        "MEAN = (0.4914, 0.4822, 0.4465)\n",
        "STD = (0.2023, 0.1994, 0.2010)\n",
        "\n",
        "# Only for cifar-10\n",
        "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
        "NUMCLASSES=len(classes)\n",
        "\n",
        "ALPHA1 = 0.7 ### cutmix alpha\n",
        "ALPHA2 = 1.2 ### mixup alpha\n",
        "\n",
        "LAMBDA1=0.9\n",
        "LAMBDA2=0.7\n",
        "LAMBDA3=0.7\n",
        "\n",
        "def fix_seed():\n",
        "    torch.manual_seed(SEED)\n",
        "    torch.cuda.manual_seed(SEED)\n",
        "    torch.cuda.manual_seed_all(SEED)\n",
        "    np.random.seed(SEED)\n",
        "\n",
        "def device():\n",
        "    return torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B26G6fbHSHiu"
      },
      "source": [
        "# Augments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q_TQP_D9-Mjk"
      },
      "outputs": [],
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "\n",
        "def mixup_criterion(criterion, pred, y_a, y_b, lam):\n",
        "    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n",
        "\n",
        "def rand_bbox(size, lam): # size : [B, C, W, H]\n",
        "    W = size[2] # 이미지의 width\n",
        "    H = size[3] # 이미지의 height\n",
        "    cut_rat = np.sqrt(1. - lam)  # 패치 크기의 비율 정하기\n",
        "    cut_w = int(W * cut_rat)  # 패치의 너비\n",
        "    cut_h = int(H * cut_rat)  # 패치의 높이\n",
        "\n",
        "    # 기존 이미지의 크기에서 랜덤하게 중간 좌표 추출\n",
        "    cx = np.random.randint(W)\n",
        "    cy = np.random.randint(H)\n",
        "\n",
        "    # 패치 부분에 대한 좌표값을 추출합니다.\n",
        "    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n",
        "    bby1 = np.clip(cy - cut_h // 2, 0, H)\n",
        "    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n",
        "    bby2 = np.clip(cy + cut_h // 2, 0, H)\n",
        "\n",
        "    return bbx1, bby1, bbx2, bby2\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ObxL-sEXSAgL"
      },
      "source": [
        "# Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HFQtpY3Y8UL4",
        "outputId": "06e4a6e0-d252-4a28-9520-c54cca2c3e32"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "True\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "### define parameter\n",
        "\n",
        "batch_size = BATCHSIZE\n",
        "mean = MEAN\n",
        "std = STD\n",
        "\n",
        "### gpu 사용여부\n",
        "device = torch.device('cuda')\n",
        "IsGPU = device==torch.device('cuda')\n",
        "print(IsGPU)\n",
        "\n",
        "### seed 고정\n",
        "fix_seed()\n",
        "\n",
        "### transformer\n",
        "\n",
        "transform = transforms.Compose([\n",
        "            ### cifar 10 size는 (32,32) 이므로 resize해줄 필요는 없음.\n",
        "            transforms.ToTensor(),\n",
        "            # transforms.Resize((256,256)),### 사진을 보기 위해서 임의로 resize했습니다.\n",
        "            transforms.Normalize(mean, std)\n",
        "])\n",
        "\n",
        "\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR10(root=f'{DEFAULTPATH}/train/', train=True,\n",
        "                                        download=True, transform=transform)\n",
        "\n",
        "train_size = int(len(trainset)*0.9)\n",
        "validation_size = int(len(trainset)*0.1)\n",
        "\n",
        "### train validation set 분리\n",
        "trainset, validationset = torch.utils.data.random_split(trainset,[train_size,validation_size])\n",
        "\n",
        "testset = torchvision.datasets.CIFAR10(root=f'{DEFAULTPATH}/test/', train=False,\n",
        "                                       download=True, transform=transform)\n",
        "\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n",
        "                                          shuffle=True, num_workers=2)\n",
        "\n",
        "validationloader = torch.utils.data.DataLoader(validationset,batch_size=batch_size,\n",
        "                                          shuffle=True, num_workers=2)\n",
        "\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n",
        "                                         shuffle=False,  num_workers=2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9yPy5L8FR4AJ"
      },
      "source": [
        "# Utils"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DT9tFovF8b94"
      },
      "outputs": [],
      "source": [
        "class AverageMeter(object):\n",
        "    r\"\"\"Computes and stores the average and current value\n",
        "    \"\"\"\n",
        "    def __init__(self, name, fmt=':f'):\n",
        "        self.name = name\n",
        "        self.fmt = fmt\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        self.val = 0\n",
        "        self.avg = 0\n",
        "        self.sum = 0\n",
        "        self.count = 0\n",
        "\n",
        "    def update(self, val, n=1):\n",
        "        self.val = val\n",
        "        self.sum += val * n\n",
        "        self.count += n\n",
        "        self.avg = self.sum / self.count\n",
        "\n",
        "    def __str__(self):\n",
        "        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'\n",
        "        return fmtstr.format(**self.__dict__)\n",
        "\n",
        "\n",
        "class ProgressMeter(object):\n",
        "    def __init__(self, num_batches, *meters, prefix=\"\"):\n",
        "        self.batch_fmtstr = self._get_batch_fmtstr(num_batches)\n",
        "        self.meters = meters\n",
        "        self.prefix = prefix\n",
        "\n",
        "    def print(self, batch):\n",
        "        entries = [self.prefix + self.batch_fmtstr.format(batch)]\n",
        "        entries += [str(meter) for meter in self.meters]\n",
        "        print('\\t'.join(entries))\n",
        "\n",
        "    def _get_batch_fmtstr(self, num_batches):\n",
        "        num_digits = len(str(num_batches // 1))\n",
        "        fmt = '{:' + str(num_digits) + 'd}'\n",
        "        return '[' + fmt + '/' + fmt.format(num_batches) + ']'\n",
        "\n",
        "\n",
        "def accuracy(output, target, topk=(1,)):\n",
        "    r\"\"\"Computes the accuracy over the $k$ top predictions for the specified values of k\n",
        "    \"\"\"\n",
        "    with torch.no_grad():\n",
        "        maxk = max(topk)\n",
        "        batch_size = target.size(0)\n",
        "\n",
        "        # _, pred = output.topk(maxk, 1, True, True)\n",
        "        # pred = pred.t()\n",
        "        # correct = pred.eq(target.view(1, -1).expand_as(pred))\n",
        "\n",
        "        # faster topk (ref: https://github.com/pytorch/pytorch/issues/22812)\n",
        "        _, idx = output.sort(descending=True)\n",
        "        pred = idx[:,:maxk]\n",
        "        pred = pred.t()\n",
        "        correct = pred.eq(target.view(1, -1).expand_as(pred))\n",
        "\n",
        "        res = []\n",
        "        for k in topk:\n",
        "            correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)\n",
        "            res.append(correct_k.mul_(100.0 / batch_size))\n",
        "        return res"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "svaQZUQMqVtL"
      },
      "source": [
        "# Resnet18\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f6yI00xqv7DX"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pBliYeQL-bfI"
      },
      "outputs": [],
      "source": [
        "### https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.model_zoo import load_url as load_state_dict_from_url\n",
        "import time\n",
        "\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "\n",
        "def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):\n",
        "    \"\"\"3x3 convolution with padding\"\"\"\n",
        "    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
        "                     padding=dilation, groups=groups, bias=False, dilation=dilation)\n",
        "\n",
        "def conv1x1(in_planes, out_planes, stride=1):\n",
        "    \"\"\"1x1 convolution\"\"\"\n",
        "    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n",
        "\n",
        "class BasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n",
        "                 base_width=64, dilation=1, norm_layer=None):\n",
        "        super(BasicBlock, self).__init__()\n",
        "        if norm_layer is None:\n",
        "            norm_layer = nn.BatchNorm2d\n",
        "        if groups != 1 or base_width != 64:\n",
        "            raise ValueError('BasicBlock only supports groups=1 and base_width=64')\n",
        "        if dilation > 1:\n",
        "            raise NotImplementedError(\"Dilation > 1 not supported in BasicBlock\")\n",
        "        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n",
        "        self.conv1 = conv3x3(inplanes, planes, stride)\n",
        "        self.bn1 = norm_layer(planes)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.conv2 = conv3x3(planes, planes)\n",
        "        self.bn2 = norm_layer(planes)\n",
        "        self.downsample = downsample\n",
        "        self.stride = stride\n",
        "\n",
        "    def forward(self, x):\n",
        "        identity = x\n",
        "\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "\n",
        "        if self.downsample is not None:\n",
        "            identity = self.downsample(x)\n",
        "\n",
        "        out += identity\n",
        "        out = self.relu(out)\n",
        "\n",
        "        return out\n",
        "\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "    def __init__(self, block, layers, num_classes=10, zero_init_residual=False,\n",
        "                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n",
        "                 norm_layer=None):\n",
        "        super(ResNet, self).__init__()\n",
        "        if norm_layer is None:\n",
        "            norm_layer = nn.BatchNorm2d\n",
        "        self._norm_layer = norm_layer\n",
        "\n",
        "        self.inplanes = 64\n",
        "        self.dilation = 1\n",
        "        if replace_stride_with_dilation is None:\n",
        "            # each element in the tuple indicates if we should replace\n",
        "            # the 2x2 stride with a dilated convolution instead\n",
        "            replace_stride_with_dilation = [False, False, False]\n",
        "        if len(replace_stride_with_dilation) != 3:\n",
        "            raise ValueError(\"replace_stride_with_dilation should be None \"\n",
        "                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n",
        "        self.groups = groups\n",
        "        self.base_width = width_per_group\n",
        "\n",
        "        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, # the first layer uses fp weights\n",
        "                               bias=False)\n",
        "        self.bn1 = norm_layer(self.inplanes)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
        "        self.layer1 = self._make_layer(block, 32, layers[0])\n",
        "        self.layer2 = self._make_layer(block, 64, layers[1], stride=2,\n",
        "                                       dilate=replace_stride_with_dilation[0])\n",
        "        self.layer3 = self._make_layer(block, 128, layers[2], stride=2,\n",
        "                                       dilate=replace_stride_with_dilation[1])\n",
        "        self.layer4 = self._make_layer(block, 256, layers[3], stride=2,\n",
        "                                       dilate=replace_stride_with_dilation[2])\n",
        "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
        "        self.fc = nn.Linear(256 * block.expansion, num_classes) # the last layer uses fp weights\n",
        "\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, nn.Conv2d):\n",
        "                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n",
        "            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n",
        "                nn.init.constant_(m.weight, 1)\n",
        "                nn.init.constant_(m.bias, 0)\n",
        "\n",
        "        # Zero-initialize the last BN in each residual branch,\n",
        "        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n",
        "        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n",
        "        if zero_init_residual:\n",
        "            for m in self.modules():\n",
        "                if isinstance(m, BasicBlock):\n",
        "                    nn.init.constant_(m.bn2.weight, 0)\n",
        "\n",
        "    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n",
        "        norm_layer = self._norm_layer\n",
        "        downsample = None\n",
        "        previous_dilation = self.dilation\n",
        "        if dilate:\n",
        "            self.dilation *= stride\n",
        "            stride = 1\n",
        "        if stride != 1 or self.inplanes != planes * block.expansion:\n",
        "            downsample = nn.Sequential(\n",
        "                conv1x1(self.inplanes, planes * block.expansion, stride),\n",
        "                norm_layer(planes * block.expansion),\n",
        "            )\n",
        "\n",
        "        layers = []\n",
        "        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n",
        "                            self.base_width, previous_dilation, norm_layer))\n",
        "        self.inplanes = planes * block.expansion\n",
        "        for _ in range(1, blocks):\n",
        "            layers.append(block(self.inplanes, planes, groups=self.groups,\n",
        "                                base_width=self.base_width, dilation=self.dilation,\n",
        "                                norm_layer=norm_layer))\n",
        "\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def _forward_impl(self, x):\n",
        "        # See note [TorchScript super()]\n",
        "        x = self.conv1(x)\n",
        "        x = self.bn1(x)\n",
        "        x = self.relu(x)\n",
        "        x = self.maxpool(x)\n",
        "\n",
        "        x = self.layer1(x)\n",
        "        x = self.layer2(x)\n",
        "        x = self.layer3(x)\n",
        "        x = self.layer4(x)\n",
        "\n",
        "        x = self.avgpool(x)\n",
        "        x = torch.flatten(x, 1)\n",
        "        x = self.fc(x)\n",
        "\n",
        "        return x\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self._forward_impl(x)\n",
        "\n",
        "def resnet18():\n",
        "    r\"\"\"ResNet-18 model from\n",
        "    `\"Deep Residual Learning for Image Recognition\" <https://arxiv.org/pdf/1512.03385.pdf>`_\n",
        "    \"\"\"\n",
        "    model = ResNet(BasicBlock, [2, 2, 2, 2])\n",
        "    return model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t89riAwEv-Kq"
      },
      "source": [
        "## MODELNAME"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D7WimL6bI6IQ"
      },
      "outputs": [],
      "source": [
        "MODELNAME = 'resnet18'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l1H1a0i0wAw4"
      },
      "source": [
        "## Train & Validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QN-DiXG8ABRS"
      },
      "outputs": [],
      "source": [
        "### main\n",
        "def main_main():\n",
        "    model = resnet18()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = main_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = main_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def main_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            temp_X_2 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam1 = LAMBDA1\n",
        "            lam2 = LAMBDA2\n",
        "            lam3 = LAMBDA3\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = input[shuffled_y,:,bbx1:bbx2, bby1:bby2]\n",
        "            temp_X_2[:,:,:,:]=temp_X_2[shuffled_y,:,:,:]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_1\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "            mixed_2 = lam3 * mixed_1 + (1 - lam3) * temp_X_2\n",
        "            mixed_2 = mixed_2.cuda() if IsGPU else mixed_2\n",
        "\n",
        "            # compute output\n",
        "            output = model(mixed_2)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam1*lam2*lam3+(1-lam2)*lam3)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def main_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qyNu4rQXJYXX"
      },
      "outputs": [],
      "source": [
        "### main\n",
        "def cutmixup_main():\n",
        "    model = resnet18()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)   \n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = cutmixup_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = cutmixup_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_cutmixup.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def cutmixup_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            temp_X_2 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam1 = LAMBDA1\n",
        "            lam2 = LAMBDA2\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "            temp_X_2[:,:,:,:]=temp_X_2[shuffled_y,:,:,:]\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_2\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "\n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = mixed_1[:,:,bbx1:bbx2, bby1:bby2]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "\n",
        "            # compute output\n",
        "            output = model(input)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,1-lam2+lam1*lam2)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def cutmixup_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KKTgjgZYAjWX"
      },
      "outputs": [],
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "###original\n",
        "def ori_main():\n",
        "    model = resnet18()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)   \n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = ori_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = ori_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_original.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def ori_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "        input = input.cuda() if IsGPU else input\n",
        "        target = target.cuda() if IsGPU else target\n",
        "\n",
        "        output = model(input)\n",
        "        loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def ori_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dd32NYIKA0gn"
      },
      "outputs": [],
      "source": [
        "def mixup_main():\n",
        "    model = resnet18()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = mixup_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = mixup_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_mixup.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "    \n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "\n",
        "\n",
        "def mixup_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam2 = LAMBDA2\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "\n",
        "            temp_X_1[:,:,:,:]=temp_X_1[shuffled_y,:,:,:]\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_1\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "\n",
        "            # compute output\n",
        "            output = model(mixed_1)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam2)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def mixup_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TLMe7ZbFBe70"
      },
      "outputs": [],
      "source": [
        "def cutmix_main():\n",
        "    model = resnet18()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = cutmix_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = cutmix_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_cutmix.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def cutmix_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "            \n",
        "            lam1 = LAMBDA1\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index] \n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = input[shuffled_y,:,bbx1:bbx2, bby1:bby2]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "\n",
        "            # compute output\n",
        "            output = model(input)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam1)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def cutmix_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "07t6KyjVwMIu"
      },
      "source": [
        "## Run"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "QD6w1Pq6k0gT",
        "outputId": "37e0a244-0e4f-4b3f-b397-46e52cb7ec8f"
      },
      "outputs": [],
      "source": [
        "print(\"--main--\")\n",
        "main_train_last_top1_acc,main_val_last_top1_acc,main_pytorch_total_params,main_train_loss_history,main_train_correct_history,main_validation_loss_history,main_validation_correct_history=main_main()\n",
        "print(\"--ori--\")\n",
        "original_train_last_top1_acc,original_val_last_top1_acc,original_pytorch_total_params,original_train_loss_history,original_train_correct_history,original_validation_loss_history,original_validation_correct_history=ori_main()\n",
        "print(\"--mixup--\")\n",
        "mixup_train_last_top1_acc,mixup_val_last_top1_acc,mixup_pytorch_total_params,mixup_train_loss_history,mixup_train_correct_history,mixup_validation_loss_history,mixup_validation_correct_history=mixup_main()\n",
        "print(\"--cutmix--\")\n",
        "cutmix_train_last_top1_acc,cutmix_val_last_top1_acc,cutmix_pytorch_total_params,cutmix_train_loss_history,cutmix_train_correct_history,cutmix_validation_loss_history,cutmix_validation_correct_history=cutmix_main()\n",
        "print(\"--cutmixup--\")\n",
        "cutmixup_train_last_top1_acc,cutmixup_val_last_top1_acc,cutmixup_pytorch_total_params,cutmixup_train_loss_history,cutmixup_train_correct_history,cutmixup_validation_loss_history,cutmixup_validation_correct_history=cutmixup_main()\n",
        "\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format('Model','Train Last Top1 Accuracy','Validation Last Top1 Accuracy','Total Parameters'))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}',original_train_last_top1_acc,original_val_last_top1_acc,original_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmix',cutmix_train_last_top1_acc,cutmix_val_last_top1_acc,cutmix_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+mixup',mixup_train_last_top1_acc,mixup_val_last_top1_acc,mixup_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmixup',cutmixup_train_last_top1_acc,cutmixup_val_last_top1_acc,cutmixup_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmix+mixup',main_train_last_top1_acc,main_val_last_top1_acc,main_pytorch_total_params))\n",
        "\n",
        "\n",
        "X1 = np.arange(len(main_train_correct_history))\n",
        "train_y1 = np.array(main_train_correct_history)\n",
        "val_y1 = np.array(main_validation_correct_history)\n",
        "train_ori_y1 = np.array(original_train_correct_history)\n",
        "val_ori_y1 = np.array(original_validation_correct_history)\n",
        "train_mixup_y1 = np.array(mixup_train_correct_history)\n",
        "val_mixup_y1 = np.array(mixup_validation_correct_history)\n",
        "train_cutmix_y1 = np.array(cutmix_train_correct_history)\n",
        "val_cutmix_y1 = np.array(cutmix_validation_correct_history)\n",
        "train_cutmixup_y1 = np.array(cutmixup_train_correct_history)\n",
        "val_cutmixup_y1 = np.array(cutmixup_validation_correct_history)\n",
        "\n",
        "X2 = np.arange(len(main_train_loss_history))\n",
        "train_y2 = np.array(main_train_loss_history)\n",
        "val_y2 = np.array(main_validation_loss_history)\n",
        "train_ori_y2 = np.array(original_train_loss_history)\n",
        "val_ori_y2 = np.array(original_validation_loss_history)\n",
        "train_mixup_y2 = np.array(mixup_train_loss_history)\n",
        "val_mixup_y2 = np.array(mixup_validation_loss_history)\n",
        "train_cutmix_y2 = np.array(cutmix_train_loss_history)\n",
        "val_cutmix_y2 = np.array(cutmix_validation_loss_history)\n",
        "train_cutmixup_y2 = np.array(cutmixup_train_loss_history)\n",
        "val_cutmixup_y2 = np.array(cutmixup_validation_loss_history)\n",
        "\n",
        "\n",
        "plt.figure(1,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_ori_y1,label=f\"{MODELNAME} Train Accuracy\",color='#3B7DB0', linestyle='-')\n",
        "plt.plot(X1,val_ori_y1,label=f\"{MODELNAME} Validation Accuracy\",color='#3B7DB0', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(Original)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_Original.png\")\n",
        "\n",
        "plt.figure(2,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_mixup_y1,label=f\"{MODELNAME} + mixup Train Accuracy\",color='#CBB162', linestyle='-')\n",
        "plt.plot(X1,val_mixup_y1,label=f\"{MODELNAME} + mixup Validation Accuracy\",color='#CBB162', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(Mixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_Mixup.png\")\n",
        "\n",
        "plt.figure(3,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_cutmix_y1,label=f\"{MODELNAME} + cutmix Train Accuracy\",color='#98DDDD', linestyle='-')\n",
        "plt.plot(X1,val_cutmix_y1,label=f\"{MODELNAME} + cutmix Validation Accuracy\",color='#98DDDD', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(CutMix)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_CutMix.png\")\n",
        "\n",
        "plt.figure(4,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_cutmixup_y1,label=f\"{MODELNAME} + cutmixup Train Accuracy\",color='#7F36A1', linestyle='-')\n",
        "plt.plot(X1,val_cutmixup_y1,label=f\"{MODELNAME} + cutmixup Validation Accuracy\",color='#7F36A1', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(CutMixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_CutMixup.png\")\n",
        "\n",
        "plt.figure(5,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_ori_y2,label=f\"{MODELNAME} Train Loss\",color='#3B7DB0', linestyle='-')\n",
        "plt.plot(X2,val_ori_y2,label=f\"{MODELNAME} Validation Loss\",color='#3B7DB0', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(Original)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_Original.png\")\n",
        "\n",
        "plt.figure(6,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_mixup_y2,label=f\"{MODELNAME} + mixup Train Loss\",color='#CBB162', linestyle='-')\n",
        "plt.plot(X2,val_mixup_y2,label=f\"{MODELNAME} + mixup Validation Loss\",color='#CBB162', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(Mixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_Mixup.png\")\n",
        "\n",
        "plt.figure(7,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_cutmix_y2,label=f\"{MODELNAME} + cutmix Train Loss\",color='#98DDDD', linestyle='-')\n",
        "plt.plot(X2,val_cutmix_y2,label=f\"{MODELNAME} + cutmix Validation Loss\",color='#98DDDD', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(CutMix)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_CutMix.png\")\n",
        "\n",
        "plt.figure(8,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_cutmixup_y2,label=f\"{MODELNAME} + cutmixup Train Loss\",color='#7F36A1', linestyle='-')\n",
        "plt.plot(X2,val_cutmixup_y2,label=f\"{MODELNAME} + cutmixup Validation Loss\",color='#7F36A1', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(CutMixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_CutMixup.png\")\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vjEv5bTWwJYW"
      },
      "source": [
        "## Test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xa9yb2QWBrZM",
        "outputId": "b48bf694-0f27-41a4-e0bf-c51e46bcd5a8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Evaluate...\n",
            "Accuracy of cutmix and mixup resnet18 on the 10000 test images: 74.95 %\n",
            "Accuracy of cutmix resnet18 on the 10000 test images: 73.15 %\n",
            "Accuracy of mixup resnet18 on the 10000 test images: 73.32 %\n",
            "Accuracy of original resnet18 on the 10000 test images: 65.46 %\n",
            "Accuracy of cutmixup resnet18 on the 10000 test images: 67.45 %\n"
          ]
        }
      ],
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "\n",
        "# import argparse\n",
        "from random import sample\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "def eval():\n",
        "    model1 = resnet18()\n",
        "    model2 = resnet18()\n",
        "    model3 = resnet18()\n",
        "    model4 = resnet18()\n",
        "    model5 = resnet18()\n",
        "    model1 = model1.cuda() if IsGPU else model1\n",
        "    model2 = model2.cuda() if IsGPU else model2\n",
        "    model3 = model3.cuda() if IsGPU else model3\n",
        "    model4 = model4.cuda() if IsGPU else model4\n",
        "    model5 = model5.cuda() if IsGPU else model5\n",
        "    model1.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}.pth'))\n",
        "    model2.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_cutmix.pth'))\n",
        "    model3.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_mixup.pth'))\n",
        "    model4.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_original.pth'))\n",
        "    model5.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_cutmixup.pth'))\n",
        "    \n",
        "    model1.eval()\n",
        "    model2.eval()\n",
        "    model3.eval()\n",
        "    model4.eval()\n",
        "    model5.eval()\n",
        "\n",
        "    total = 0\n",
        "    correct1 = 0\n",
        "    correct2 = 0\n",
        "    correct3 = 0\n",
        "    correct4 = 0\n",
        "    correct5 = 0\n",
        "    print('Evaluate...')\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(testloader):\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "            output1 = model1(input)\n",
        "            output2 = model2(input)\n",
        "            output3 = model3(input)\n",
        "            output4 = model4(input)\n",
        "            output5 = model5(input)\n",
        "            _, predictions1 = torch.max(output1, 1)\n",
        "            _, predictions2 = torch.max(output2, 1)\n",
        "            _, predictions3 = torch.max(output3, 1)\n",
        "            _, predictions4 = torch.max(output4, 1)\n",
        "            _, predictions5 = torch.max(output5, 1)\n",
        "            total += target.size(0)\n",
        "            correct1 += (predictions1 == target).sum().item()\n",
        "            correct2 += (predictions2 == target).sum().item()\n",
        "            correct3 += (predictions3 == target).sum().item()\n",
        "            correct4 += (predictions4 == target).sum().item()\n",
        "            correct5 += (predictions5 == target).sum().item()\n",
        "        \n",
        "\n",
        "    print(f'Accuracy of cutmix and mixup {MODELNAME} on the 10000 test images: {100 * correct1 / total:.2f} %')\n",
        "    print(f'Accuracy of cutmix {MODELNAME} on the 10000 test images: {100 * correct2 / total:.2f} %')\n",
        "    print(f'Accuracy of mixup {MODELNAME} on the 10000 test images: {100 * correct3 / total:.2f} %')\n",
        "    print(f'Accuracy of original {MODELNAME} on the 10000 test images: {100 * correct4 / total:.2f} %')\n",
        "    print(f'Accuracy of cutmixup {MODELNAME} on the 10000 test images: {100 * correct5 / total:.2f} %')\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    eval()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Eae14b2gqhZb"
      },
      "source": [
        "# DenseNet169\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-BWasuLuSU0K"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r3DSCGzep_vG"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.utils.checkpoint as cp\n",
        "from collections import OrderedDict\n",
        "from torch import Tensor\n",
        "from torch.jit.annotations import List\n",
        "\n",
        "class _DenseLayer(nn.Module):\n",
        "    def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, memory_efficient=False):\n",
        "        super(_DenseLayer, self).__init__()\n",
        "        self.add_module('norm1', nn.BatchNorm2d(num_input_features)),\n",
        "        self.add_module('relu1', nn.ReLU(inplace=True)),\n",
        "        self.add_module('conv1', nn.Conv2d(num_input_features, bn_size *\n",
        "                                           growth_rate, kernel_size=1, stride=1,\n",
        "                                           bias=False)),\n",
        "        self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)),\n",
        "        self.add_module('relu2', nn.ReLU(inplace=True)),\n",
        "        self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate,\n",
        "                                           kernel_size=3, stride=1, padding=1,\n",
        "                                           bias=False)),\n",
        "        self.drop_rate = float(drop_rate)\n",
        "        self.memory_efficient = memory_efficient\n",
        "\n",
        "    def bn_function(self, inputs):\n",
        "        # type: (List[Tensor]) -> Tensor\n",
        "        concated_features = torch.cat(inputs, 1)\n",
        "        bottleneck_output = self.conv1(self.relu1(self.norm1(concated_features)))  # noqa: T484\n",
        "        return bottleneck_output\n",
        "\n",
        "    # todo: rewrite when torchscript supports any\n",
        "    def any_requires_grad(self, input):\n",
        "        # type: (List[Tensor]) -> bool\n",
        "        for tensor in input:\n",
        "            if tensor.requires_grad:\n",
        "                return True\n",
        "        return False\n",
        "\n",
        "    @torch.jit.unused  # noqa: T484\n",
        "    def call_checkpoint_bottleneck(self, input):\n",
        "        # type: (List[Tensor]) -> Tensor\n",
        "        def closure(*inputs):\n",
        "            return self.bn_function(inputs)\n",
        "\n",
        "        return cp.checkpoint(closure, *input)\n",
        "\n",
        "    @torch.jit._overload_method  # noqa: F811\n",
        "    def forward(self, input):\n",
        "        # type: (List[Tensor]) -> (Tensor)\n",
        "        pass\n",
        "\n",
        "    @torch.jit._overload_method  # noqa: F811\n",
        "    def forward(self, input):\n",
        "        # type: (Tensor) -> (Tensor)\n",
        "        pass\n",
        "\n",
        "    # torchscript does not yet support *args, so we overload method\n",
        "    # allowing it to take either a List[Tensor] or single Tensor\n",
        "    def forward(self, input):  # noqa: F811\n",
        "        if isinstance(input, Tensor):\n",
        "            prev_features = [input]\n",
        "        else:\n",
        "            prev_features = input\n",
        "\n",
        "        if self.memory_efficient and self.any_requires_grad(prev_features):\n",
        "            if torch.jit.is_scripting():\n",
        "                raise Exception(\"Memory Efficient not supported in JIT\")\n",
        "\n",
        "            bottleneck_output = self.call_checkpoint_bottleneck(prev_features)\n",
        "        else:\n",
        "            bottleneck_output = self.bn_function(prev_features)\n",
        "\n",
        "        new_features = self.conv2(self.relu2(self.norm2(bottleneck_output)))\n",
        "        if self.drop_rate > 0:\n",
        "            new_features = F.dropout(new_features, p=self.drop_rate,\n",
        "                                     training=self.training)\n",
        "        return new_features\n",
        "\n",
        "\n",
        "class _DenseBlock(nn.ModuleDict):\n",
        "    _version = 2\n",
        "\n",
        "    def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, memory_efficient=False):\n",
        "        super(_DenseBlock, self).__init__()\n",
        "        for i in range(num_layers):\n",
        "            layer = _DenseLayer(\n",
        "                num_input_features + i * growth_rate,\n",
        "                growth_rate=growth_rate,\n",
        "                bn_size=bn_size,\n",
        "                drop_rate=drop_rate,\n",
        "                memory_efficient=memory_efficient,\n",
        "            )\n",
        "            self.add_module('denselayer%d' % (i + 1), layer)\n",
        "\n",
        "    def forward(self, init_features):\n",
        "        features = [init_features]\n",
        "        for name, layer in self.items():\n",
        "            new_features = layer(features)\n",
        "            features.append(new_features)\n",
        "        return torch.cat(features, 1)\n",
        "\n",
        "\n",
        "class _Transition(nn.Sequential):\n",
        "    def __init__(self, num_input_features, num_output_features):\n",
        "        super(_Transition, self).__init__()\n",
        "        self.add_module('norm', nn.BatchNorm2d(num_input_features))\n",
        "        self.add_module('relu', nn.ReLU(inplace=True))\n",
        "        self.add_module('conv', nn.Conv2d(num_input_features, num_output_features,\n",
        "                                          kernel_size=1, stride=1, bias=False))\n",
        "        self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2))\n",
        "\n",
        "\n",
        "class DenseNet(nn.Module):\n",
        "    r\"\"\"Densenet-BC model class, based on\n",
        "    `\"Densely Connected Convolutional Networks\" <https://arxiv.org/pdf/1608.06993.pdf>`_\n",
        "\n",
        "    Args:\n",
        "        growth_rate (int) - how many filters to add each layer (`k` in paper)\n",
        "        block_config (list of 4 ints) - how many layers in each pooling block\n",
        "        num_init_features (int) - the number of filters to learn in the first convolution layer\n",
        "        bn_size (int) - multiplicative factor for number of bottle neck layers\n",
        "          (i.e. bn_size * k features in the bottleneck layer)\n",
        "        drop_rate (float) - dropout rate after each dense layer\n",
        "        num_classes (int) - number of classification classes\n",
        "        memory_efficient (bool) - If True, uses checkpointing. Much more memory efficient,\n",
        "          but slower. Default: *False*. See `\"paper\" <https://arxiv.org/pdf/1707.06990.pdf>`_\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16),\n",
        "                 num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000, memory_efficient=False):\n",
        "\n",
        "        super(DenseNet, self).__init__()\n",
        "\n",
        "        # First convolution\n",
        "        self.features = nn.Sequential(OrderedDict([\n",
        "            ('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2,\n",
        "                                padding=3, bias=False)),\n",
        "            ('norm0', nn.BatchNorm2d(num_init_features)),\n",
        "            ('relu0', nn.ReLU(inplace=True)),\n",
        "            ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),\n",
        "        ]))\n",
        "\n",
        "        # Each denseblock\n",
        "        num_features = num_init_features\n",
        "        for i, num_layers in enumerate(block_config):\n",
        "            block = _DenseBlock(\n",
        "                num_layers=num_layers,\n",
        "                num_input_features=num_features,\n",
        "                bn_size=bn_size,\n",
        "                growth_rate=growth_rate,\n",
        "                drop_rate=drop_rate,\n",
        "                memory_efficient=memory_efficient\n",
        "            )\n",
        "            self.features.add_module('denseblock%d' % (i + 1), block)\n",
        "            num_features = num_features + num_layers * growth_rate\n",
        "            if i != len(block_config) - 1:\n",
        "                trans = _Transition(num_input_features=num_features,\n",
        "                                    num_output_features=num_features // 2)\n",
        "                self.features.add_module('transition%d' % (i + 1), trans)\n",
        "                num_features = num_features // 2\n",
        "\n",
        "        # Final batch norm\n",
        "        self.features.add_module('norm5', nn.BatchNorm2d(num_features))\n",
        "\n",
        "        # Linear layer\n",
        "        self.classifier = nn.Linear(num_features, num_classes)\n",
        "\n",
        "        # Official init from torch repo.\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, nn.Conv2d):\n",
        "                nn.init.kaiming_normal_(m.weight)\n",
        "            elif isinstance(m, nn.BatchNorm2d):\n",
        "                nn.init.constant_(m.weight, 1)\n",
        "                nn.init.constant_(m.bias, 0)\n",
        "            elif isinstance(m, nn.Linear):\n",
        "                nn.init.constant_(m.bias, 0)\n",
        "\n",
        "    def forward(self, x):\n",
        "        features = self.features(x)\n",
        "        out = F.relu(features, inplace=True)\n",
        "        out = F.adaptive_avg_pool2d(out, (1, 1))\n",
        "        out = torch.flatten(out, 1)\n",
        "        out = self.classifier(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "\n",
        "def _densenet(arch, growth_rate, block_config, num_init_features, progress,\n",
        "              **kwargs):\n",
        "    model = DenseNet(growth_rate, block_config, num_init_features, **kwargs)\n",
        "    return model\n",
        "\n",
        "\n",
        "\n",
        "def densenet169(progress=True, **kwargs):\n",
        "    return _densenet('densenet169', 32, (6, 12, 32, 32), 64, progress,\n",
        "                     **kwargs)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CQKhulH_SYD9"
      },
      "source": [
        "## MODELNAME"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DXF05rJYqvjZ"
      },
      "outputs": [],
      "source": [
        "MODELNAME = 'densenet169'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R66D6fxHSbco"
      },
      "source": [
        "## Train & Validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8Osbld3Kq9nh"
      },
      "outputs": [],
      "source": [
        "### main\n",
        "def main_main():\n",
        "    model = densenet169(num_classes=NUMCLASSES)\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)  \n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = main_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = main_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def main_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            temp_X_2 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam1 = np.random.normal(ALPHA1,0.01)\n",
        "            lam2 = np.random.beta(ALPHA2,ALPHA2)\n",
        "            lam3 = np.random.beta(ALPHA2,ALPHA2)\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = input[shuffled_y,:,bbx1:bbx2, bby1:bby2]\n",
        "            temp_X_2[:,:,:,:]=temp_X_2[shuffled_y,:,:,:]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_1\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "            mixed_2 = lam3 * mixed_1 + (1 - lam3) * temp_X_2\n",
        "            mixed_2 = mixed_2.cuda() if IsGPU else mixed_2\n",
        "\n",
        "            # compute output\n",
        "            output = model(mixed_2)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam1*lam2*lam3+(1-lam2)*lam3)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def main_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xa27M8_eNd5j"
      },
      "outputs": [],
      "source": [
        "### main\n",
        "def cutmixup_main():\n",
        "    model = densenet169(num_classes=NUMCLASSES)\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)   \n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = cutmixup_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = cutmixup_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_cutmixup.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def cutmixup_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            temp_X_2 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam1 = np.random.normal(ALPHA1,0.01) ##cutmix\n",
        "            lam2 = np.random.beta(ALPHA2,ALPHA2) ##mixup\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "            temp_X_2[:,:,:,:]=temp_X_2[shuffled_y,:,:,:]\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_2\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "\n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = mixed_1[:,:,bbx1:bbx2, bby1:bby2]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "\n",
        "            # compute output\n",
        "            output = model(input)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,1-lam2+lam1*lam2)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def cutmixup_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gkD3mD08rOY3"
      },
      "outputs": [],
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "###original\n",
        "def ori_main():\n",
        "    model = densenet169(num_classes=NUMCLASSES)\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)   \n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "    \n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = ori_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = ori_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_original.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def ori_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "        input = input.cuda() if IsGPU else input\n",
        "        target = target.cuda() if IsGPU else target\n",
        "\n",
        "        output = model(input)\n",
        "        loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def ori_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KOIzDrwyr40y"
      },
      "outputs": [],
      "source": [
        "def mixup_main():\n",
        "    model = densenet169(num_classes=NUMCLASSES)\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = mixup_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = mixup_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_mixup.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "    \n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "\n",
        "\n",
        "def mixup_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam2 = np.random.beta(ALPHA2,ALPHA2)\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "\n",
        "            temp_X_1[:,:,:,:]=temp_X_1[shuffled_y,:,:,:]\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_1\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "\n",
        "            # compute output\n",
        "            output = model(mixed_1)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam2)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def mixup_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2h66gq6TsB7Z"
      },
      "outputs": [],
      "source": [
        "def cutmix_main():\n",
        "    model = densenet169(num_classes=NUMCLASSES)\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = cutmix_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = cutmix_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_cutmix.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def cutmix_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam1 = np.random.normal(ALPHA1,0.01)\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = input[shuffled_y,:,bbx1:bbx2, bby1:bby2]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "\n",
        "            # compute output\n",
        "            output = model(input)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam1)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def cutmix_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Vdpy5mBSjc1"
      },
      "source": [
        "## Run"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Vp37H4RvsScR",
        "outputId": "a1144afb-bfc6-4d50-9540-111e93b0e2f3"
      },
      "outputs": [],
      "source": [
        "print(\"--main--\")\n",
        "main_train_last_top1_acc,main_val_last_top1_acc,main_pytorch_total_params,main_train_loss_history,main_train_correct_history,main_validation_loss_history,main_validation_correct_history=main_main()\n",
        "print(\"--ori--\")\n",
        "original_train_last_top1_acc,original_val_last_top1_acc,original_pytorch_total_params,original_train_loss_history,original_train_correct_history,original_validation_loss_history,original_validation_correct_history=ori_main()\n",
        "print(\"--mixup--\")\n",
        "mixup_train_last_top1_acc,mixup_val_last_top1_acc,mixup_pytorch_total_params,mixup_train_loss_history,mixup_train_correct_history,mixup_validation_loss_history,mixup_validation_correct_history=mixup_main()\n",
        "print(\"--cutmix--\")\n",
        "cutmix_train_last_top1_acc,cutmix_val_last_top1_acc,cutmix_pytorch_total_params,cutmix_train_loss_history,cutmix_train_correct_history,cutmix_validation_loss_history,cutmix_validation_correct_history=cutmix_main()\n",
        "print(\"--cutmixup--\")\n",
        "cutmixup_train_last_top1_acc,cutmixup_val_last_top1_acc,cutmixup_pytorch_total_params,cutmixup_train_loss_history,cutmixup_train_correct_history,cutmixup_validation_loss_history,cutmixup_validation_correct_history=cutmixup_main()\n",
        "\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format('Model','Train Last Top1 Accuracy','Validation Last Top1 Accuracy','Total Parameters'))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}',original_train_last_top1_acc,original_val_last_top1_acc,original_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmix',cutmix_train_last_top1_acc,cutmix_val_last_top1_acc,cutmix_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+mixup',mixup_train_last_top1_acc,mixup_val_last_top1_acc,mixup_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmixup',cutmixup_train_last_top1_acc,cutmixup_val_last_top1_acc,cutmixup_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmix+mixup',main_train_last_top1_acc,main_val_last_top1_acc,main_pytorch_total_params))\n",
        "\n",
        "\n",
        "X1 = np.arange(len(main_train_correct_history))\n",
        "train_y1 = np.array(main_train_correct_history)\n",
        "val_y1 = np.array(main_validation_correct_history)\n",
        "train_ori_y1 = np.array(original_train_correct_history)\n",
        "val_ori_y1 = np.array(original_validation_correct_history)\n",
        "train_mixup_y1 = np.array(mixup_train_correct_history)\n",
        "val_mixup_y1 = np.array(mixup_validation_correct_history)\n",
        "train_cutmix_y1 = np.array(cutmix_train_correct_history)\n",
        "val_cutmix_y1 = np.array(cutmix_validation_correct_history)\n",
        "train_cutmixup_y1 = np.array(cutmixup_train_correct_history)\n",
        "val_cutmixup_y1 = np.array(cutmixup_validation_correct_history)\n",
        "\n",
        "X2 = np.arange(len(main_train_loss_history))\n",
        "train_y2 = np.array(main_train_loss_history)\n",
        "val_y2 = np.array(main_validation_loss_history)\n",
        "train_ori_y2 = np.array(original_train_loss_history)\n",
        "val_ori_y2 = np.array(original_validation_loss_history)\n",
        "train_mixup_y2 = np.array(mixup_train_loss_history)\n",
        "val_mixup_y2 = np.array(mixup_validation_loss_history)\n",
        "train_cutmix_y2 = np.array(cutmix_train_loss_history)\n",
        "val_cutmix_y2 = np.array(cutmix_validation_loss_history)\n",
        "train_cutmixup_y2 = np.array(cutmixup_train_loss_history)\n",
        "val_cutmixup_y2 = np.array(cutmixup_validation_loss_history)\n",
        "\n",
        "\n",
        "plt.figure(1,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_ori_y1,label=f\"{MODELNAME} Train Accuracy\",color='#3B7DB0', linestyle='-')\n",
        "plt.plot(X1,val_ori_y1,label=f\"{MODELNAME} Validation Accuracy\",color='#3B7DB0', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(Original)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_Original.png\")\n",
        "\n",
        "plt.figure(2,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_mixup_y1,label=f\"{MODELNAME} + mixup Train Accuracy\",color='#CBB162', linestyle='-')\n",
        "plt.plot(X1,val_mixup_y1,label=f\"{MODELNAME} + mixup Validation Accuracy\",color='#CBB162', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(Mixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_Mixup.png\")\n",
        "\n",
        "plt.figure(3,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_cutmix_y1,label=f\"{MODELNAME} + cutmix Train Accuracy\",color='#98DDDD', linestyle='-')\n",
        "plt.plot(X1,val_cutmix_y1,label=f\"{MODELNAME} + cutmix Validation Accuracy\",color='#98DDDD', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(CutMix)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_CutMix.png\")\n",
        "\n",
        "plt.figure(4,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_cutmixup_y1,label=f\"{MODELNAME} + cutmixup Train Accuracy\",color='#7F36A1', linestyle='-')\n",
        "plt.plot(X1,val_cutmixup_y1,label=f\"{MODELNAME} + cutmixup Validation Accuracy\",color='#7F36A1', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(CutMixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_CutMixup.png\")\n",
        "\n",
        "plt.figure(5,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_ori_y2,label=f\"{MODELNAME} Train Loss\",color='#3B7DB0', linestyle='-')\n",
        "plt.plot(X2,val_ori_y2,label=f\"{MODELNAME} Validation Loss\",color='#3B7DB0', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(Original)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_Original.png\")\n",
        "\n",
        "plt.figure(6,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_mixup_y2,label=f\"{MODELNAME} + mixup Train Loss\",color='#CBB162', linestyle='-')\n",
        "plt.plot(X2,val_mixup_y2,label=f\"{MODELNAME} + mixup Validation Loss\",color='#CBB162', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(Mixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_Mixup.png\")\n",
        "\n",
        "plt.figure(7,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_cutmix_y2,label=f\"{MODELNAME} + cutmix Train Loss\",color='#98DDDD', linestyle='-')\n",
        "plt.plot(X2,val_cutmix_y2,label=f\"{MODELNAME} + cutmix Validation Loss\",color='#98DDDD', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(CutMix)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_CutMix.png\")\n",
        "\n",
        "plt.figure(8,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_cutmixup_y2,label=f\"{MODELNAME} + cutmixup Train Loss\",color='#7F36A1', linestyle='-')\n",
        "plt.plot(X2,val_cutmixup_y2,label=f\"{MODELNAME} + cutmixup Validation Loss\",color='#7F36A1', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(CutMixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_CutMixup.png\")\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-q8vxUmKSoLq"
      },
      "source": [
        "## Test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lyfm5H6asnpu",
        "outputId": "b701d925-6f4d-4026-d9e8-cab8bdc898be"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Evaluate...\n",
            "Accuracy of cutmix and mixup densenet169 on the 10000 test images: 77.95 %\n",
            "Accuracy of cutmix densenet169 on the 10000 test images: 77.72 %\n",
            "Accuracy of mixup densenet169 on the 10000 test images: 77.07 %\n",
            "Accuracy of original densenet169 on the 10000 test images: 71.45 %\n",
            "Accuracy of cutmixup densenet169 on the 10000 test images: 76.87 %\n"
          ]
        }
      ],
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "\n",
        "# import argparse\n",
        "from random import sample\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "def eval():\n",
        "    model1 = densenet169(num_classes=NUMCLASSES)\n",
        "    model2 = densenet169(num_classes=NUMCLASSES)\n",
        "    model3 = densenet169(num_classes=NUMCLASSES)\n",
        "    model4 = densenet169(num_classes=NUMCLASSES)\n",
        "    model5 = densenet169(num_classes=NUMCLASSES)\n",
        "    model1 = model1.cuda() if IsGPU else model1\n",
        "    model2 = model2.cuda() if IsGPU else model2\n",
        "    model3 = model3.cuda() if IsGPU else model3\n",
        "    model4 = model4.cuda() if IsGPU else model4\n",
        "    model5 = model5.cuda() if IsGPU else model5\n",
        "    model1.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}.pth'))\n",
        "    model2.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_cutmix.pth'))\n",
        "    model3.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_mixup.pth'))\n",
        "    model4.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_original.pth'))\n",
        "    model5.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_cutmixup.pth'))\n",
        "    \n",
        "    model1.eval()\n",
        "    model2.eval()\n",
        "    model3.eval()\n",
        "    model4.eval()\n",
        "    model5.eval()\n",
        "\n",
        "    total = 0\n",
        "    correct1 = 0\n",
        "    correct2 = 0\n",
        "    correct3 = 0\n",
        "    correct4 = 0\n",
        "    correct5 = 0\n",
        "    print('Evaluate...')\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(testloader):\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "            output1 = model1(input)\n",
        "            output2 = model2(input)\n",
        "            output3 = model3(input)\n",
        "            output4 = model4(input)\n",
        "            output5 = model5(input)\n",
        "            _, predictions1 = torch.max(output1, 1)\n",
        "            _, predictions2 = torch.max(output2, 1)\n",
        "            _, predictions3 = torch.max(output3, 1)\n",
        "            _, predictions4 = torch.max(output4, 1)\n",
        "            _, predictions5 = torch.max(output5, 1)\n",
        "            total += target.size(0)\n",
        "            correct1 += (predictions1 == target).sum().item()\n",
        "            correct2 += (predictions2 == target).sum().item()\n",
        "            correct3 += (predictions3 == target).sum().item()\n",
        "            correct4 += (predictions4 == target).sum().item()\n",
        "            correct5 += (predictions5 == target).sum().item()\n",
        "        \n",
        "\n",
        "    print(f'Accuracy of cutmix and mixup {MODELNAME} on the 10000 test images: {100 * correct1 / total:.2f} %')\n",
        "    print(f'Accuracy of cutmix {MODELNAME} on the 10000 test images: {100 * correct2 / total:.2f} %')\n",
        "    print(f'Accuracy of mixup {MODELNAME} on the 10000 test images: {100 * correct3 / total:.2f} %')\n",
        "    print(f'Accuracy of original {MODELNAME} on the 10000 test images: {100 * correct4 / total:.2f} %')\n",
        "    print(f'Accuracy of cutmixup {MODELNAME} on the 10000 test images: {100 * correct5 / total:.2f} %')\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    eval()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zUtvkZyJ_Rkh"
      },
      "source": [
        "# resnet34"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZhJcMaiiSwHz"
      },
      "source": [
        "## MODEL"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EyzAFfCB_QCg"
      },
      "outputs": [],
      "source": [
        "def resnet34():\n",
        "    r\"\"\"ResNet-18 model from\n",
        "    `\"Deep Residual Learning for Image Recognition\" <https://arxiv.org/pdf/1512.03385.pdf>`_\n",
        "    \"\"\"\n",
        "    model = ResNet(BasicBlock, [3, 4, 6, 3])\n",
        "    return model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2MxeXR7MSymh"
      },
      "source": [
        "## MODELNAME"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rBcN39xe_Uvk"
      },
      "outputs": [],
      "source": [
        "MODELNAME='resnet34'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gz36EFXLS0pc"
      },
      "source": [
        "## Train & Validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q1of8qh7_uUH"
      },
      "outputs": [],
      "source": [
        "### main\n",
        "def main_main():\n",
        "    model = resnet34()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = main_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = main_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def main_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            temp_X_2 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam1 = LAMBDA1\n",
        "            lam2 = LAMBDA2\n",
        "            lam3 = LAMBDA3\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = input[shuffled_y,:,bbx1:bbx2, bby1:bby2]\n",
        "            temp_X_2[:,:,:,:]=temp_X_2[shuffled_y,:,:,:]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_1\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "            mixed_2 = lam3 * mixed_1 + (1 - lam3) * temp_X_2\n",
        "            mixed_2 = mixed_2.cuda() if IsGPU else mixed_2\n",
        "\n",
        "            # compute output\n",
        "            output = model(mixed_2)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam1*lam2*lam3+(1-lam2)*lam3)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def main_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1uxjnN-rNs5N"
      },
      "outputs": [],
      "source": [
        "### main\n",
        "def cutmixup_main():\n",
        "    model = resnet34()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = cutmixup_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = cutmixup_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_cutmixup.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def cutmixup_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            temp_X_2 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam1 = LAMBDA1\n",
        "            lam2 = LAMBDA2\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "            temp_X_2[:,:,:,:]=temp_X_2[shuffled_y,:,:,:]\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_2\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "\n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = mixed_1[:,:,bbx1:bbx2, bby1:bby2]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "\n",
        "            # compute output\n",
        "            output = model(input)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,1-lam2+lam1*lam2)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def cutmixup_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l1saVqwgADN-"
      },
      "outputs": [],
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "###original\n",
        "def ori_main():\n",
        "    model = resnet34()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = ori_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = ori_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_original.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def ori_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "        input = input.cuda() if IsGPU else input\n",
        "        target = target.cuda() if IsGPU else target\n",
        "\n",
        "        output = model(input)\n",
        "        loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def ori_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o2MZ6RnuANDv"
      },
      "outputs": [],
      "source": [
        "def mixup_main():\n",
        "    model = resnet34()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = mixup_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = mixup_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_mixup.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "    \n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "\n",
        "\n",
        "def mixup_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            temp_X_1 = input.clone().detach().cuda() if IsGPU else input.clone().detach()\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam2 = LAMBDA2\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "\n",
        "            temp_X_1[:,:,:,:]=temp_X_1[shuffled_y,:,:,:]\n",
        "            mixed_1 = lam2 * input + (1 - lam2) * temp_X_1\n",
        "            mixed_1 = mixed_1.cuda() if IsGPU else mixed_1\n",
        "\n",
        "            # compute output\n",
        "            output = model(mixed_1)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam2)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def mixup_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hqFn5iOzAZ2s"
      },
      "outputs": [],
      "source": [
        "def cutmix_main():\n",
        "    model = resnet34()\n",
        "\n",
        "    ##### optimizer / learning rate scheduler / criterion #####\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNINGRATE,\n",
        "                                weight_decay=WEIGHTDECAY)\n",
        "    scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=MINLR, \n",
        "                                            step_size_up=STEPSIZEUP, max_lr=LEARNINGRATE, \n",
        "                                            gamma=GAMMA_CYCLE, mode='triangular2',cycle_momentum=False)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    ###########################################################\n",
        "\n",
        "    model = model.cuda() if IsGPU else model\n",
        "    criterion = criterion.cuda() if IsGPU else criterion\n",
        "\n",
        "    # Check number of parameters your model\n",
        "    pytorch_total_params = sum(p.numel() for p in model.parameters())\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    train_last_top1_acc = 0\n",
        "    val_last_top1_acc = 0\n",
        "    train_loss_history = []\n",
        "    train_correct_history = []\n",
        "    validation_loss_history = []\n",
        "    validation_correct_history = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        print(\"\\n----- epoch: {}, lr: {} -----\".format(\n",
        "            epoch, optimizer.param_groups[0][\"lr\"]))\n",
        "        \n",
        "        # train for one epoch\n",
        "        start_time = time.time()\n",
        "        train_last_top1_acc, train_loss = cutmix_training(trainloader, epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to train this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "        \n",
        "        # validate for one epoch\n",
        "        start_time = time.time()\n",
        "        val_last_top1_acc, val_loss = cutmix_validating(validationloader,epoch, model, optimizer, criterion)\n",
        "        elapsed_time = time.time() - start_time\n",
        "        print('==> {:.2f} seconds to validate this epoch\\n'.format(\n",
        "            elapsed_time))\n",
        "\n",
        "        # learning rate scheduling\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss_history.append(train_loss)\n",
        "        train_correct_history.append(train_last_top1_acc)\n",
        "        validation_loss_history.append(val_loss)\n",
        "        validation_correct_history.append(val_last_top1_acc)\n",
        "\n",
        "        # Save model each epoch\n",
        "        torch.save(model.state_dict(), f'{MODELPATH}/model_weight_{MODELNAME}_cutmix.pth')\n",
        "\n",
        "    print(f\"Train Last Top-1 Accuracy: {train_last_top1_acc}\")\n",
        "    print(f\"Validation Last Top-1 Accuracy: {val_last_top1_acc}\")\n",
        "    print(f\"Number of parameters: {pytorch_total_params}\")\n",
        "\n",
        "    return train_last_top1_acc,val_last_top1_acc,pytorch_total_params,train_loss_history,train_correct_history,validation_loss_history,validation_correct_history\n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "def cutmix_training(train_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(train_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    torch.cuda.empty_cache()\n",
        "    model.train()\n",
        "\n",
        "    end = time.time()\n",
        "    for i, (input, target) in enumerate(train_loader):\n",
        "        # measure data loading time\n",
        "        data_time.update(time.time() - end)\n",
        "\n",
        "        if i % np.random.randint(1,6) == 0:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            lam1 = LAMBDA1\n",
        "            rand_index = torch.randperm(input.size()[0]).cuda() if IsGPU else torch.randperm(input.size()[0])\n",
        "            shuffled_y = target[rand_index]\n",
        "\n",
        "            bbx1, bby1, bbx2, bby2 = rand_bbox(input.size(), lam1)\n",
        "            input[:,:,bbx1:bbx2, bby1:bby2] = input[shuffled_y,:,bbx1:bbx2, bby1:bby2]\n",
        "            lam1 = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (input.size()[-1] * input.size()[-2]))\n",
        "\n",
        "            # compute output\n",
        "            output = model(input)\n",
        "            loss = mixup_criterion(criterion,output,target,shuffled_y,lam1)\n",
        "\n",
        "        else:\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "\n",
        "        # measure accuracy and record loss, accuracy \n",
        "        acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "        losses.update(loss.item(), input.size(0))\n",
        "        top1.update(acc1[0].item(), input.size(0))\n",
        "        top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "        # compute gradient and do Adam step\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # measure elapsed time\n",
        "        batch_time.update(time.time() - end)\n",
        "        end = time.time()\n",
        "\n",
        "        if i % PRINTFREQ == 0:\n",
        "            progress.print(i)\n",
        "\n",
        "    print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "          .format(top1=top1, top5=top5))\n",
        "    return top1.avg, losses.avg\n",
        "\n",
        "def cutmix_validating(val_loader, epoch, model, optimizer, criterion):\n",
        "    batch_time = AverageMeter('Time', ':6.3f')\n",
        "    data_time = AverageMeter('Data', ':6.3f')\n",
        "    losses = AverageMeter('Loss', ':.4e')\n",
        "    top1 = AverageMeter('Acc@1', ':6.2f')\n",
        "    top5 = AverageMeter('Acc@5', ':6.2f')\n",
        "    progress = ProgressMeter(len(val_loader), batch_time, data_time, losses,\n",
        "                             top1, top5, prefix=\"Epoch: [{}]\".format(epoch))\n",
        "    # switch to train mode\n",
        "    model.eval()\n",
        "\n",
        "    end = time.time()\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(val_loader):\n",
        "            # measure data loading time\n",
        "            data_time.update(time.time() - end)\n",
        "\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "\n",
        "            output = model(input)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # measure accuracy and record loss, accuracy \n",
        "            acc1, acc5 = accuracy(output, target, topk=(1, 5))\n",
        "            losses.update(loss.item(), input.size(0))\n",
        "            top1.update(acc1[0].item(), input.size(0))\n",
        "            top5.update(acc5[0].item(), input.size(0))\n",
        "\n",
        "            # measure elapsed time\n",
        "            batch_time.update(time.time() - end)\n",
        "            end = time.time()\n",
        "\n",
        "            if i % PRINTFREQ == 0:\n",
        "                progress.print(i)\n",
        "\n",
        "        print('=> Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'\n",
        "            .format(top1=top1, top5=top5))\n",
        "        return top1.avg, losses.avg\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YJH1n3W8TFSg"
      },
      "source": [
        "## Run"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "ZG3D28MIAi8W",
        "outputId": "775e7ed1-8c29-4ca5-c606-8c6dccaf09b5"
      },
      "outputs": [],
      "source": [
        "print(\"--main--\")\n",
        "main_train_last_top1_acc,main_val_last_top1_acc,main_pytorch_total_params,main_train_loss_history,main_train_correct_history,main_validation_loss_history,main_validation_correct_history=main_main()\n",
        "print(\"--ori--\")\n",
        "original_train_last_top1_acc,original_val_last_top1_acc,original_pytorch_total_params,original_train_loss_history,original_train_correct_history,original_validation_loss_history,original_validation_correct_history=ori_main()\n",
        "print(\"--mixup--\")\n",
        "mixup_train_last_top1_acc,mixup_val_last_top1_acc,mixup_pytorch_total_params,mixup_train_loss_history,mixup_train_correct_history,mixup_validation_loss_history,mixup_validation_correct_history=mixup_main()\n",
        "print(\"--cutmix--\")\n",
        "cutmix_train_last_top1_acc,cutmix_val_last_top1_acc,cutmix_pytorch_total_params,cutmix_train_loss_history,cutmix_train_correct_history,cutmix_validation_loss_history,cutmix_validation_correct_history=cutmix_main()\n",
        "print(\"--cutmixup--\")\n",
        "cutmixup_train_last_top1_acc,cutmixup_val_last_top1_acc,cutmixup_pytorch_total_params,cutmixup_train_loss_history,cutmixup_train_correct_history,cutmixup_validation_loss_history,cutmixup_validation_correct_history=cutmixup_main()\n",
        "\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format('Model','Train Last Top1 Accuracy','Validation Last Top1 Accuracy','Total Parameters'))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}',original_train_last_top1_acc,original_val_last_top1_acc,original_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmix',cutmix_train_last_top1_acc,cutmix_val_last_top1_acc,cutmix_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+mixup',mixup_train_last_top1_acc,mixup_val_last_top1_acc,mixup_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmixup',cutmixup_train_last_top1_acc,cutmixup_val_last_top1_acc,cutmixup_pytorch_total_params))\n",
        "print (\"{:<30} {:<30} {:<30} {:<30}\".format(f'{MODELNAME}+cutmix+mixup',main_train_last_top1_acc,main_val_last_top1_acc,main_pytorch_total_params))\n",
        "\n",
        "\n",
        "X1 = np.arange(len(main_train_correct_history))\n",
        "train_y1 = np.array(main_train_correct_history)\n",
        "val_y1 = np.array(main_validation_correct_history)\n",
        "train_ori_y1 = np.array(original_train_correct_history)\n",
        "val_ori_y1 = np.array(original_validation_correct_history)\n",
        "train_mixup_y1 = np.array(mixup_train_correct_history)\n",
        "val_mixup_y1 = np.array(mixup_validation_correct_history)\n",
        "train_cutmix_y1 = np.array(cutmix_train_correct_history)\n",
        "val_cutmix_y1 = np.array(cutmix_validation_correct_history)\n",
        "train_cutmixup_y1 = np.array(cutmixup_train_correct_history)\n",
        "val_cutmixup_y1 = np.array(cutmixup_validation_correct_history)\n",
        "\n",
        "X2 = np.arange(len(main_train_loss_history))\n",
        "train_y2 = np.array(main_train_loss_history)\n",
        "val_y2 = np.array(main_validation_loss_history)\n",
        "train_ori_y2 = np.array(original_train_loss_history)\n",
        "val_ori_y2 = np.array(original_validation_loss_history)\n",
        "train_mixup_y2 = np.array(mixup_train_loss_history)\n",
        "val_mixup_y2 = np.array(mixup_validation_loss_history)\n",
        "train_cutmix_y2 = np.array(cutmix_train_loss_history)\n",
        "val_cutmix_y2 = np.array(cutmix_validation_loss_history)\n",
        "train_cutmixup_y2 = np.array(cutmixup_train_loss_history)\n",
        "val_cutmixup_y2 = np.array(cutmixup_validation_loss_history)\n",
        "\n",
        "\n",
        "plt.figure(1,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_ori_y1,label=f\"{MODELNAME} Train Accuracy\",color='#3B7DB0', linestyle='-')\n",
        "plt.plot(X1,val_ori_y1,label=f\"{MODELNAME} Validation Accuracy\",color='#3B7DB0', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(Original)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_Original.png\")\n",
        "\n",
        "plt.figure(2,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_mixup_y1,label=f\"{MODELNAME} + mixup Train Accuracy\",color='#CBB162', linestyle='-')\n",
        "plt.plot(X1,val_mixup_y1,label=f\"{MODELNAME} + mixup Validation Accuracy\",color='#CBB162', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(Mixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_Mixup.png\")\n",
        "\n",
        "plt.figure(3,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_cutmix_y1,label=f\"{MODELNAME} + cutmix Train Accuracy\",color='#98DDDD', linestyle='-')\n",
        "plt.plot(X1,val_cutmix_y1,label=f\"{MODELNAME} + cutmix Validation Accuracy\",color='#98DDDD', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(CutMix)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_CutMix.png\")\n",
        "\n",
        "plt.figure(4,figsize=IMGSIZE)\n",
        "plt.plot(X1,train_y1,label=f\"{MODELNAME} + cutmix + mixup Train Accuracy\",color='#C24752', linestyle='-')\n",
        "plt.plot(X1,val_y1,label=f\"{MODELNAME} + cutmix + mixup Validation Accuracy\",color='#C24752', linestyle='--')\n",
        "plt.plot(X1,train_cutmixup_y1,label=f\"{MODELNAME} + cutmixup Train Accuracy\",color='#7F36A1', linestyle='-')\n",
        "plt.plot(X1,val_cutmixup_y1,label=f\"{MODELNAME} + cutmixup Validation Accuracy\",color='#7F36A1', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title(f'{MODELNAME} Compare Accuracy(CutMixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Accuracy_CutMixup.png\")\n",
        "\n",
        "plt.figure(5,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_ori_y2,label=f\"{MODELNAME} Train Loss\",color='#3B7DB0', linestyle='-')\n",
        "plt.plot(X2,val_ori_y2,label=f\"{MODELNAME} Validation Loss\",color='#3B7DB0', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(Original)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_Original.png\")\n",
        "\n",
        "plt.figure(6,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_mixup_y2,label=f\"{MODELNAME} + mixup Train Loss\",color='#CBB162', linestyle='-')\n",
        "plt.plot(X2,val_mixup_y2,label=f\"{MODELNAME} + mixup Validation Loss\",color='#CBB162', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(Mixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_Mixup.png\")\n",
        "\n",
        "plt.figure(7,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_cutmix_y2,label=f\"{MODELNAME} + cutmix Train Loss\",color='#98DDDD', linestyle='-')\n",
        "plt.plot(X2,val_cutmix_y2,label=f\"{MODELNAME} + cutmix Validation Loss\",color='#98DDDD', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(CutMix)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_CutMix.png\")\n",
        "\n",
        "plt.figure(8,figsize=IMGSIZE)\n",
        "plt.plot(X2,train_y2,label=f\"{MODELNAME} + cutmix + mixup Train Loss\",color='#C24752', linestyle='-')\n",
        "plt.plot(X2,val_y2,label=f\"{MODELNAME} + cutmix + mixup Validation Loss\",color='#C24752', linestyle='--')\n",
        "plt.plot(X2,train_cutmixup_y2,label=f\"{MODELNAME} + cutmixup Train Loss\",color='#7F36A1', linestyle='-')\n",
        "plt.plot(X2,val_cutmixup_y2,label=f\"{MODELNAME} + cutmixup Validation Loss\",color='#7F36A1', linestyle='--')\n",
        "plt.legend(loc='best')\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Loss')\n",
        "plt.title(f'{MODELNAME} Compare Loss(CutMixup)')\n",
        "plt.savefig(f\"{IMGPATH}/{MODELNAME}_Compare_Loss_CutMixup.png\")\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KpkCc_IVTH_Z"
      },
      "source": [
        "## Test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O1KE1UlxAxvt",
        "outputId": "fe9b9e4e-5f4b-415a-a604-90ed2d24bebf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Evaluate...\n",
            "Accuracy of cutmix and mixup resnet34 on the 10000 test images: 73.95 %\n",
            "Accuracy of cutmix resnet34 on the 10000 test images: 74.54 %\n",
            "Accuracy of mixup resnet34 on the 10000 test images: 72.96 %\n",
            "Accuracy of original resnet34 on the 10000 test images: 65.50 %\n",
            "Accuracy of cutmixup resnet34 on the 10000 test images: 67.78 %\n"
          ]
        }
      ],
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "\n",
        "# import argparse\n",
        "from random import sample\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "def eval():\n",
        "    model1 = resnet34()\n",
        "    model2 = resnet34()\n",
        "    model3 = resnet34()\n",
        "    model4 = resnet34()\n",
        "    model5 = resnet34()\n",
        "    model1 = model1.cuda() if IsGPU else model1\n",
        "    model2 = model2.cuda() if IsGPU else model2\n",
        "    model3 = model3.cuda() if IsGPU else model3\n",
        "    model4 = model4.cuda() if IsGPU else model4\n",
        "    model5 = model5.cuda() if IsGPU else model5\n",
        "    model1.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}.pth'))\n",
        "    model2.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_cutmix.pth'))\n",
        "    model3.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_mixup.pth'))\n",
        "    model4.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_original.pth'))\n",
        "    model5.load_state_dict(torch.load(f'{MODELPATH}/model_weight_{MODELNAME}_cutmixup.pth'))\n",
        "    \n",
        "    model1.eval()\n",
        "    model2.eval()\n",
        "    model3.eval()\n",
        "    model4.eval()\n",
        "    model5.eval()\n",
        "\n",
        "    total = 0\n",
        "    correct1 = 0\n",
        "    correct2 = 0\n",
        "    correct3 = 0\n",
        "    correct4 = 0\n",
        "    correct5 = 0\n",
        "    # 학습 중이 아니므로, 출력에 대한 변화도를 계산할 필요가 없습니다\n",
        "    print('Evaluate...')\n",
        "    with torch.no_grad():\n",
        "        for i, (input, target) in enumerate(testloader):\n",
        "            input = input.cuda() if IsGPU else input\n",
        "            target = target.cuda() if IsGPU else target\n",
        "            # 신경망에 이미지를 통과시켜 출력을 계산합니다\n",
        "            output1 = model1(input)\n",
        "            output2 = model2(input)\n",
        "            output3 = model3(input)\n",
        "            output4 = model4(input)\n",
        "            output5 = model5(input)\n",
        "            _, predictions1 = torch.max(output1, 1)\n",
        "            _, predictions2 = torch.max(output2, 1)\n",
        "            _, predictions3 = torch.max(output3, 1)\n",
        "            _, predictions4 = torch.max(output4, 1)\n",
        "            _, predictions5 = torch.max(output5, 1)\n",
        "            # 가장 높은 값(energy)를 갖는 분류(class)를 정답으로 선택하겠습니다\n",
        "            total += target.size(0)\n",
        "            correct1 += (predictions1 == target).sum().item()\n",
        "            correct2 += (predictions2 == target).sum().item()\n",
        "            correct3 += (predictions3 == target).sum().item()\n",
        "            correct4 += (predictions4 == target).sum().item()\n",
        "            correct5 += (predictions5 == target).sum().item()\n",
        "        \n",
        "\n",
        "    print(f'Accuracy of cutmix and mixup {MODELNAME} on the 10000 test images: {100 * correct1 / total:.2f} %')\n",
        "    print(f'Accuracy of cutmix {MODELNAME} on the 10000 test images: {100 * correct2 / total:.2f} %')\n",
        "    print(f'Accuracy of mixup {MODELNAME} on the 10000 test images: {100 * correct3 / total:.2f} %')\n",
        "    print(f'Accuracy of original {MODELNAME} on the 10000 test images: {100 * correct4 / total:.2f} %')\n",
        "    print(f'Accuracy of cutmixup {MODELNAME} on the 10000 test images: {100 * correct5 / total:.2f} %')\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    eval()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "result.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}