run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "from custom_wav_loader import wavLoader\n",
    "import numpy as np\n",
    "from model import LeNet, VGG, ResNet, BasicBlock, resnet34, resnet18\n",
    "from train import train, test\n",
    "import os\n",
    "import visdom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Without the incoming socket you cannot receive events from the server or register event handlers to your Visdom client.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "vis = visdom.Visdom(use_incoming_socket=False)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNet\n",
      "load pre-trained model of ResNet\n",
      "\n",
      "{'acc': 1.0847732960676946, 'epoch': 2, 'net': ResNet(\n",
      "  (conv1): Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
      "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (relu): ReLU(inplace)\n",
      "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
      "  (layer1): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (layer2): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (downsample): Sequential(\n",
      "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (layer3): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (downsample): Sequential(\n",
      "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (layer4): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (downsample): Sequential(\n",
      "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (avgpool): AvgPool2d(kernel_size=2, stride=1, padding=0)\n",
      "  (fc): Linear(in_features=32256, out_features=1000, bias=True)\n",
      ")}\n",
      "\n",
      "Start training...\n",
      "Train Epoch: 1 [0/4637 (0%)]\tLoss: 6.752669\n",
      "Train Epoch: 1 [2000/4637 (43%)]\tLoss: 1.034534\n",
      "Train Epoch: 1 [4000/4637 (86%)]\tLoss: 0.680993\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/keewon/.local/lib/python3.5/site-packages/torch/nn/functional.py:52: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss set: Average loss: 0.6446, Accuracy: 3475/4637 (74%)\n",
      "Valid loss set: Average loss: 1.0652, Accuracy: 131/238 (55%)\n",
      "Test loss set: Average loss: 1.3102, Accuracy: 79/235 (33%)\n",
      "\n",
      "Saving model of ResNet\n",
      "\n",
      "Train Epoch: 2 [0/4637 (0%)]\tLoss: 1.038964\n",
      "Train Epoch: 2 [2000/4637 (43%)]\tLoss: 0.664493\n",
      "Train Epoch: 2 [4000/4637 (86%)]\tLoss: 0.647960\n",
      "Train loss set: Average loss: 0.5000, Accuracy: 3951/4637 (85%)\n",
      "Valid loss set: Average loss: 1.0979, Accuracy: 109/238 (45%)\n",
      "Test loss set: Average loss: 1.3350, Accuracy: 82/235 (34%)\n",
      "\n",
      "Loss was not improved, iteration 1\n",
      "\n",
      "Train Epoch: 3 [0/4637 (0%)]\tLoss: 0.349056\n",
      "Train Epoch: 3 [2000/4637 (43%)]\tLoss: 0.451657\n",
      "Train Epoch: 3 [4000/4637 (86%)]\tLoss: 0.534172\n",
      "Train loss set: Average loss: 0.4194, Accuracy: 3968/4637 (85%)\n",
      "Valid loss set: Average loss: 1.1116, Accuracy: 135/238 (56%)\n",
      "Test loss set: Average loss: 1.4010, Accuracy: 87/235 (37%)\n",
      "\n",
      "Loss was not improved, iteration 2\n",
      "\n",
      "Train Epoch: 4 [0/4637 (0%)]\tLoss: 0.560872\n",
      "Train Epoch: 4 [2000/4637 (43%)]\tLoss: 0.279568\n",
      "Train Epoch: 4 [4000/4637 (86%)]\tLoss: 0.427493\n",
      "Train loss set: Average loss: 0.3376, Accuracy: 4268/4637 (92%)\n",
      "Valid loss set: Average loss: 1.1527, Accuracy: 97/238 (40%)\n",
      "Test loss set: Average loss: 1.4007, Accuracy: 84/235 (35%)\n",
      "\n",
      "Loss was not improved, iteration 3\n",
      "\n",
      "Train Epoch: 5 [0/4637 (0%)]\tLoss: 0.374406\n",
      "Train Epoch: 5 [2000/4637 (43%)]\tLoss: 0.575071\n",
      "Train Epoch: 5 [4000/4637 (86%)]\tLoss: 0.321153\n",
      "Train loss set: Average loss: 0.2860, Accuracy: 4322/4637 (93%)\n",
      "Valid loss set: Average loss: 1.2203, Accuracy: 74/238 (31%)\n",
      "Test loss set: Average loss: 1.2907, Accuracy: 103/235 (43%)\n",
      "\n",
      "Loss was not improved, iteration 4\n",
      "\n",
      "Train Epoch: 6 [0/4637 (0%)]\tLoss: 0.418977\n",
      "Train Epoch: 6 [2000/4637 (43%)]\tLoss: 0.290994\n",
      "Train Epoch: 6 [4000/4637 (86%)]\tLoss: 0.193180\n",
      "Train loss set: Average loss: 0.2398, Accuracy: 4370/4637 (94%)\n",
      "Valid loss set: Average loss: 1.2341, Accuracy: 78/238 (32%)\n",
      "Test loss set: Average loss: 1.3053, Accuracy: 102/235 (43%)\n",
      "\n",
      "Loss was not improved, iteration 5\n",
      "\n",
      "Finished!!\n"
     ]
    }
   ],
   "source": [
    "train_path = 'dataset/train/'\n",
    "valid_path = 'dataset/valid/'\n",
    "test_path = 'dataset/test/'\n",
    "\n",
    "# parameter\n",
    "optimizer = 'adadelta' # adadelta adam SGD\n",
    "lr = 0.001 # to do : adaptive lr\n",
    "epochs = 1000\n",
    "epoch = 1\n",
    "momentum = 0.9 # for SGD\n",
    "\n",
    "iteration = 0\n",
    "patience = 5\n",
    "log_interval=100\n",
    "\n",
    "seed = 1234  # random seed\n",
    "batch_size=20 #100\n",
    "test_batch_size= 10\n",
    "arc='ResNet'  # LeNet, VGG11, VGG13, VGG16, VGG19' ResNet\n",
    "\n",
    "\n",
    "#sound setting\n",
    "window_size=0.01   # 0.02\n",
    "window_stride=0.01 # 0.01\n",
    "window_type='hamming'\n",
    "normalize=True\n",
    "\n",
    "# loading data\n",
    "train_dataset = wavLoader(train_path, window_size=window_size, window_stride=window_stride, window_type=window_type, normalize=normalize)\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True, sampler=None)\n",
    "valid_dataset = wavLoader(valid_path, window_size=window_size, window_stride=window_stride, window_type=window_type, normalize=normalize)\n",
    "valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=None, num_workers=4, pin_memory=True, sampler=None)\n",
    "test_dataset = wavLoader(test_path, window_size=window_size, window_stride=window_stride, window_type=window_type, normalize=normalize)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_batch_size, shuffle=None, num_workers=4, pin_memory=True, sampler=None)\n",
    "\n",
    "\n",
    "# build model\n",
    "if arc == 'LeNet':\n",
    "    model = LeNet()\n",
    "    print(\"LeNet\")\n",
    "elif arc.startswith('VGG'):\n",
    "    model = VGG(arc)\n",
    "    print(\"VGG\")\n",
    "elif arc.startswith('Res'):\n",
    "    model = resnet18()\n",
    "    print(\"ResNet\")\n",
    "else:\n",
    "    model = LeNet()\n",
    "\n",
    "model = torch.nn.DataParallel(model).cuda()\n",
    "\n",
    "# define optimizer\n",
    "if optimizer.lower() == 'adam': #adadelta\n",
    "    optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "elif optimizer.lower() == 'adadelta': #adadelta\n",
    "    optimizer = optim.Adadelta(model.parameters(), lr=lr)\n",
    "elif optimizer.lower() == 'sgd':\n",
    "    optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum)\n",
    "else:\n",
    "    optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum)\n",
    "\n",
    "best_valid_loss = np.inf\n",
    "\n",
    "if os.path.isfile('./checkpoint/'+str(arc)+'.pth'):\n",
    "    state = torch.load('./checkpoint/'+str(arc)+'.pth')\n",
    "    print('load pre-trained model of ' + str(arc)+'\\n')\n",
    "    print(state)\n",
    "    best_valid_loss = state['acc']\n",
    "\n",
    "# visdom    \n",
    "loss_graph = vis.line(Y=np.column_stack([10,10,10]),X=np.column_stack([0,0,0]),opts=dict(title='loss',legend=['Train loss','Valid loss','Test loss'],showlegend=True,xlabel='epoch'))\n",
    "    \n",
    "# trainint with early stopping\n",
    "\n",
    "print('\\nStart training...')\n",
    "while (epoch < epochs + 1) and (iteration < patience):\n",
    "    train(train_loader, model, optimizer, epoch, True, log_interval)\n",
    "    train_loss = test(train_loader, model, True, mode='Train loss')\n",
    "    valid_loss = test(valid_loader, model, True, mode='Valid loss')\n",
    "    test_loss  = test(test_loader,model,True,mode='Test loss')\n",
    "    \n",
    "    if valid_loss > best_valid_loss:\n",
    "        iteration += 1\n",
    "        print('\\nLoss was not improved, iteration {0}\\n'.format(str(iteration)))\n",
    "    else:\n",
    "        print('\\nSaving model of ' + str(arc) +'\\n')\n",
    "        iteration = 0\n",
    "        best_valid_loss = valid_loss\n",
    "        state = {\n",
    "            'net': model.module if True else model,\n",
    "            'acc': valid_loss,\n",
    "            'epoch': epoch,\n",
    "        }\n",
    "        if not os.path.isdir('checkpoint'):          # model load should be \n",
    "            os.mkdir('checkpoint')\n",
    "        torch.save(state, './checkpoint/'+str(arc)+'.pth')\n",
    "\n",
    "    vis.line(Y=np.column_stack([train_loss,valid_loss,test_loss]),X=np.column_stack([epoch,epoch,epoch]),win=loss_graph,update='append',opts=dict(legend=['Train loss','Valid loss','Test loss'],showlegend=True))\n",
    "    epoch += 1\n",
    "# test(test_loader,model,True,mode='test loss')\n",
    "print('Finished!!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test(test_loader,model,True,mode='test loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlxtend.evaluate import confusion_matrix\n",
    "from mlxtend.plotting import plot_confusion_matrix\n",
    "import numpy as np\n",
    "\n",
    "y_target =    [1, 1, 1, 0, 0, 2, 0, 3]\n",
    "y_predicted = [1, 0, 1, 0, 0, 2, 1, 3]\n",
    "\n",
    "cm = confusion_matrix(y_target=y_target, \n",
    "                      y_predicted=y_predicted, \n",
    "                      binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'np' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-d43cde7a9b3c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mmlxtend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0max\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplot_confusion_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf_mat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/mlxtend/plotting/plot_confusion_matrix.py\u001b[0m in \u001b[0;36mplot_confusion_matrix\u001b[0;34m(conf_mat, hide_spines, hide_ticks, figsize, cmap, colorbar, show_absolute, show_normed)\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mAssertionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Both show_absolute and show_normed are False'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m     \u001b[0mtotal_samples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconf_mat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnewaxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m     \u001b[0mnormed_conf_mat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconf_mat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mtotal_samples\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from mlxtend.evaluate import confusion_matrix\n",
    "\n",
    "fig, ax = plot_confusion_matrix(conf_mat=cm)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confusion(prediction, truth):\n",
    "    confusion_vector = prediction / truth\n",
    "\n",
    "    true_positives = torch.sum(confusion_vector == 1).item()\n",
    "    false_positives = torch.sum(confusion_vector == float('inf')).item()\n",
    "    true_negatives = torch.sum(torch.isnan(confusion_vector)).item()\n",
    "    false_negatives = torch.sum(confusion_vector == 0).item()\n",
    "\n",
    "    return true_positives, false_positives, true_negatives, false_negatives"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}