Breakout_Deep_Q_RL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DCXtWi5hHqfv"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/ShivamShrirao/deep_Q_learning_from_scratch/blob/master/Breakout_Deep_Q_RL.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 357
    },
    "colab_type": "code",
    "id": "LbuhMj8QkFUO",
    "outputId": "844e730c-0f2a-4eff-83d6-5715eed596d8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Aug 24 08:08:24 2020       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 450.57       Driver Version: 418.67       CUDA Version: 10.1     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
      "| N/A   38C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
      "|                               |                      |                 ERR! |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "TQZiX7WgkQeu",
    "outputId": "ddbe2aac-f330-4004-dc6b-637495ef7899"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 207
    },
    "colab_type": "code",
    "id": "v6pXJGDekFNt",
    "outputId": "feed2ea1-03f7-4d7a-8b87-96317468b501"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/ShivamShrirao/dnn_from_scratch.git\n",
      "  Cloning https://github.com/ShivamShrirao/dnn_from_scratch.git to /tmp/pip-req-build-9qf32oxg\n",
      "  Running command git clone -q https://github.com/ShivamShrirao/dnn_from_scratch.git /tmp/pip-req-build-9qf32oxg\n",
      "Building wheels for collected packages: dnn-from-scratch\n",
      "  Building wheel for dnn-from-scratch (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for dnn-from-scratch: filename=dnn_from_scratch-0.1.dev1-cp36-none-any.whl size=39847 sha256=01a1375b9c06e7edc746df0c24561f3f86cf21b34c77b1cf008a835fbfa4cf4d\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-3dwvotrk/wheels/33/77/6c/1140f6295368e7045f3b49fdda98e7f07b2513b8b55093ca7b\n",
      "Successfully built dnn-from-scratch\n",
      "Installing collected packages: dnn-from-scratch\n",
      "Successfully installed dnn-from-scratch-0.1.dev1\n"
     ]
    }
   ],
   "source": [
    "!python -m pip install git+https://github.com/ShivamShrirao/dnn_from_scratch.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "id": "CgT-5T6EkO8a",
    "outputId": "78ca8cdd-5a26-45ba-9dc1-ade9ef787092"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'deep_Q_learning_from_scratch'...\n",
      "remote: Enumerating objects: 271, done.\u001b[K\n",
      "remote: Counting objects:   0% (1/271)\u001b[K\r",
      "remote: Counting objects:   1% (3/271)\u001b[K\r",
      "remote: Counting objects:   2% (6/271)\u001b[K\r",
      "remote: Counting objects:   3% (9/271)\u001b[K\r",
      "remote: Counting objects:   4% (11/271)\u001b[K\r",
      "remote: Counting objects:   5% (14/271)\u001b[K\r",
      "remote: Counting objects:   6% (17/271)\u001b[K\r",
      "remote: Counting objects:   7% (19/271)\u001b[K\r",
      "remote: Counting objects:   8% (22/271)\u001b[K\r",
      "remote: Counting objects:   9% (25/271)\u001b[K\r",
      "remote: Counting objects:  10% (28/271)\u001b[K\r",
      "remote: Counting objects:  11% (30/271)\u001b[K\r",
      "remote: Counting objects:  12% (33/271)\u001b[K\r",
      "remote: Counting objects:  13% (36/271)\u001b[K\r",
      "remote: Counting objects:  14% (38/271)\u001b[K\r",
      "remote: Counting objects:  15% (41/271)\u001b[K\r",
      "remote: Counting objects:  16% (44/271)\u001b[K\r",
      "remote: Counting objects:  17% (47/271)\u001b[K\r",
      "remote: Counting objects:  18% (49/271)\u001b[K\r",
      "remote: Counting objects:  19% (52/271)\u001b[K\r",
      "remote: Counting objects:  20% (55/271)\u001b[K\r",
      "remote: Counting objects:  21% (57/271)\u001b[K\r",
      "remote: Counting objects:  22% (60/271)\u001b[K\r",
      "remote: Counting objects:  23% (63/271)\u001b[K\r",
      "remote: Counting objects:  24% (66/271)\u001b[K\r",
      "remote: Counting objects:  25% (68/271)\u001b[K\r",
      "remote: Counting objects:  26% (71/271)\u001b[K\r",
      "remote: Counting objects:  27% (74/271)\u001b[K\r",
      "remote: Counting objects:  28% (76/271)\u001b[K\r",
      "remote: Counting objects:  29% (79/271)\u001b[K\r",
      "remote: Counting objects:  30% (82/271)\u001b[K\r",
      "remote: Counting objects:  31% (85/271)\u001b[K\r",
      "remote: Counting objects:  32% (87/271)\u001b[K\r",
      "remote: Counting objects:  33% (90/271)\u001b[K\r",
      "remote: Counting objects:  34% (93/271)\u001b[K\r",
      "remote: Counting objects:  35% (95/271)\u001b[K\r",
      "remote: Counting objects:  36% (98/271)\u001b[K\r",
      "remote: Counting objects:  37% (101/271)\u001b[K\r",
      "remote: Counting objects:  38% (103/271)\u001b[K\r",
      "remote: Counting objects:  39% (106/271)\u001b[K\r",
      "remote: Counting objects:  40% (109/271)\u001b[K\r",
      "remote: Counting objects:  41% (112/271)\u001b[K\r",
      "remote: Counting objects:  42% (114/271)\u001b[K\r",
      "remote: Counting objects:  43% (117/271)\u001b[K\r",
      "remote: Counting objects:  44% (120/271)\u001b[K\r",
      "remote: Counting objects:  45% (122/271)\u001b[K\r",
      "remote: Counting objects:  46% (125/271)\u001b[K\r",
      "remote: Counting objects:  47% (128/271)\u001b[K\r",
      "remote: Counting objects:  48% (131/271)\u001b[K\r",
      "remote: Counting objects:  49% (133/271)\u001b[K\r",
      "remote: Counting objects:  50% (136/271)\u001b[K\r",
      "remote: Counting objects:  51% (139/271)\u001b[K\r",
      "remote: Counting objects:  52% (141/271)\u001b[K\r",
      "remote: Counting objects:  53% (144/271)\u001b[K\r",
      "remote: Counting objects:  54% (147/271)\u001b[K\r",
      "remote: Counting objects:  55% (150/271)\u001b[K\r",
      "remote: Counting objects:  56% (152/271)\u001b[K\r",
      "remote: Counting objects:  57% (155/271)\u001b[K\r",
      "remote: Counting objects:  58% (158/271)\u001b[K\r",
      "remote: Counting objects:  59% (160/271)\u001b[K\r",
      "remote: Counting objects:  60% (163/271)\u001b[K\r",
      "remote: Counting objects:  61% (166/271)\u001b[K\r",
      "remote: Counting objects:  62% (169/271)\u001b[K\r",
      "remote: Counting objects:  63% (171/271)\u001b[K\r",
      "remote: Counting objects:  64% (174/271)\u001b[K\r",
      "remote: Counting objects:  65% (177/271)\u001b[K\r",
      "remote: Counting objects:  66% (179/271)\u001b[K\r",
      "remote: Counting objects:  67% (182/271)\u001b[K\r",
      "remote: Counting objects:  68% (185/271)\u001b[K\r",
      "remote: Counting objects:  69% (187/271)\u001b[K\r",
      "remote: Counting objects:  70% (190/271)\u001b[K\r",
      "remote: Counting objects:  71% (193/271)\u001b[K\r",
      "remote: Counting objects:  72% (196/271)\u001b[K\r",
      "remote: Counting objects:  73% (198/271)\u001b[K\r",
      "remote: Counting objects:  74% (201/271)\u001b[K\r",
      "remote: Counting objects:  75% (204/271)\u001b[K\r",
      "remote: Counting objects:  76% (206/271)\u001b[K\r",
      "remote: Counting objects:  77% (209/271)\u001b[K\r",
      "remote: Counting objects:  78% (212/271)\u001b[K\r",
      "remote: Counting objects:  79% (215/271)\u001b[K\r",
      "remote: Counting objects:  80% (217/271)\u001b[K\r",
      "remote: Counting objects:  81% (220/271)\u001b[K\r",
      "remote: Counting objects:  82% (223/271)\u001b[K\r",
      "remote: Counting objects:  83% (225/271)\u001b[K\r",
      "remote: Counting objects:  84% (228/271)\u001b[K\r",
      "remote: Counting objects:  85% (231/271)\u001b[K\r",
      "remote: Counting objects:  86% (234/271)\u001b[K\r",
      "remote: Counting objects:  87% (236/271)\u001b[K\r",
      "remote: Counting objects:  88% (239/271)\u001b[K\r",
      "remote: Counting objects:  89% (242/271)\u001b[K\r",
      "remote: Counting objects:  90% (244/271)\u001b[K\r",
      "remote: Counting objects:  91% (247/271)\u001b[K\r",
      "remote: Counting objects:  92% (250/271)\u001b[K\r",
      "remote: Counting objects:  93% (253/271)\u001b[K\r",
      "remote: Counting objects:  94% (255/271)\u001b[K\r",
      "remote: Counting objects:  95% (258/271)\u001b[K\r",
      "remote: Counting objects:  96% (261/271)\u001b[K\r",
      "remote: Counting objects:  97% (263/271)\u001b[K\r",
      "remote: Counting objects:  98% (266/271)\u001b[K\r",
      "remote: Counting objects:  99% (269/271)\u001b[K\r",
      "remote: Counting objects: 100% (271/271)\u001b[K\r",
      "remote: Counting objects: 100% (271/271), done.\u001b[K\n",
      "remote: Compressing objects:   0% (1/176)\u001b[K\r",
      "remote: Compressing objects:   1% (2/176)\u001b[K\r",
      "remote: Compressing objects:   2% (4/176)\u001b[K\r",
      "remote: Compressing objects:   3% (6/176)\u001b[K\r",
      "remote: Compressing objects:   4% (8/176)\u001b[K\r",
      "remote: Compressing objects:   5% (9/176)\u001b[K\r",
      "remote: Compressing objects:   6% (11/176)\u001b[K\r",
      "remote: Compressing objects:   7% (13/176)\u001b[K\r",
      "remote: Compressing objects:   8% (15/176)\u001b[K\r",
      "remote: Compressing objects:   9% (16/176)\u001b[K\r",
      "remote: Compressing objects:  10% (18/176)\u001b[K\r",
      "remote: Compressing objects:  11% (20/176)\u001b[K\r",
      "remote: Compressing objects:  12% (22/176)\u001b[K\r",
      "remote: Compressing objects:  13% (23/176)\u001b[K\r",
      "remote: Compressing objects:  14% (25/176)\u001b[K\r",
      "remote: Compressing objects:  15% (27/176)\u001b[K\r",
      "remote: Compressing objects:  16% (29/176)\u001b[K\r",
      "remote: Compressing objects:  17% (30/176)\u001b[K\r",
      "remote: Compressing objects:  18% (32/176)\u001b[K\r",
      "remote: Compressing objects:  19% (34/176)\u001b[K\r",
      "remote: Compressing objects:  20% (36/176)\u001b[K\r",
      "remote: Compressing objects:  21% (37/176)\u001b[K\r",
      "remote: Compressing objects:  22% (39/176)\u001b[K\r",
      "remote: Compressing objects:  23% (41/176)\u001b[K\r",
      "remote: Compressing objects:  24% (43/176)\u001b[K\r",
      "remote: Compressing objects:  25% (44/176)\u001b[K\r",
      "remote: Compressing objects:  26% (46/176)\u001b[K\r",
      "remote: Compressing objects:  27% (48/176)\u001b[K\r",
      "remote: Compressing objects:  28% (50/176)\u001b[K\r",
      "remote: Compressing objects:  29% (52/176)\u001b[K\r",
      "remote: Compressing objects:  30% (53/176)\u001b[K\r",
      "remote: Compressing objects:  31% (55/176)\u001b[K\r",
      "remote: Compressing objects:  32% (57/176)\u001b[K\r",
      "remote: Compressing objects:  33% (59/176)\u001b[K\r",
      "remote: Compressing objects:  34% (60/176)\u001b[K\r",
      "remote: Compressing objects:  35% (62/176)\u001b[K\r",
      "remote: Compressing objects:  36% (64/176)\u001b[K\r",
      "remote: Compressing objects:  37% (66/176)\u001b[K\r",
      "remote: Compressing objects:  38% (67/176)\u001b[K\r",
      "remote: Compressing objects:  39% (69/176)\u001b[K\r",
      "remote: Compressing objects:  40% (71/176)\u001b[K\r",
      "remote: Compressing objects:  41% (73/176)\u001b[K\r",
      "remote: Compressing objects:  42% (74/176)\u001b[K\r",
      "remote: Compressing objects:  43% (76/176)\u001b[K\r",
      "remote: Compressing objects:  44% (78/176)\u001b[K\r",
      "remote: Compressing objects:  45% (80/176)\u001b[K\r",
      "remote: Compressing objects:  46% (81/176)\u001b[K\r",
      "remote: Compressing objects:  47% (83/176)\u001b[K\r",
      "remote: Compressing objects:  48% (85/176)\u001b[K\r",
      "remote: Compressing objects:  49% (87/176)\u001b[K\r",
      "remote: Compressing objects:  50% (88/176)\u001b[K\r",
      "remote: Compressing objects:  51% (90/176)\u001b[K\r",
      "remote: Compressing objects:  52% (92/176)\u001b[K\r",
      "remote: Compressing objects:  53% (94/176)\u001b[K\r",
      "remote: Compressing objects:  54% (96/176)\u001b[K\r",
      "remote: Compressing objects:  55% (97/176)\u001b[K\r",
      "remote: Compressing objects:  56% (99/176)\u001b[K\r",
      "remote: Compressing objects:  57% (101/176)\u001b[K\r",
      "remote: Compressing objects:  58% (103/176)\u001b[K\r",
      "remote: Compressing objects:  59% (104/176)\u001b[K\r",
      "remote: Compressing objects:  60% (106/176)\u001b[K\r",
      "remote: Compressing objects:  61% (108/176)\u001b[K\r",
      "remote: Compressing objects:  62% (110/176)\u001b[K\r",
      "remote: Compressing objects:  63% (111/176)\u001b[K\r",
      "remote: Compressing objects:  64% (113/176)\u001b[K\r",
      "remote: Compressing objects:  65% (115/176)\u001b[K\r",
      "remote: Compressing objects:  66% (117/176)\u001b[K\r",
      "remote: Compressing objects:  67% (118/176)\u001b[K\r",
      "remote: Compressing objects:  68% (120/176)\u001b[K\r",
      "remote: Compressing objects:  69% (122/176)\u001b[K\r",
      "remote: Compressing objects:  70% (124/176)\u001b[K\r",
      "remote: Compressing objects:  71% (125/176)\u001b[K\r",
      "remote: Compressing objects:  72% (127/176)\u001b[K\r",
      "remote: Compressing objects:  73% (129/176)\u001b[K\r",
      "remote: Compressing objects:  74% (131/176)\u001b[K\r",
      "remote: Compressing objects:  75% (132/176)\u001b[K\r",
      "remote: Compressing objects:  76% (134/176)\u001b[K\r",
      "remote: Compressing objects:  77% (136/176)\u001b[K\r",
      "remote: Compressing objects:  78% (138/176)\u001b[K\r",
      "remote: Compressing objects:  79% (140/176)\u001b[K\r",
      "remote: Compressing objects:  80% (141/176)\u001b[K\r",
      "remote: Compressing objects:  81% (143/176)\u001b[K\r",
      "remote: Compressing objects:  82% (145/176)\u001b[K\r",
      "remote: Compressing objects:  83% (147/176)\u001b[K\r",
      "remote: Compressing objects:  84% (148/176)\u001b[K\r",
      "remote: Compressing objects:  85% (150/176)\u001b[K\r",
      "remote: Compressing objects:  86% (152/176)\u001b[K\r",
      "remote: Compressing objects:  87% (154/176)\u001b[K\r",
      "remote: Compressing objects:  88% (155/176)\u001b[K\r",
      "remote: Compressing objects:  89% (157/176)\u001b[K\r",
      "remote: Compressing objects:  90% (159/176)\u001b[K\r",
      "remote: Compressing objects:  91% (161/176)\u001b[K\r",
      "remote: Compressing objects:  92% (162/176)\u001b[K\r",
      "remote: Compressing objects:  93% (164/176)\u001b[K\r",
      "remote: Compressing objects:  94% (166/176)\u001b[K\r",
      "remote: Compressing objects:  95% (168/176)\u001b[K\r",
      "remote: Compressing objects:  96% (169/176)\u001b[K\r",
      "remote: Compressing objects:  97% (171/176)\u001b[K\r",
      "remote: Compressing objects:  98% (173/176)\u001b[K\r",
      "remote: Compressing objects:  99% (175/176)\u001b[K\r",
      "remote: Compressing objects: 100% (176/176)\u001b[K\r",
      "remote: Compressing objects: 100% (176/176), done.\u001b[K\n",
      "Receiving objects:   0% (1/271)   \r",
      "Receiving objects:   1% (3/271)   \r",
      "Receiving objects:   2% (6/271)   \r",
      "Receiving objects:   3% (9/271)   \r",
      "Receiving objects:   4% (11/271)   \r",
      "Receiving objects:   5% (14/271)   \r",
      "Receiving objects:   6% (17/271)   \r",
      "Receiving objects:   7% (19/271)   \r",
      "Receiving objects:   8% (22/271)   \r",
      "Receiving objects:   9% (25/271)   \r",
      "Receiving objects:  10% (28/271)   \r",
      "Receiving objects:  11% (30/271)   \r",
      "Receiving objects:  12% (33/271)   \r",
      "Receiving objects:  13% (36/271)   \r",
      "Receiving objects:  14% (38/271)   \r",
      "Receiving objects:  15% (41/271)   \r",
      "Receiving objects:  16% (44/271)   \r",
      "Receiving objects:  17% (47/271)   \r",
      "Receiving objects:  18% (49/271)   \r",
      "Receiving objects:  19% (52/271)   \r",
      "Receiving objects:  20% (55/271)   \r",
      "Receiving objects:  21% (57/271)   \r",
      "Receiving objects:  22% (60/271)   \r",
      "Receiving objects:  23% (63/271)   \r",
      "Receiving objects:  24% (66/271)   \r",
      "Receiving objects:  25% (68/271)   \r",
      "Receiving objects:  26% (71/271)   \r",
      "Receiving objects:  27% (74/271)   \r",
      "Receiving objects:  28% (76/271)   \r",
      "Receiving objects:  29% (79/271)   \r",
      "Receiving objects:  30% (82/271)   \r",
      "Receiving objects:  31% (85/271)   \r",
      "Receiving objects:  32% (87/271)   \r",
      "Receiving objects:  33% (90/271)   \r",
      "Receiving objects:  34% (93/271)   \r",
      "Receiving objects:  35% (95/271)   \r",
      "Receiving objects:  36% (98/271)   \r",
      "Receiving objects:  37% (101/271)   \r",
      "Receiving objects:  38% (103/271)   \r",
      "Receiving objects:  39% (106/271)   \r",
      "Receiving objects:  40% (109/271)   \r",
      "Receiving objects:  41% (112/271)   \r",
      "Receiving objects:  42% (114/271)   \r",
      "Receiving objects:  43% (117/271)   \r",
      "Receiving objects:  44% (120/271)   \r",
      "Receiving objects:  45% (122/271)   \r",
      "Receiving objects:  46% (125/271)   \r",
      "Receiving objects:  47% (128/271)   \r",
      "Receiving objects:  48% (131/271)   \r",
      "Receiving objects:  49% (133/271)   \r",
      "Receiving objects:  50% (136/271)   \r",
      "Receiving objects:  51% (139/271)   \r",
      "Receiving objects:  52% (141/271)   \r",
      "Receiving objects:  53% (144/271)   \r",
      "Receiving objects:  54% (147/271)   \r",
      "Receiving objects:  55% (150/271)   \r",
      "Receiving objects:  56% (152/271)   \r",
      "Receiving objects:  57% (155/271)   \r",
      "Receiving objects:  58% (158/271)   \r",
      "Receiving objects:  59% (160/271)   \r",
      "Receiving objects:  60% (163/271)   \r",
      "Receiving objects:  61% (166/271)   \r",
      "Receiving objects:  62% (169/271)   \r",
      "Receiving objects:  63% (171/271)   \r",
      "Receiving objects:  64% (174/271)   \r",
      "Receiving objects:  65% (177/271)   \r",
      "Receiving objects:  66% (179/271)   \r",
      "Receiving objects:  67% (182/271)   \r",
      "Receiving objects:  68% (185/271)   \r",
      "Receiving objects:  69% (187/271)   \r",
      "Receiving objects:  70% (190/271)   \r",
      "Receiving objects:  71% (193/271)   \r",
      "Receiving objects:  72% (196/271)   \r",
      "Receiving objects:  73% (198/271)   \r",
      "Receiving objects:  74% (201/271)   \r",
      "Receiving objects:  75% (204/271)   \r",
      "Receiving objects:  76% (206/271)   \r",
      "Receiving objects:  77% (209/271)   \r",
      "Receiving objects:  78% (212/271)   \r",
      "Receiving objects:  79% (215/271)   \r",
      "Receiving objects:  80% (217/271)   \r",
      "Receiving objects:  81% (220/271)   \r",
      "Receiving objects:  82% (223/271)   \r",
      "remote: Total 271 (delta 173), reused 186 (delta 88), pack-reused 0\u001b[K\n",
      "Receiving objects:  83% (225/271)   \r",
      "Receiving objects:  84% (228/271)   \r",
      "Receiving objects:  85% (231/271)   \r",
      "Receiving objects:  86% (234/271)   \r",
      "Receiving objects:  87% (236/271)   \r",
      "Receiving objects:  88% (239/271)   \r",
      "Receiving objects:  89% (242/271)   \r",
      "Receiving objects:  90% (244/271)   \r",
      "Receiving objects:  91% (247/271)   \r",
      "Receiving objects:  92% (250/271)   \r",
      "Receiving objects:  93% (253/271)   \r",
      "Receiving objects:  94% (255/271)   \r",
      "Receiving objects:  95% (258/271)   \r",
      "Receiving objects:  96% (261/271)   \r",
      "Receiving objects:  97% (263/271)   \r",
      "Receiving objects:  98% (266/271)   \r",
      "Receiving objects:  99% (269/271)   \r",
      "Receiving objects: 100% (271/271)   \r",
      "Receiving objects: 100% (271/271), 216.32 KiB | 1.21 MiB/s, done.\n",
      "Resolving deltas:   0% (0/173)   \r",
      "Resolving deltas:   1% (2/173)   \r",
      "Resolving deltas:   4% (7/173)   \r",
      "Resolving deltas:  15% (27/173)   \r",
      "Resolving deltas:  22% (39/173)   \r",
      "Resolving deltas:  47% (83/173)   \r",
      "Resolving deltas:  76% (132/173)   \r",
      "Resolving deltas:  78% (135/173)   \r",
      "Resolving deltas:  79% (137/173)   \r",
      "Resolving deltas:  82% (142/173)   \r",
      "Resolving deltas:  89% (155/173)   \r",
      "Resolving deltas:  91% (159/173)   \r",
      "Resolving deltas:  92% (160/173)   \r",
      "Resolving deltas:  93% (162/173)   \r",
      "Resolving deltas:  99% (172/173)   \r",
      "Resolving deltas: 100% (173/173)   \r",
      "Resolving deltas: 100% (173/173), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/ShivamShrirao/deep_Q_learning_from_scratch.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "caNfRIj1kRcf",
    "outputId": "f90beeb7-396d-4de2-b794-ad05579be68b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/deep_Q_learning_from_scratch\n"
     ]
    }
   ],
   "source": [
    "%cd deep_Q_learning_from_scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "OzhhtvRQkWJW",
    "outputId": "5536385b-e614-4e36-b26f-6d2578c98d76"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From https://github.com/ShivamShrirao/deep_Q_learning_from_scratch\n",
      " * branch            master     -> FETCH_HEAD\n",
      "Already up to date.\n"
     ]
    }
   ],
   "source": [
    "!git pull origin master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "W0SnQz--kXCg"
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "import time\n",
    "import cv2\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "\n",
    "from settings import *\n",
    "from agent import *\n",
    "from experience import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AO6IVdWYUyYT"
   },
   "outputs": [],
   "source": [
    "from nnet_gpu.network import Sequential\n",
    "from nnet_gpu.layers import Conv2D,Flatten,Dense,Dropout\n",
    "from nnet_gpu import optimizers\n",
    "from nnet_gpu import functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aC7jFXo3djvV"
   },
   "outputs": [],
   "source": [
    "HEIGHT = 80\n",
    "WIDTH = 72"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make Convolutional Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_ALvjRInG3j1"
   },
   "outputs": [],
   "source": [
    "def get_model(input_shape=(HEIGHT,WIDTH,NFRAMES), no_of_actions=3):\n",
    "    model=Sequential()\n",
    "    model.add(Conv2D(num_kernels=32, kernel_size=3, stride=(2, 2), activation=functions.relu, input_shape=input_shape))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Conv2D(num_kernels=64, kernel_size=3, stride=(2, 2), activation=functions.relu))\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Conv2D(num_kernels=128, kernel_size=3, stride=(2, 2), activation=functions.relu))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(512, activation=functions.relu))\n",
    "    model.add(Dense(no_of_actions, activation=functions.echo))\n",
    "\n",
    "    model.compile(optimizer=optimizers.adam, loss=functions.mean_squared_error, learning_rate=0.0001)\n",
    "    return model\n",
    "\n",
    "def preproc_obsv(obsv):\n",
    "    obsv = cv2.cvtColor(obsv, cv2.COLOR_RGB2GRAY)\n",
    "    obsv = obsv[34:194:2,8:152:2]\n",
    "    return obsv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize the agent with model and target model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 459
    },
    "colab_type": "code",
    "id": "hc4bwkXTkX7_",
    "outputId": "d43dd0f6-7758-4533-f4ca-3e05d3a3cd38"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽\n",
      "Layer (type)               Output Shape             Activation        Param #\n",
      "==========================================================================================\n",
      "- InputLayer(InputLayer)  (None, 80, 72, 4)          echo             0\n",
      "__________________________________________________________________________________________\n",
      "0 Conv2D(Conv2D)          (None, 40, 36, 32)         relu             1184\n",
      "__________________________________________________________________________________________\n",
      "1 Dropout(Dropout)        (None, 40, 36, 32)         echo             0\n",
      "__________________________________________________________________________________________\n",
      "2 Conv2D(Conv2D)          (None, 20, 18, 64)         relu             18496\n",
      "__________________________________________________________________________________________\n",
      "3 Dropout(Dropout)        (None, 20, 18, 64)         echo             0\n",
      "__________________________________________________________________________________________\n",
      "4 Conv2D(Conv2D)          (None, 10, 9, 128)         relu             73856\n",
      "__________________________________________________________________________________________\n",
      "5 Flatten(Flatten)        (None, 11520)              echo             0\n",
      "__________________________________________________________________________________________\n",
      "6 Dropout(Dropout)        (None, 11520)              echo             0\n",
      "__________________________________________________________________________________________\n",
      "7 Dense(Dense)            (None, 512)                relu             5898752\n",
      "__________________________________________________________________________________________\n",
      "8 Dense(Dense)            (None, 3)                  echo             1539\n",
      "==========================================================================================\n",
      "Total Params: 5,993,827\n",
      "Trainable Params: 5,993,827\n",
      "Non-trainable Params: 0\n"
     ]
    }
   ],
   "source": [
    "agt = DQN_Agent(actions=[0,2,3], epsilon=1, min_epsilon=0.1, eps_decay=1e-6, target_update_thresh=3000)\n",
    "agt.model = get_model(input_shape=(HEIGHT,WIDTH,NFRAMES), no_of_actions=len(agt.actions))\n",
    "agt.target = get_model(input_shape=(HEIGHT,WIDTH,NFRAMES), no_of_actions=len(agt.actions))\n",
    "agt.model.load_weights(\"/content/drive/My Drive/breakout.w8s\")\n",
    "agt.update_target()\n",
    "agt.model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H_R_avOVHcaY"
   },
   "outputs": [],
   "source": [
    "D_exp = ReplayMemory(capacity=1_000_000, nlap=1, height=HEIGHT, width=WIDTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "MDlH0rWlH2Kt",
    "outputId": "07cabab6-6a33-4f1e-bf71-0f87c0e06890"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.364418029785156"
      ]
     },
     "execution_count": 13,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D_exp.current_state.nbytes/1024/1024/1024"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## make game environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XFlEoOqOkbcy"
   },
   "outputs": [],
   "source": [
    "env = gym.make('Breakout-v4')       # v4 to ensure ball is fired when required"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4wbt1kZnq3bf"
   },
   "outputs": [],
   "source": [
    "total_frames = 0\n",
    "scores = deque([], maxlen=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "do2kpfL9Yd_o",
    "outputId": "82e81c2d-ee8a-491e-9c88-0c7978fa978c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "599364"
      ]
     },
     "execution_count": 34,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D_exp.len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1-zscT4Tkd2Z"
   },
   "outputs": [],
   "source": [
    "for i_episode in range(3000):\n",
    "    obinit = env.reset()\n",
    "    if not i_episode:                  # init first state queue\n",
    "        observation = obinit\n",
    "        state = preproc_obsv(observation)\n",
    "        state_que = deque([], maxlen=NFRAMES)\n",
    "        for i in range(NFRAMES):\n",
    "            state_que.append(state)\n",
    "    ep_score = 0\n",
    "    lives = 5\n",
    "    fire = True\n",
    "    start = time.time()\n",
    "    for t in range(10_000):\n",
    "        s_s = time.time()\n",
    "#         env.render()\n",
    "        state = preproc_obsv(observation)        # preprocess the observation\n",
    "        state_que.append(state)                  # add to state queue\n",
    "        if fire:                                 # to start the game, fire the ball first\n",
    "            action = 1\n",
    "            fire = False\n",
    "        else:\n",
    "            action = agt.get_action(state_que)   # get action epsilon greedy\n",
    "        next_observation, reward, done, info = env.step(action)  # take a action in environment and observe\n",
    "        ep_score += reward\n",
    "        if lives != info['ale.lives']:           # check if agent died\n",
    "            lives = info['ale.lives']\n",
    "            reward = -2\n",
    "            fire = True\n",
    "        reward = np.clip(reward, -2, 2)\n",
    "        if action==1:\n",
    "            action = 0\n",
    "        D_exp.store_transition(state, agt.actions.index(action), reward, done)   # store the state to experience replay\n",
    "        observation = next_observation\n",
    "\n",
    "        if (D_exp.len-D_exp.idx_len) > BATCH_SIZE:\n",
    "            grads = agt.train(D_exp, BATCH_SIZE)    # train from experience\n",
    "        print('\\r', t, action, ep_score, end='  ')\n",
    "        if done:\n",
    "            break\n",
    "    total_frames += (t+1)\n",
    "    scores.append(ep_score)\n",
    "    print(f\"\\rEpisode {i_episode+1} finished after {t+1} timesteps, Score: {ep_score}, Avg: {sum(scores)/len(scores):.2f}, Epsilon: {agt.epsilon:.6f}, Time: {time.time()-start:.2f}, Frames: {total_frames:,}\")\n",
    "    if not i_episode%8:\n",
    "        agt.model.save_weights(\"/content/drive/My Drive/breakout.w8s\")\n",
    "        if total_frames > 1_000_000:\n",
    "            agt.continue_decay = True\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D7hG8g7Snkyh"
   },
   "outputs": [],
   "source": [
    "agt.epsilon = 0.11\n",
    "agt.min_epsilon = 0.1\n",
    "agt.eps_decay = 1e-6\n",
    "agt.target_update_thresh = 3000\n",
    "agt.continue_decay = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5GCMcH0otSUd"
   },
   "outputs": [],
   "source": [
    "from gym import wrappers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7DqektqyH6Wb"
   },
   "outputs": [],
   "source": [
    "wrapped_env = wrappers.Monitor(env.env, '/content/videos/' + str(time.time()) + '/')\n",
    "for i_episode in range(1):\n",
    "    obinit = wrapped_env.reset()\n",
    "    if not i_episode:\n",
    "        observation = obinit\n",
    "        state = preproc_obsv(observation)\n",
    "        state_que = deque([], maxlen=NFRAMES)\n",
    "        for i in range(NFRAMES):\n",
    "            state_que.append(state)\n",
    "    ep_score = 0\n",
    "    lives = 5\n",
    "    fire = True\n",
    "    preds = []\n",
    "    reward_history = []\n",
    "    start = time.time()\n",
    "    t=-1\n",
    "    while 1:\n",
    "        t+=1\n",
    "        s_s = time.time()\n",
    "#         wrapped_env.render()\n",
    "        state = preproc_obsv(observation)\n",
    "        state_que.append(state)\n",
    "        if fire:\n",
    "            action = 1\n",
    "            fire = False\n",
    "        else:\n",
    "            out = agt.predict(state_que)\n",
    "            pidx = cp.argmax(out[0]).item()\n",
    "            preds.append(out[0][pidx].item())\n",
    "            action = agt.actions[pidx]\n",
    "        next_observation, reward, done, info = wrapped_env.step(action)\n",
    "        if lives != info['ale.lives']:\n",
    "            lives = info['ale.lives']\n",
    "            reward = -1\n",
    "            fire = True\n",
    "        ep_score += reward\n",
    "        reward_history.append(reward)\n",
    "        if action==1:\n",
    "            action = 0\n",
    "        observation = next_observation\n",
    "        print('\\r', t, ep_score, end='  ')\n",
    "        if done:\n",
    "            break\n",
    "    print(f\"\\rEpisode {i_episode+1} finished after {t+1} timesteps, Score: {ep_score}, Epsilon: {agt.epsilon:.6f}, Time: {time.time()-start:.2f}\")\n",
    "wrapped_env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PFHCbvnPtn8q"
   },
   "outputs": [],
   "source": [
    "!rm /content/videos/* -rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0ImTh1UpurGl"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Breakout_Deep_Q_RL.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}