doom.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf      # Deep Learning library\n",
    "import numpy as np           # Handle matrices\n",
    "from vizdoom import *        # Doom Environment\n",
    "\n",
    "import random                # Handling random number generation\n",
    "import time                  # Handling time calculation\n",
    "from skimage import transform# Help us to preprocess the frames\n",
    "\n",
    "from collections import deque# Ordered collection with ends\n",
    "import matplotlib.pyplot as plt # Display graphs\n",
    "\n",
    "import warnings\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['basic', 'cig', 'deadly_corridor', 'deathmatch', 'defend_the_center', 'defend_the_line', 'health_gathering', 'health_gathering_supreme', 'learning', 'multi', 'multi_duel', 'my_way_home', 'oblige', 'predict_position', 'rocket_basic', 'simpler_basic', 'take_cover']\n"
     ]
    }
   ],
   "source": [
    "#Extract all level files from scenarios directory\n",
    "levels = []\n",
    "\n",
    "for file in os.listdir(\"scenarios/\"):\n",
    "    if file.endswith(\".cfg\"):\n",
    "        levels.append(os.path.splitext(file)[0])\n",
    "        \n",
    "print(levels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Here we create our environment\n",
    "\"\"\"\n",
    "def create_environment(level=0, launch_game=True):\n",
    "    \n",
    "    game = DoomGame()\n",
    "    # Load the correct configuration\n",
    "    game.load_config(\"scenarios/\" + levels[level] + \".cfg\")\n",
    "    \n",
    "    # Load the correct scenario\n",
    "    game.set_doom_scenario_path(\"scenarios/\" + levels[level] + \".wad\")\n",
    "    game.set_screen_resolution(RES_640X480)\n",
    "    \n",
    "    if launch_game:\n",
    "        game.init()\n",
    "    \n",
    "    # List of possible controls from vizdoom environment\n",
    "    controls = game.get_available_buttons()\n",
    "    action_size = game.get_available_buttons_size()\n",
    "    \n",
    "    action_names = []\n",
    "    for index in range(action_size):\n",
    "        action_names.append(os.path.splitext(str(controls[index]))[1][1:])\n",
    "        \n",
    "    action_space = np.eye(action_size).astype(int).tolist()\n",
    "    \n",
    "    return game, action_space, action_names\n",
    "       \n",
    "def test_environment(level=0, episodes=10, print_info=True):\n",
    "    game, actions, action_names = create_environment(level)\n",
    "    caches = []\n",
    "    \n",
    "    episodes = 1\n",
    "    \n",
    "    for i in range(episodes):\n",
    "        game.new_episode()\n",
    "        while not game.is_episode_finished():\n",
    "            state = game.get_state()\n",
    "            img = state.screen_buffer\n",
    "            misc = state.game_variables\n",
    "            action = random.choice(actions)\n",
    "            \n",
    "            reward = game.make_action(action)\n",
    "            \n",
    "            if print_info:\n",
    "                print(str(action_names[np.argmax(action)]) + \" : \" + str(action))\n",
    "                print (\"\\treward:\", reward)\n",
    "            \n",
    "            time.sleep(0.02)\n",
    "            cache = (i, state, img, misc, action)\n",
    "            caches.append(cache)\n",
    "        print (\"Result:\", game.get_total_reward())\n",
    "        time.sleep(2)\n",
    "    game.close()\n",
    "    return caches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "TURN_LEFT : [1, 0, 0]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: -0.001\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "TURN_RIGHT : [0, 1, 0]\n",
      "\treward: 0.999\n",
      "ATTACK : [0, 0, 1]\n",
      "\treward: -0.001\n",
      "Result: 0.958\n"
     ]
    }
   ],
   "source": [
    "caches = test_environment(level=-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Greyscale, crop, normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def rgb2gray(rgb):\n",
    "    return np.dot(rgb[...,:3], [0.2989, 0.5870, 0.1140])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def preprocess_frame(frame):\n",
    "\n",
    "    #Transpose channels (ch, h, w) --> (h, w, ch)\n",
    "    transposed = frame.transpose(1,2,0)\n",
    "    \n",
    "    # Greyscale\n",
    "    gray = rgb2gray(transposed)\n",
    "    \n",
    "    # Crop the screen\n",
    "    crop = gray[30:-10,30:-30]\n",
    "    \n",
    "    # Normalize Pixel Values\n",
    "    normalized = crop/255.0\n",
    "    \n",
    "    # Resize\n",
    "    preprocessed = transform.resize(normalized, [84,84])\n",
    "    \n",
    "    return preprocessed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Frame Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "stack_size = 4\n",
    "stacked_frames = deque([np.zeros((84,84), dtype=np.int) for i in range(stack_size)], maxlen=4)\n",
    "\n",
    "def stack_frames(stacked_frames, state, is_new_episode):\n",
    "    \n",
    "    frame = preprocess_frame(state)\n",
    "    \n",
    "    if is_new_episode:\n",
    "        # Clear our stacked_frames\n",
    "        stacked_frames = deque([np.zeros((84,84), dtype=np.int) for i in range(stack_size)], maxlen=4)\n",
    "        \n",
    "        #Fill in array with new episode frame\n",
    "        for i in range(stack_size):\n",
    "            stacked_frames.append(frame)\n",
    "            \n",
    "        # Stack the frames: (h x w x stack)\n",
    "        stacked_state = np.stack(stacked_frames, axis=2)\n",
    "        \n",
    "    else:\n",
    "        stacked_frames.append(frame)\n",
    "        stacked_state = np.stack(stacked_frames, axis=2) \n",
    "    \n",
    "    return stacked_state, stacked_frames"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Epsilon Greedy Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def choose_action(explore_start, explore_stop, decay_rate, decay_step, state, actions):\n",
    "    # Choose action a from state s with a probability epsilon_ using epsilon greedy.\n",
    "    \n",
    "    #explore-exploit\n",
    "    exp_exp_tradeoff = np.random.rand()\n",
    "\n",
    "    # Here we'll use an improved version of our epsilon greedy strategy used in Q-learning notebook\n",
    "    epsilon_ = explore_stop + (explore_start - explore_stop) * np.exp(-decay_rate * decay_step)\n",
    "    \n",
    "    if (epsilon_ > exp_exp_tradeoff):\n",
    "        # Explore\n",
    "        action = random.choice(possible_actions)\n",
    "        \n",
    "    else:\n",
    "        # Exploit\n",
    "        Qs = sess.run(DQNetwork.output, feed_dict = {DQNetwork.inputs_: state.reshape((1, *state.shape))})\n",
    "        \n",
    "        # Take the biggest Q value (= the best action)\n",
    "        choice = np.argmax(Qs)\n",
    "        action = possible_actions[int(choice)]\n",
    "                \n",
    "    return action, epsilon_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# ENVIRONMENT PARAMETERS\n",
    "level = -4\n",
    "game, _, action_names = create_environment(level, launch_game = False)\n",
    "action_size = len(action_names)\n",
    "game.close()\n",
    "\n",
    "# MODEL HYPERPARAMETERS\n",
    "state_size = [84,84,4]      # Our input is a stack of 4 frames hence 84x84x4 (Width, height, channels) \n",
    "learning_rate =  0.0005      # Alpha (aka learning rate)\n",
    "\n",
    "# TRAINING HYPERPARAMETERS\n",
    "total_episodes = 501        # Total episodes for training\n",
    "max_steps = 150              # Max possible steps in an episode\n",
    "batch_size = 64             \n",
    "\n",
    "# Exploration parameters for epsilon greedy strategy\n",
    "explore_start = 1.0            # exploration probability at start\n",
    "explore_stop = 0.01            # minimum exploration probability \n",
    "decay_rate = 0.0001            # exponential decay rate for exploration prob\n",
    "\n",
    "# Q learning hyperparameters\n",
    "gamma = 0.98               # Discounting rate\n",
    "\n",
    "# MEMORY HYPERPARAMETERS\n",
    "pretrain_length = batch_size   # Number of experiences stored in the Memory when initialized for the first time\n",
    "memory_size = 16000            # Number of experiences the Memory can keep\n",
    "\n",
    "# MODIFY THIS TO FALSE IF YOU JUST WANT TO SEE THE TRAINED AGENT\n",
    "training = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deep Q Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class DQNetwork:\n",
    "    def __init__(self, state_size, action_size, learning_rate, name='DQNetwork'):\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.learning_rate = learning_rate\n",
    "        \n",
    "        with tf.variable_scope(name):\n",
    "            # We create the placeholders\n",
    "            # *state_size means that we take each elements of state_size in tuple hence is like if we wrote\n",
    "            # [None, 84, 84, 4]\n",
    "            self.inputs_ = tf.placeholder(tf.float32, [None, *state_size], name=\"inputs\")\n",
    "            self.actions_ = tf.placeholder(tf.float32, [None, action_size], name=\"actions_\")\n",
    "            \n",
    "            # Remember that target_Q is the R(s,a) + ymax Qhat(s', a')\n",
    "            self.target_Q = tf.placeholder(tf.float32, [None], name=\"target\")\n",
    "            \n",
    "            \n",
    "            ###################### CONV NET start #################################\n",
    "            # Input is 84x84x4\n",
    "            self.conv1 = tf.layers.conv2d(inputs = self.inputs_,\n",
    "                                         filters = 32,\n",
    "                                         kernel_size = [8,8],\n",
    "                                         strides = [4,4],\n",
    "                                         padding = \"VALID\",\n",
    "                                          kernel_initializer=tf.contrib.layers.xavier_initializer_conv2d(),\n",
    "                                         name = \"conv1\")\n",
    "            \n",
    "            self.conv1_batchnorm = tf.layers.batch_normalization(self.conv1,\n",
    "                                                   training = True,\n",
    "                                                   epsilon = 1e-5,\n",
    "                                                     name = 'batch_norm1')\n",
    "            \n",
    "            self.conv1_out = tf.nn.relu(self.conv1_batchnorm, name=\"conv1_out\")\n",
    "            ## --> [20, 20, 32]\n",
    "            \n",
    "            self.conv2 = tf.layers.conv2d(inputs = self.conv1_out,\n",
    "                                 filters = 64,\n",
    "                                 kernel_size = [4,4],\n",
    "                                 strides = [2,2],\n",
    "                                 padding = \"VALID\",\n",
    "                                kernel_initializer=tf.contrib.layers.xavier_initializer_conv2d(),\n",
    "                                 name = \"conv2\")\n",
    "        \n",
    "            self.conv2_batchnorm = tf.layers.batch_normalization(self.conv2,\n",
    "                                                   training = True,\n",
    "                                                   epsilon = 1e-5,\n",
    "                                                     name = 'batch_norm2')\n",
    "\n",
    "            self.conv2_out = tf.nn.relu(self.conv2_batchnorm, name=\"conv2_out\")\n",
    "            ## --> [9, 9, 64]\n",
    "            \n",
    "            self.conv3 = tf.layers.conv2d(inputs = self.conv2_out,\n",
    "                                 filters = 128,\n",
    "                                 kernel_size = [4,4],\n",
    "                                 strides = [2,2],\n",
    "                                 padding = \"VALID\",\n",
    "                                kernel_initializer=tf.contrib.layers.xavier_initializer_conv2d(),\n",
    "                                 name = \"conv3\")\n",
    "        \n",
    "            self.conv3_batchnorm = tf.layers.batch_normalization(self.conv3,\n",
    "                                                   training = True,\n",
    "                                                   epsilon = 1e-5,\n",
    "                                                     name = 'batch_norm3')\n",
    "\n",
    "            self.conv3_out = tf.nn.relu(self.conv3_batchnorm, name=\"conv3_out\")\n",
    "            ## --> [3, 3, 128]\n",
    "            \n",
    "            \n",
    "            self.flatten = tf.layers.flatten(self.conv3_out)\n",
    "            ## --> [1152]\n",
    "            \n",
    "            self.fc = tf.layers.dense(inputs = self.flatten,\n",
    "                                  units = 512,\n",
    "                                  activation = tf.nn.elu,\n",
    "                                       kernel_initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                                name=\"fc1\")\n",
    "            \n",
    "            self.output = tf.layers.dense(inputs = self.fc, \n",
    "                                           kernel_initializer=tf.contrib.layers.xavier_initializer(),\n",
    "                                          units = action_size, \n",
    "                                        activation=None)\n",
    "\n",
    "            ###################### CONV NET end ###################################\n",
    "            \n",
    "            # Q is our predicted Q value.\n",
    "            self.Q = tf.reduce_sum(tf.multiply(self.output, self.actions_), axis=1)\n",
    "        \n",
    "            # mean(Qtarget - Q)^2\n",
    "            self.loss = tf.reduce_mean(tf.square(self.target_Q - self.Q))\n",
    "            \n",
    "            self.optimizer = tf.train.RMSPropOptimizer(self.learning_rate).minimize(self.loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BF888D9648>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BF888D9648>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BF888D9648>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BF888D9648>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002C089701688>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002C089701688>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002C089701688>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002C089701688>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BFE35F0348>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BFE35F0348>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BFE35F0348>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BFE35F0348>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002BFF2AF2848>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002BFF2AF2848>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002BFF2AF2848>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002BFF2AF2848>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BF88964448>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BF88964448>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BF88964448>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Conv.call of <tensorflow.python.layers.convolutional.Conv2D object at 0x000002BF88964448>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002C0897698C8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002C0897698C8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002C0897698C8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method BatchNormalization.call of <tensorflow.python.layers.normalization.BatchNormalization object at 0x000002C0897698C8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method Flatten.call of <tensorflow.python.layers.core.Flatten object at 0x000002C0897698C8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Flatten.call of <tensorflow.python.layers.core.Flatten object at 0x000002C0897698C8>>: AttributeError: module 'gast' has no attribute 'Num'\n",
      "WARNING: Entity <bound method Flatten.call of <tensorflow.python.layers.core.Flatten object at 0x000002C0897698C8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Flatten.call of <tensorflow.python.layers.core.Flatten object at 0x000002C0897698C8>>: AttributeError: module 'gast' has no attribute 'Num'\n",
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x000002C0897698C8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x000002C0897698C8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x000002C0897698C8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x000002C0897698C8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x000002C0897698C8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x000002C0897698C8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x000002C0897698C8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Dense.call of <tensorflow.python.layers.core.Dense object at 0x000002C0897698C8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    }
   ],
   "source": [
    "# Reset the graph\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Instantiate the DQNetwork\n",
    "DQNetwork = DQNetwork(state_size, action_size, learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Memory, Experience Replay, and TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class Memory():\n",
    "    def __init__(self, max_size):\n",
    "        self.buffer = deque(maxlen = max_size)\n",
    "    \n",
    "    def add(self, experience):\n",
    "        self.buffer.append(experience)\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        buffer_size = len(self.buffer)\n",
    "        index = np.random.choice(np.arange(buffer_size),\n",
    "                                size = batch_size,\n",
    "                                replace = False)\n",
    "        \n",
    "        return [self.buffer[i] for i in index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Instantiate memory\n",
    "memory = Memory(max_size = memory_size)\n",
    "\n",
    "# Render the environment\n",
    "game, possible_actions, action_names = create_environment(level)\n",
    "game.new_episode()\n",
    "\n",
    "for i in range(pretrain_length):\n",
    "    # If it's the first step\n",
    "    if i == 0:\n",
    "        # First we need a state\n",
    "        state = game.get_state().screen_buffer\n",
    "        state, stacked_frames = stack_frames(stacked_frames, state, True)\n",
    "    \n",
    "    # Random action\n",
    "    action = random.choice(possible_actions)\n",
    "    \n",
    "    # Get the rewards\n",
    "    reward = game.make_action(action)\n",
    "    \n",
    "    # Look if the episode is finished\n",
    "    done = game.is_episode_finished()\n",
    "    \n",
    "    # If we're dead\n",
    "    if done:\n",
    "        # We finished the episode\n",
    "        next_state = np.zeros(state.shape)\n",
    "        \n",
    "        # Add experience to memory\n",
    "        memory.add((state, action, reward, next_state, done))\n",
    "        \n",
    "        # Start a new episode\n",
    "        game.new_episode()\n",
    "        \n",
    "        # First we need a state\n",
    "        state = game.get_state().screen_buffer\n",
    "        \n",
    "        # Stack the frames\n",
    "        state, stacked_frames = stack_frames(stacked_frames, state, True)\n",
    "        \n",
    "    else:\n",
    "        # Get the next state\n",
    "        next_state = game.get_state().screen_buffer\n",
    "        next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)\n",
    "        \n",
    "        # Add experience to memory\n",
    "        memory.add((state, action, reward, next_state, done))\n",
    "        \n",
    "        # Our state is now the next_state\n",
    "        state = next_state\n",
    "\n",
    "game.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Setup TensorBoard Writer\n",
    "writer = tf.summary.FileWriter(\"/tensorboard/dqn/1\")\n",
    "\n",
    "## Losses\n",
    "tf.summary.scalar(\"Loss\", DQNetwork.loss)\n",
    "\n",
    "write_op = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0 Total reward: 0.951 Training loss: 3.9937 Explore P: 0.9952\n",
      "Model Saved\n",
      "Episode: 1 Total reward: -0.060000000000000005 Training loss: 14.6103 Explore P: 0.9893\n",
      "Episode: 2 Total reward: -0.059000000000000004 Training loss: 4.4238 Explore P: 0.9835\n",
      "Episode: 3 Total reward: 0.962 Training loss: 13.6550 Explore P: 0.9798\n",
      "Episode: 4 Total reward: -0.057 Training loss: 6.8514 Explore P: 0.9743\n",
      "Episode: 5 Total reward: -0.061000000000000006 Training loss: 3.8420 Explore P: 0.9684\n",
      "Episode: 6 Total reward: -0.056 Training loss: 3.5324 Explore P: 0.9631\n",
      "Episode: 7 Total reward: 0.952 Training loss: 2.3169 Explore P: 0.9585\n",
      "Episode: 8 Total reward: -0.057 Training loss: 1.3511 Explore P: 0.9531\n",
      "Episode: 9 Total reward: -0.060000000000000005 Training loss: 0.7125 Explore P: 0.9475\n",
      "Episode: 10 Total reward: -0.057 Training loss: 1.5491 Explore P: 0.9422\n",
      "Episode: 11 Total reward: -0.058 Training loss: 0.4836 Explore P: 0.9368\n",
      "Episode: 12 Total reward: -0.058 Training loss: 0.7889 Explore P: 0.9314\n",
      "Episode: 13 Total reward: 0.957 Training loss: 0.4093 Explore P: 0.9275\n",
      "Episode: 14 Total reward: -0.061000000000000006 Training loss: 0.1543 Explore P: 0.9219\n",
      "Episode: 15 Total reward: -0.059000000000000004 Training loss: 0.1208 Explore P: 0.9165\n",
      "Episode: 16 Total reward: -0.058 Training loss: 0.4025 Explore P: 0.9113\n",
      "Episode: 17 Total reward: -0.059000000000000004 Training loss: 2.1016 Explore P: 0.9060\n",
      "Episode: 18 Total reward: -0.060000000000000005 Training loss: 0.1365 Explore P: 0.9006\n",
      "Episode: 19 Total reward: -0.059000000000000004 Training loss: 0.1178 Explore P: 0.8954\n",
      "Episode: 20 Total reward: -0.056 Training loss: 0.1020 Explore P: 0.8904\n",
      "Episode: 21 Total reward: -0.058 Training loss: 0.2387 Explore P: 0.8853\n",
      "Episode: 22 Total reward: -0.062000000000000006 Training loss: 0.1053 Explore P: 0.8799\n",
      "Episode: 23 Total reward: -0.061000000000000006 Training loss: 0.1125 Explore P: 0.8746\n",
      "Episode: 24 Total reward: -0.063 Training loss: 0.0282 Explore P: 0.8692\n",
      "Episode: 25 Total reward: -0.059000000000000004 Training loss: 0.0424 Explore P: 0.8641\n",
      "Model Saved\n",
      "Episode: 26 Total reward: -0.059000000000000004 Training loss: 0.0191 Explore P: 0.8591\n",
      "Episode: 27 Total reward: -0.057 Training loss: 0.0634 Explore P: 0.8543\n",
      "Episode: 28 Total reward: -0.059000000000000004 Training loss: 0.0626 Explore P: 0.8493\n",
      "Episode: 29 Total reward: -0.059000000000000004 Training loss: 0.0190 Explore P: 0.8444\n",
      "Episode: 30 Total reward: -0.059000000000000004 Training loss: 0.1227 Explore P: 0.8395\n",
      "Episode: 31 Total reward: -0.057 Training loss: 0.0663 Explore P: 0.8348\n",
      "Episode: 32 Total reward: -0.060000000000000005 Training loss: 0.3099 Explore P: 0.8298\n",
      "Episode: 33 Total reward: -0.060000000000000005 Training loss: 0.0368 Explore P: 0.8249\n",
      "Episode: 34 Total reward: -0.058 Training loss: 0.1703 Explore P: 0.8202\n",
      "Episode: 35 Total reward: -0.060000000000000005 Training loss: 0.0611 Explore P: 0.8154\n",
      "Episode: 36 Total reward: -0.056 Training loss: 0.2944 Explore P: 0.8109\n",
      "Episode: 37 Total reward: -0.063 Training loss: 0.1003 Explore P: 0.8058\n",
      "Episode: 38 Total reward: -0.061000000000000006 Training loss: 0.7454 Explore P: 0.8010\n",
      "Episode: 39 Total reward: -0.056 Training loss: 0.0583 Explore P: 0.7966\n",
      "Episode: 40 Total reward: 0.951 Training loss: 0.0693 Explore P: 0.7927\n",
      "Episode: 41 Total reward: -0.057 Training loss: 1.2164 Explore P: 0.7883\n",
      "Episode: 42 Total reward: -0.056 Training loss: 0.0570 Explore P: 0.7839\n",
      "Episode: 43 Total reward: -0.058 Training loss: 0.0631 Explore P: 0.7795\n",
      "Episode: 44 Total reward: -0.056 Training loss: 0.1292 Explore P: 0.7752\n",
      "Episode: 45 Total reward: -0.058 Training loss: 0.3240 Explore P: 0.7708\n",
      "Episode: 46 Total reward: -0.059000000000000004 Training loss: 0.0332 Explore P: 0.7663\n",
      "Episode: 47 Total reward: -0.059000000000000004 Training loss: 0.1639 Explore P: 0.7618\n",
      "Episode: 48 Total reward: -0.057 Training loss: 0.1366 Explore P: 0.7576\n",
      "Episode: 49 Total reward: -0.056 Training loss: 0.0585 Explore P: 0.7534\n",
      "Episode: 50 Total reward: -0.056 Training loss: 0.1825 Explore P: 0.7492\n",
      "Model Saved\n",
      "Episode: 51 Total reward: -0.057 Training loss: 0.2048 Explore P: 0.7450\n",
      "Episode: 52 Total reward: -0.058 Training loss: 0.3939 Explore P: 0.7408\n",
      "Episode: 53 Total reward: -0.059000000000000004 Training loss: 0.1133 Explore P: 0.7365\n",
      "Episode: 54 Total reward: -0.057 Training loss: 0.1546 Explore P: 0.7323\n",
      "Episode: 55 Total reward: -0.056 Training loss: 0.1294 Explore P: 0.7283\n",
      "Episode: 56 Total reward: -0.058 Training loss: 0.3766 Explore P: 0.7242\n",
      "Episode: 57 Total reward: -0.061000000000000006 Training loss: 0.5134 Explore P: 0.7198\n",
      "Episode: 58 Total reward: -0.065 Training loss: 0.0627 Explore P: 0.7152\n",
      "Episode: 59 Total reward: -0.058 Training loss: 0.0643 Explore P: 0.7111\n",
      "Episode: 60 Total reward: -0.059000000000000004 Training loss: 0.1207 Explore P: 0.7070\n",
      "Episode: 61 Total reward: -0.056 Training loss: 0.0384 Explore P: 0.7031\n",
      "Episode: 62 Total reward: -0.061000000000000006 Training loss: 0.0780 Explore P: 0.6989\n",
      "Episode: 63 Total reward: -0.056 Training loss: 0.1285 Explore P: 0.6951\n",
      "Episode: 64 Total reward: -0.060000000000000005 Training loss: 0.2597 Explore P: 0.6910\n",
      "Episode: 65 Total reward: -0.061000000000000006 Training loss: 0.2854 Explore P: 0.6868\n",
      "Episode: 66 Total reward: -0.07200000000000001 Training loss: 0.0715 Explore P: 0.6820\n",
      "Episode: 67 Total reward: -0.060000000000000005 Training loss: 0.4165 Explore P: 0.6779\n",
      "Episode: 68 Total reward: -0.056 Training loss: 0.1029 Explore P: 0.6742\n",
      "Episode: 69 Total reward: -0.057 Training loss: 0.1400 Explore P: 0.6704\n",
      "Episode: 70 Total reward: -0.056 Training loss: 0.6858 Explore P: 0.6668\n",
      "Episode: 71 Total reward: -0.057 Training loss: 0.3525 Explore P: 0.6630\n",
      "Episode: 72 Total reward: -0.062000000000000006 Training loss: 0.1271 Explore P: 0.6590\n",
      "Episode: 73 Total reward: -0.060000000000000005 Training loss: 0.1827 Explore P: 0.6551\n",
      "Episode: 74 Total reward: -0.059000000000000004 Training loss: 0.1590 Explore P: 0.6513\n",
      "Episode: 75 Total reward: -0.059000000000000004 Training loss: 2.4043 Explore P: 0.6475\n",
      "Model Saved\n",
      "Episode: 76 Total reward: -0.057 Training loss: 0.3528 Explore P: 0.6439\n",
      "Episode: 77 Total reward: -0.059000000000000004 Training loss: 0.0813 Explore P: 0.6402\n",
      "Episode: 78 Total reward: -0.057 Training loss: 0.1145 Explore P: 0.6366\n",
      "Episode: 79 Total reward: -0.056 Training loss: 0.2894 Explore P: 0.6331\n",
      "Episode: 80 Total reward: -0.057 Training loss: 0.2123 Explore P: 0.6296\n",
      "Episode: 81 Total reward: -0.060000000000000005 Training loss: 0.0742 Explore P: 0.6259\n",
      "Episode: 82 Total reward: -0.060000000000000005 Training loss: 0.0793 Explore P: 0.6222\n",
      "Episode: 83 Total reward: -0.056 Training loss: 0.0930 Explore P: 0.6187\n",
      "Episode: 84 Total reward: -0.057 Training loss: 0.1189 Explore P: 0.6153\n",
      "Episode: 85 Total reward: -0.057 Training loss: 0.0902 Explore P: 0.6118\n",
      "Episode: 86 Total reward: -0.059000000000000004 Training loss: 0.0584 Explore P: 0.6083\n",
      "Episode: 87 Total reward: 0.957 Training loss: 0.0299 Explore P: 0.6057\n",
      "Episode: 88 Total reward: -0.067 Training loss: 0.0665 Explore P: 0.6018\n",
      "Episode: 89 Total reward: -0.057 Training loss: 0.0415 Explore P: 0.5984\n",
      "Episode: 90 Total reward: 0.957 Training loss: 0.0310 Explore P: 0.5959\n",
      "Episode: 91 Total reward: -0.058 Training loss: 0.0243 Explore P: 0.5925\n",
      "Episode: 92 Total reward: -0.056 Training loss: 0.0214 Explore P: 0.5892\n",
      "Episode: 93 Total reward: -0.059000000000000004 Training loss: 0.0588 Explore P: 0.5858\n",
      "Episode: 94 Total reward: -0.057 Training loss: 0.0068 Explore P: 0.5826\n",
      "Episode: 95 Total reward: 0.952 Training loss: 0.0496 Explore P: 0.5798\n",
      "Episode: 96 Total reward: 0.959 Training loss: 0.0339 Explore P: 0.5775\n",
      "Episode: 97 Total reward: -0.057 Training loss: 0.0595 Explore P: 0.5743\n",
      "Episode: 98 Total reward: 0.957 Training loss: 0.0715 Explore P: 0.5718\n",
      "Episode: 99 Total reward: 0.955 Training loss: 0.0892 Explore P: 0.5693\n",
      "Episode: 100 Total reward: -0.058 Training loss: 0.0407 Explore P: 0.5661\n",
      "Model Saved\n",
      "Episode: 101 Total reward: -0.057 Training loss: 0.1026 Explore P: 0.5629\n",
      "Episode: 102 Total reward: -0.057 Training loss: 0.0330 Explore P: 0.5598\n",
      "Episode: 103 Total reward: -0.058 Training loss: 0.0567 Explore P: 0.5566\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 104 Total reward: 0.956 Training loss: 0.0310 Explore P: 0.5542\n",
      "Episode: 105 Total reward: -0.057 Training loss: 0.0352 Explore P: 0.5511\n",
      "Episode: 106 Total reward: -0.057 Training loss: 0.0407 Explore P: 0.5480\n",
      "Episode: 107 Total reward: -0.060000000000000005 Training loss: 0.0557 Explore P: 0.5448\n",
      "Episode: 108 Total reward: -0.057 Training loss: 0.0233 Explore P: 0.5418\n",
      "Episode: 109 Total reward: -0.059000000000000004 Training loss: 0.0426 Explore P: 0.5386\n",
      "Episode: 110 Total reward: -0.061000000000000006 Training loss: 0.0446 Explore P: 0.5354\n",
      "Episode: 111 Total reward: -0.060000000000000005 Training loss: 0.0356 Explore P: 0.5323\n",
      "Episode: 112 Total reward: -0.062000000000000006 Training loss: 0.0446 Explore P: 0.5291\n",
      "Episode: 113 Total reward: -0.059000000000000004 Training loss: 0.0167 Explore P: 0.5260\n",
      "Episode: 114 Total reward: -0.057 Training loss: 0.0730 Explore P: 0.5231\n",
      "Episode: 115 Total reward: -0.059000000000000004 Training loss: 0.0371 Explore P: 0.5200\n",
      "Episode: 116 Total reward: -0.057 Training loss: 0.0134 Explore P: 0.5171\n",
      "Episode: 117 Total reward: 0.953 Training loss: 0.0484 Explore P: 0.5148\n",
      "Episode: 118 Total reward: -0.057 Training loss: 0.0394 Explore P: 0.5119\n",
      "Episode: 119 Total reward: -0.067 Training loss: 0.0088 Explore P: 0.5086\n",
      "Episode: 120 Total reward: 0.949 Training loss: 0.0125 Explore P: 0.5060\n",
      "Episode: 121 Total reward: -0.059000000000000004 Training loss: 0.0339 Explore P: 0.5031\n",
      "Episode: 122 Total reward: -0.057 Training loss: 0.0338 Explore P: 0.5003\n",
      "Episode: 123 Total reward: -0.059000000000000004 Training loss: 0.0082 Explore P: 0.4974\n",
      "Episode: 124 Total reward: -0.059000000000000004 Training loss: 0.0303 Explore P: 0.4945\n",
      "Episode: 125 Total reward: -0.064 Training loss: 0.0282 Explore P: 0.4915\n",
      "Model Saved\n",
      "Episode: 126 Total reward: -0.057 Training loss: 0.0112 Explore P: 0.4887\n",
      "Episode: 127 Total reward: -0.059000000000000004 Training loss: 0.0103 Explore P: 0.4859\n",
      "Episode: 128 Total reward: -0.056 Training loss: 0.0088 Explore P: 0.4832\n",
      "Episode: 129 Total reward: -0.060000000000000005 Training loss: 0.0156 Explore P: 0.4804\n",
      "Episode: 130 Total reward: -0.062000000000000006 Training loss: 0.0212 Explore P: 0.4775\n",
      "Episode: 131 Total reward: -0.057 Training loss: 0.0240 Explore P: 0.4748\n",
      "Episode: 132 Total reward: -0.057 Training loss: 0.0471 Explore P: 0.4722\n",
      "Episode: 133 Total reward: -0.062000000000000006 Training loss: 0.0389 Explore P: 0.4693\n",
      "Episode: 134 Total reward: -0.060000000000000005 Training loss: 0.0738 Explore P: 0.4666\n",
      "Episode: 135 Total reward: -0.062000000000000006 Training loss: 0.0542 Explore P: 0.4638\n",
      "Episode: 136 Total reward: -0.058 Training loss: 0.0383 Explore P: 0.4612\n",
      "Episode: 137 Total reward: -0.059000000000000004 Training loss: 0.0548 Explore P: 0.4585\n",
      "Episode: 138 Total reward: -0.056 Training loss: 0.0806 Explore P: 0.4560\n",
      "Episode: 139 Total reward: -0.058 Training loss: 0.0551 Explore P: 0.4534\n",
      "Episode: 140 Total reward: -0.060000000000000005 Training loss: 0.0955 Explore P: 0.4508\n",
      "Episode: 141 Total reward: -0.057 Training loss: 0.0317 Explore P: 0.4483\n",
      "Episode: 142 Total reward: -0.058 Training loss: 0.0179 Explore P: 0.4457\n",
      "Episode: 143 Total reward: -0.056 Training loss: 0.0221 Explore P: 0.4433\n",
      "Episode: 144 Total reward: -0.057 Training loss: 0.0386 Explore P: 0.4408\n",
      "Episode: 145 Total reward: -0.063 Training loss: 0.0418 Explore P: 0.4381\n",
      "Episode: 146 Total reward: -0.056 Training loss: 0.0803 Explore P: 0.4357\n",
      "Episode: 147 Total reward: -0.057 Training loss: 0.0578 Explore P: 0.4333\n",
      "Episode: 148 Total reward: -0.056 Training loss: 0.0352 Explore P: 0.4309\n",
      "Episode: 149 Total reward: -0.057 Training loss: 0.1009 Explore P: 0.4286\n",
      "Episode: 150 Total reward: -0.063 Training loss: 0.1256 Explore P: 0.4259\n",
      "Model Saved\n",
      "Episode: 151 Total reward: -0.062000000000000006 Training loss: 0.0284 Explore P: 0.4234\n",
      "Episode: 152 Total reward: -0.056 Training loss: 0.0767 Explore P: 0.4210\n",
      "Episode: 153 Total reward: 0.956 Training loss: 0.0610 Explore P: 0.4192\n",
      "Episode: 154 Total reward: -0.060000000000000005 Training loss: 0.0521 Explore P: 0.4168\n",
      "Episode: 155 Total reward: -0.057 Training loss: 0.0358 Explore P: 0.4145\n",
      "Episode: 156 Total reward: -0.058 Training loss: 0.0309 Explore P: 0.4121\n",
      "Episode: 157 Total reward: -0.056 Training loss: 0.0992 Explore P: 0.4099\n",
      "Episode: 158 Total reward: -0.056 Training loss: 0.0664 Explore P: 0.4077\n",
      "Episode: 159 Total reward: -0.057 Training loss: 0.0624 Explore P: 0.4054\n",
      "Episode: 160 Total reward: -0.056 Training loss: 0.0558 Explore P: 0.4032\n",
      "Episode: 161 Total reward: -0.057 Training loss: 0.0673 Explore P: 0.4010\n",
      "Episode: 162 Total reward: -0.056 Training loss: 0.1229 Explore P: 0.3988\n",
      "Episode: 163 Total reward: -0.056 Training loss: 0.0378 Explore P: 0.3966\n",
      "Episode: 164 Total reward: -0.057 Training loss: 0.1153 Explore P: 0.3944\n",
      "Episode: 165 Total reward: -0.058 Training loss: 0.0174 Explore P: 0.3922\n",
      "Episode: 166 Total reward: -0.061000000000000006 Training loss: 0.0578 Explore P: 0.3899\n",
      "Episode: 167 Total reward: -0.061000000000000006 Training loss: 0.0737 Explore P: 0.3876\n",
      "Episode: 168 Total reward: -0.056 Training loss: 0.0637 Explore P: 0.3854\n",
      "Episode: 169 Total reward: -0.056 Training loss: 0.0734 Explore P: 0.3833\n",
      "Episode: 170 Total reward: -0.059000000000000004 Training loss: 0.0439 Explore P: 0.3811\n",
      "Episode: 171 Total reward: -0.058 Training loss: 0.0099 Explore P: 0.3790\n",
      "Episode: 172 Total reward: -0.056 Training loss: 0.0399 Explore P: 0.3769\n",
      "Episode: 173 Total reward: -0.067 Training loss: 0.0348 Explore P: 0.3745\n",
      "Episode: 174 Total reward: -0.066 Training loss: 0.0311 Explore P: 0.3721\n",
      "Episode: 175 Total reward: -0.058 Training loss: 0.0368 Explore P: 0.3700\n",
      "Model Saved\n",
      "Episode: 176 Total reward: -0.057 Training loss: 0.0351 Explore P: 0.3680\n",
      "Episode: 177 Total reward: -0.057 Training loss: 0.0542 Explore P: 0.3659\n",
      "Episode: 178 Total reward: -0.057 Training loss: 0.0920 Explore P: 0.3639\n",
      "Episode: 179 Total reward: -0.056 Training loss: 0.0565 Explore P: 0.3619\n",
      "Episode: 180 Total reward: -0.064 Training loss: 0.0347 Explore P: 0.3597\n",
      "Episode: 181 Total reward: 0.952 Training loss: 0.0116 Explore P: 0.3580\n",
      "Episode: 182 Total reward: -0.056 Training loss: 0.0186 Explore P: 0.3561\n",
      "Episode: 183 Total reward: -0.057 Training loss: 0.0666 Explore P: 0.3541\n",
      "Episode: 184 Total reward: -0.058 Training loss: 0.0440 Explore P: 0.3521\n",
      "Episode: 185 Total reward: -0.060000000000000005 Training loss: 0.7226 Explore P: 0.3501\n",
      "Episode: 186 Total reward: -0.059000000000000004 Training loss: 0.1727 Explore P: 0.3481\n",
      "Episode: 187 Total reward: -0.056 Training loss: 0.0260 Explore P: 0.3462\n",
      "Episode: 188 Total reward: -0.057 Training loss: 0.0738 Explore P: 0.3443\n",
      "Episode: 189 Total reward: -0.056 Training loss: 0.3960 Explore P: 0.3424\n",
      "Episode: 190 Total reward: -0.057 Training loss: 0.0251 Explore P: 0.3405\n",
      "Episode: 191 Total reward: -0.056 Training loss: 0.0256 Explore P: 0.3387\n",
      "Episode: 192 Total reward: -0.060000000000000005 Training loss: 0.3310 Explore P: 0.3367\n",
      "Episode: 193 Total reward: -0.059000000000000004 Training loss: 0.0618 Explore P: 0.3348\n",
      "Episode: 194 Total reward: -0.061000000000000006 Training loss: 0.0064 Explore P: 0.3328\n",
      "Episode: 195 Total reward: -0.056 Training loss: 0.0101 Explore P: 0.3310\n",
      "Episode: 196 Total reward: -0.058 Training loss: 0.0601 Explore P: 0.3291\n",
      "Episode: 197 Total reward: 0.957 Training loss: 0.0649 Explore P: 0.3278\n",
      "Episode: 198 Total reward: -0.058 Training loss: 0.2272 Explore P: 0.3259\n",
      "Episode: 199 Total reward: -0.062000000000000006 Training loss: 0.1012 Explore P: 0.3240\n",
      "Episode: 200 Total reward: -0.057 Training loss: 0.2955 Explore P: 0.3222\n",
      "Model Saved\n",
      "Episode: 201 Total reward: 0.961 Training loss: 0.0565 Explore P: 0.3210\n",
      "Episode: 202 Total reward: -0.059000000000000004 Training loss: 0.1856 Explore P: 0.3191\n",
      "Episode: 203 Total reward: -0.057 Training loss: 0.0341 Explore P: 0.3174\n",
      "Episode: 204 Total reward: -0.062000000000000006 Training loss: 0.0868 Explore P: 0.3155\n",
      "Episode: 205 Total reward: -0.059000000000000004 Training loss: 0.0690 Explore P: 0.3137\n",
      "Episode: 206 Total reward: -0.057 Training loss: 0.0805 Explore P: 0.3120\n",
      "Episode: 207 Total reward: -0.059000000000000004 Training loss: 0.1008 Explore P: 0.3102\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 208 Total reward: -0.065 Training loss: 0.0324 Explore P: 0.3082\n",
      "Episode: 209 Total reward: -0.056 Training loss: 0.0259 Explore P: 0.3066\n",
      "Episode: 210 Total reward: 0.957 Training loss: 0.1450 Explore P: 0.3053\n",
      "Episode: 211 Total reward: -0.056 Training loss: 0.0462 Explore P: 0.3037\n",
      "Episode: 212 Total reward: -0.059000000000000004 Training loss: 0.0920 Explore P: 0.3019\n",
      "Episode: 213 Total reward: -0.060000000000000005 Training loss: 0.1588 Explore P: 0.3002\n",
      "Episode: 214 Total reward: -0.056 Training loss: 0.0852 Explore P: 0.2986\n",
      "Episode: 215 Total reward: -0.056 Training loss: 0.0303 Explore P: 0.2969\n",
      "Episode: 216 Total reward: -0.059000000000000004 Training loss: 0.0602 Explore P: 0.2953\n",
      "Episode: 217 Total reward: -0.063 Training loss: 0.0637 Explore P: 0.2935\n",
      "Episode: 218 Total reward: -0.057 Training loss: 0.0838 Explore P: 0.2919\n",
      "Episode: 219 Total reward: 0.954 Training loss: 0.1155 Explore P: 0.2906\n",
      "Episode: 220 Total reward: -0.058 Training loss: 0.0453 Explore P: 0.2889\n",
      "Episode: 221 Total reward: -0.061000000000000006 Training loss: 0.0939 Explore P: 0.2872\n",
      "Episode: 222 Total reward: -0.062000000000000006 Training loss: 0.0439 Explore P: 0.2855\n",
      "Episode: 223 Total reward: 0.963 Training loss: 0.0542 Explore P: 0.2845\n",
      "Episode: 224 Total reward: -0.057 Training loss: 0.0463 Explore P: 0.2830\n",
      "Episode: 225 Total reward: -0.07 Training loss: 0.0434 Explore P: 0.2811\n",
      "Model Saved\n",
      "Episode: 226 Total reward: -0.056 Training loss: 0.0411 Explore P: 0.2795\n",
      "Episode: 227 Total reward: -0.061000000000000006 Training loss: 0.0489 Explore P: 0.2779\n",
      "Episode: 228 Total reward: -0.058 Training loss: 0.0392 Explore P: 0.2763\n",
      "Episode: 229 Total reward: -0.057 Training loss: 0.1736 Explore P: 0.2748\n",
      "Episode: 230 Total reward: -0.058 Training loss: 0.0271 Explore P: 0.2733\n",
      "Episode: 231 Total reward: -0.061000000000000006 Training loss: 0.0454 Explore P: 0.2717\n",
      "Episode: 232 Total reward: -0.057 Training loss: 0.0368 Explore P: 0.2702\n",
      "Episode: 233 Total reward: -0.058 Training loss: 0.0480 Explore P: 0.2687\n",
      "Episode: 234 Total reward: -0.058 Training loss: 0.0271 Explore P: 0.2672\n",
      "Episode: 235 Total reward: -0.057 Training loss: 0.0270 Explore P: 0.2658\n",
      "Episode: 236 Total reward: -0.058 Training loss: 0.0221 Explore P: 0.2643\n",
      "Episode: 237 Total reward: -0.056 Training loss: 0.0277 Explore P: 0.2629\n",
      "Episode: 238 Total reward: -0.062000000000000006 Training loss: 0.0386 Explore P: 0.2613\n",
      "Episode: 239 Total reward: -0.065 Training loss: 0.0929 Explore P: 0.2597\n",
      "Episode: 240 Total reward: -0.060000000000000005 Training loss: 0.0336 Explore P: 0.2582\n",
      "Episode: 241 Total reward: -0.056 Training loss: 0.0626 Explore P: 0.2568\n",
      "Episode: 242 Total reward: 0.955 Training loss: 0.0423 Explore P: 0.2557\n",
      "Episode: 243 Total reward: -0.061000000000000006 Training loss: 0.0275 Explore P: 0.2542\n",
      "Episode: 244 Total reward: -0.064 Training loss: 0.0466 Explore P: 0.2526\n",
      "Episode: 245 Total reward: -0.057 Training loss: 0.0524 Explore P: 0.2512\n",
      "Episode: 246 Total reward: -0.057 Training loss: 0.1865 Explore P: 0.2499\n",
      "Episode: 247 Total reward: -0.057 Training loss: 0.0765 Explore P: 0.2485\n",
      "Episode: 248 Total reward: -0.059000000000000004 Training loss: 0.6946 Explore P: 0.2471\n",
      "Episode: 249 Total reward: -0.056 Training loss: 0.0617 Explore P: 0.2458\n",
      "Episode: 250 Total reward: -0.057 Training loss: 0.0942 Explore P: 0.2444\n",
      "Model Saved\n",
      "Episode: 251 Total reward: -0.056 Training loss: 0.0750 Explore P: 0.2431\n",
      "Episode: 252 Total reward: -0.056 Training loss: 0.0436 Explore P: 0.2418\n",
      "Episode: 253 Total reward: -0.057 Training loss: 0.0387 Explore P: 0.2405\n",
      "Episode: 254 Total reward: -0.058 Training loss: 0.1363 Explore P: 0.2392\n",
      "Episode: 255 Total reward: -0.059000000000000004 Training loss: 0.1318 Explore P: 0.2378\n",
      "Episode: 256 Total reward: -0.060000000000000005 Training loss: 0.0564 Explore P: 0.2365\n",
      "Episode: 257 Total reward: -0.065 Training loss: 0.0474 Explore P: 0.2350\n",
      "Episode: 258 Total reward: -0.057 Training loss: 0.2289 Explore P: 0.2337\n",
      "Episode: 259 Total reward: -0.056 Training loss: 0.3836 Explore P: 0.2325\n",
      "Episode: 260 Total reward: -0.056 Training loss: 0.0695 Explore P: 0.2312\n",
      "Episode: 261 Total reward: -0.057 Training loss: 0.1265 Explore P: 0.2300\n",
      "Episode: 262 Total reward: -0.056 Training loss: 0.0235 Explore P: 0.2287\n",
      "Episode: 263 Total reward: -0.058 Training loss: 0.3792 Explore P: 0.2275\n",
      "Episode: 264 Total reward: -0.058 Training loss: 0.1610 Explore P: 0.2262\n",
      "Episode: 265 Total reward: -0.058 Training loss: 0.0553 Explore P: 0.2250\n",
      "Episode: 266 Total reward: -0.057 Training loss: 0.0870 Explore P: 0.2237\n",
      "Episode: 267 Total reward: -0.056 Training loss: 0.0547 Explore P: 0.2226\n",
      "Episode: 268 Total reward: 0.956 Training loss: 0.1609 Explore P: 0.2216\n",
      "Episode: 269 Total reward: -0.056 Training loss: 0.0801 Explore P: 0.2204\n",
      "Episode: 270 Total reward: -0.058 Training loss: 0.0291 Explore P: 0.2192\n",
      "Episode: 271 Total reward: -0.058 Training loss: 0.0872 Explore P: 0.2180\n",
      "Episode: 272 Total reward: -0.057 Training loss: 0.0932 Explore P: 0.2168\n",
      "Episode: 273 Total reward: -0.059000000000000004 Training loss: 0.1602 Explore P: 0.2156\n",
      "Episode: 274 Total reward: -0.057 Training loss: 0.0265 Explore P: 0.2144\n",
      "Episode: 275 Total reward: 0.956 Training loss: 0.0271 Explore P: 0.2135\n",
      "Model Saved\n",
      "Episode: 276 Total reward: -0.057 Training loss: 0.3777 Explore P: 0.2124\n",
      "Episode: 277 Total reward: -0.057 Training loss: 0.4023 Explore P: 0.2112\n",
      "Episode: 278 Total reward: -0.056 Training loss: 0.0724 Explore P: 0.2101\n",
      "Episode: 279 Total reward: -0.059000000000000004 Training loss: 0.1643 Explore P: 0.2089\n",
      "Episode: 280 Total reward: -0.057 Training loss: 0.0356 Explore P: 0.2078\n",
      "Episode: 281 Total reward: -0.065 Training loss: 0.1172 Explore P: 0.2065\n",
      "Episode: 282 Total reward: -0.057 Training loss: 0.0551 Explore P: 0.2054\n",
      "Episode: 283 Total reward: -0.056 Training loss: 0.0456 Explore P: 0.2043\n",
      "Episode: 284 Total reward: -0.063 Training loss: 0.0345 Explore P: 0.2031\n",
      "Episode: 285 Total reward: 0.954 Training loss: 0.0293 Explore P: 0.2022\n",
      "Episode: 286 Total reward: 0.956 Training loss: 0.0361 Explore P: 0.2014\n",
      "Episode: 287 Total reward: 0.955 Training loss: 0.0757 Explore P: 0.2005\n",
      "Episode: 288 Total reward: -0.057 Training loss: 0.1772 Explore P: 0.1994\n",
      "Episode: 289 Total reward: -0.057 Training loss: 0.0435 Explore P: 0.1984\n",
      "Episode: 290 Total reward: -0.059000000000000004 Training loss: 0.0531 Explore P: 0.1972\n",
      "Episode: 291 Total reward: -0.056 Training loss: 0.0478 Explore P: 0.1962\n",
      "Episode: 292 Total reward: -0.056 Training loss: 0.0461 Explore P: 0.1952\n",
      "Episode: 293 Total reward: -0.060000000000000005 Training loss: 0.0142 Explore P: 0.1940\n",
      "Episode: 294 Total reward: -0.056 Training loss: 0.0489 Explore P: 0.1930\n",
      "Episode: 295 Total reward: -0.056 Training loss: 0.0598 Explore P: 0.1920\n",
      "Episode: 296 Total reward: -0.057 Training loss: 0.0437 Explore P: 0.1910\n",
      "Episode: 297 Total reward: -0.057 Training loss: 0.0458 Explore P: 0.1899\n",
      "Episode: 298 Total reward: -0.056 Training loss: 0.0250 Explore P: 0.1889\n",
      "Episode: 299 Total reward: -0.058 Training loss: 0.0616 Explore P: 0.1879\n",
      "Episode: 300 Total reward: -0.058 Training loss: 0.0467 Explore P: 0.1869\n",
      "Model Saved\n",
      "Episode: 301 Total reward: 0.956 Training loss: 0.0168 Explore P: 0.1861\n",
      "Episode: 302 Total reward: -0.057 Training loss: 0.0125 Explore P: 0.1851\n",
      "Episode: 303 Total reward: -0.057 Training loss: 0.0388 Explore P: 0.1841\n",
      "Episode: 304 Total reward: -0.059000000000000004 Training loss: 0.0276 Explore P: 0.1831\n",
      "Episode: 305 Total reward: -0.057 Training loss: 0.0047 Explore P: 0.1821\n",
      "Episode: 306 Total reward: -0.056 Training loss: 0.0161 Explore P: 0.1811\n",
      "Episode: 307 Total reward: -0.056 Training loss: 0.0222 Explore P: 0.1802\n",
      "Episode: 308 Total reward: -0.058 Training loss: 0.0284 Explore P: 0.1792\n",
      "Episode: 309 Total reward: -0.059000000000000004 Training loss: 0.0604 Explore P: 0.1782\n",
      "Episode: 310 Total reward: -0.056 Training loss: 0.0448 Explore P: 0.1773\n",
      "Episode: 311 Total reward: -0.061000000000000006 Training loss: 0.0468 Explore P: 0.1762\n",
      "Episode: 312 Total reward: -0.056 Training loss: 0.0633 Explore P: 0.1753\n",
      "Episode: 313 Total reward: -0.061000000000000006 Training loss: 0.0281 Explore P: 0.1743\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 314 Total reward: -0.057 Training loss: 0.0228 Explore P: 0.1734\n",
      "Episode: 315 Total reward: 0.949 Training loss: 0.0149 Explore P: 0.1725\n",
      "Episode: 316 Total reward: -0.056 Training loss: 0.0106 Explore P: 0.1716\n",
      "Episode: 317 Total reward: 0.948 Training loss: 0.0662 Explore P: 0.1708\n",
      "Episode: 318 Total reward: -0.056 Training loss: 0.1454 Explore P: 0.1699\n",
      "Episode: 319 Total reward: -0.057 Training loss: 0.0792 Explore P: 0.1690\n",
      "Episode: 320 Total reward: -0.059000000000000004 Training loss: 0.0760 Explore P: 0.1680\n",
      "Episode: 321 Total reward: -0.056 Training loss: 0.0248 Explore P: 0.1672\n",
      "Episode: 322 Total reward: -0.057 Training loss: 0.0434 Explore P: 0.1663\n",
      "Episode: 323 Total reward: -0.058 Training loss: 0.0957 Explore P: 0.1654\n",
      "Episode: 324 Total reward: -0.061000000000000006 Training loss: 0.1759 Explore P: 0.1644\n",
      "Episode: 325 Total reward: -0.064 Training loss: 0.0339 Explore P: 0.1634\n",
      "Model Saved\n",
      "Episode: 326 Total reward: 0.958 Training loss: 0.0742 Explore P: 0.1628\n",
      "Episode: 327 Total reward: -0.059000000000000004 Training loss: 0.0385 Explore P: 0.1619\n",
      "Episode: 328 Total reward: -0.058 Training loss: 0.0067 Explore P: 0.1610\n",
      "Episode: 329 Total reward: -0.059000000000000004 Training loss: 0.0276 Explore P: 0.1601\n",
      "Episode: 330 Total reward: -0.057 Training loss: 0.0739 Explore P: 0.1593\n",
      "Episode: 331 Total reward: -0.057 Training loss: 0.1141 Explore P: 0.1584\n",
      "Episode: 332 Total reward: -0.057 Training loss: 0.0354 Explore P: 0.1576\n",
      "Episode: 333 Total reward: -0.057 Training loss: 0.1337 Explore P: 0.1567\n",
      "Episode: 334 Total reward: -0.056 Training loss: 0.0136 Explore P: 0.1559\n",
      "Episode: 335 Total reward: -0.058 Training loss: 0.0299 Explore P: 0.1551\n",
      "Episode: 336 Total reward: -0.057 Training loss: 0.0473 Explore P: 0.1543\n",
      "Episode: 337 Total reward: -0.056 Training loss: 0.0568 Explore P: 0.1535\n",
      "Episode: 338 Total reward: -0.065 Training loss: 0.0411 Explore P: 0.1525\n",
      "Episode: 339 Total reward: -0.067 Training loss: 0.0431 Explore P: 0.1516\n",
      "Episode: 340 Total reward: -0.057 Training loss: 0.0613 Explore P: 0.1508\n",
      "Episode: 341 Total reward: -0.063 Training loss: 0.0525 Explore P: 0.1499\n",
      "Episode: 342 Total reward: -0.057 Training loss: 0.0989 Explore P: 0.1491\n",
      "Episode: 343 Total reward: -0.061000000000000006 Training loss: 0.0572 Explore P: 0.1482\n",
      "Episode: 344 Total reward: -0.056 Training loss: 0.0198 Explore P: 0.1475\n",
      "Episode: 345 Total reward: -0.057 Training loss: 0.0271 Explore P: 0.1467\n",
      "Episode: 346 Total reward: -0.065 Training loss: 0.1351 Explore P: 0.1458\n",
      "Episode: 347 Total reward: -0.056 Training loss: 0.0150 Explore P: 0.1450\n",
      "Episode: 348 Total reward: -0.056 Training loss: 0.0123 Explore P: 0.1443\n",
      "Episode: 349 Total reward: -0.057 Training loss: 0.0267 Explore P: 0.1435\n",
      "Episode: 350 Total reward: -0.057 Training loss: 0.0412 Explore P: 0.1428\n",
      "Model Saved\n",
      "Episode: 351 Total reward: -0.058 Training loss: 0.0507 Explore P: 0.1420\n",
      "Episode: 352 Total reward: 0.956 Training loss: 0.0626 Explore P: 0.1414\n",
      "Episode: 353 Total reward: -0.056 Training loss: 0.0180 Explore P: 0.1407\n",
      "Episode: 354 Total reward: 0.956 Training loss: 0.1028 Explore P: 0.1401\n",
      "Episode: 355 Total reward: -0.064 Training loss: 0.0803 Explore P: 0.1393\n",
      "Episode: 356 Total reward: -0.056 Training loss: 0.0556 Explore P: 0.1386\n",
      "Episode: 357 Total reward: -0.057 Training loss: 0.0454 Explore P: 0.1378\n",
      "Episode: 358 Total reward: -0.057 Training loss: 0.0318 Explore P: 0.1371\n",
      "Episode: 359 Total reward: -0.058 Training loss: 0.0838 Explore P: 0.1364\n",
      "Episode: 360 Total reward: -0.063 Training loss: 0.0218 Explore P: 0.1356\n",
      "Episode: 361 Total reward: -0.061000000000000006 Training loss: 0.0198 Explore P: 0.1348\n",
      "Episode: 362 Total reward: -0.061000000000000006 Training loss: 0.1380 Explore P: 0.1341\n",
      "Episode: 363 Total reward: -0.056 Training loss: 0.0479 Explore P: 0.1334\n",
      "Episode: 364 Total reward: -0.067 Training loss: 0.0282 Explore P: 0.1325\n",
      "Episode: 365 Total reward: -0.056 Training loss: 0.1930 Explore P: 0.1319\n",
      "Episode: 366 Total reward: -0.058 Training loss: 0.1427 Explore P: 0.1311\n",
      "Episode: 367 Total reward: -0.056 Training loss: 0.0228 Explore P: 0.1305\n",
      "Episode: 368 Total reward: -0.057 Training loss: 0.3031 Explore P: 0.1298\n",
      "Episode: 369 Total reward: -0.056 Training loss: 0.0422 Explore P: 0.1291\n",
      "Episode: 370 Total reward: 0.956 Training loss: 0.0628 Explore P: 0.1286\n",
      "Episode: 371 Total reward: -0.057 Training loss: 0.0440 Explore P: 0.1279\n",
      "Episode: 372 Total reward: -0.059000000000000004 Training loss: 0.0648 Explore P: 0.1272\n",
      "Episode: 373 Total reward: -0.059000000000000004 Training loss: 0.0569 Explore P: 0.1265\n",
      "Episode: 374 Total reward: -0.057 Training loss: 0.0885 Explore P: 0.1259\n",
      "Episode: 375 Total reward: -0.057 Training loss: 0.1156 Explore P: 0.1252\n",
      "Model Saved\n",
      "Episode: 376 Total reward: 0.959 Training loss: 0.2227 Explore P: 0.1247\n",
      "Episode: 377 Total reward: -0.056 Training loss: 0.0323 Explore P: 0.1241\n",
      "Episode: 378 Total reward: -0.059000000000000004 Training loss: 0.0424 Explore P: 0.1234\n",
      "Episode: 379 Total reward: -0.056 Training loss: 0.2032 Explore P: 0.1228\n",
      "Episode: 380 Total reward: -0.057 Training loss: 0.0654 Explore P: 0.1222\n",
      "Episode: 381 Total reward: -0.056 Training loss: 0.3111 Explore P: 0.1215\n",
      "Episode: 382 Total reward: -0.056 Training loss: 0.1124 Explore P: 0.1209\n",
      "Episode: 383 Total reward: -0.062000000000000006 Training loss: 0.0346 Explore P: 0.1202\n",
      "Episode: 384 Total reward: -0.057 Training loss: 0.0992 Explore P: 0.1196\n",
      "Episode: 385 Total reward: -0.057 Training loss: 0.0281 Explore P: 0.1190\n",
      "Episode: 386 Total reward: -0.058 Training loss: 0.0671 Explore P: 0.1183\n",
      "Episode: 387 Total reward: -0.056 Training loss: 0.0579 Explore P: 0.1177\n",
      "Episode: 388 Total reward: -0.063 Training loss: 0.0369 Explore P: 0.1171\n",
      "Episode: 389 Total reward: -0.057 Training loss: 0.0274 Explore P: 0.1165\n",
      "Episode: 390 Total reward: -0.056 Training loss: 0.0839 Explore P: 0.1159\n",
      "Episode: 391 Total reward: -0.059000000000000004 Training loss: 0.1037 Explore P: 0.1152\n",
      "Episode: 392 Total reward: -0.062000000000000006 Training loss: 0.0951 Explore P: 0.1146\n",
      "Episode: 393 Total reward: -0.060000000000000005 Training loss: 0.1383 Explore P: 0.1140\n",
      "Episode: 394 Total reward: -0.057 Training loss: 0.0286 Explore P: 0.1134\n",
      "Episode: 395 Total reward: 0.957 Training loss: 0.0704 Explore P: 0.1129\n",
      "Episode: 396 Total reward: -0.056 Training loss: 0.0781 Explore P: 0.1124\n",
      "Episode: 397 Total reward: -0.057 Training loss: 0.0923 Explore P: 0.1118\n",
      "Episode: 398 Total reward: -0.056 Training loss: 0.0269 Explore P: 0.1112\n",
      "Episode: 399 Total reward: -0.064 Training loss: 0.0373 Explore P: 0.1106\n",
      "Episode: 400 Total reward: -0.056 Training loss: 0.2270 Explore P: 0.1100\n",
      "Model Saved\n",
      "Episode: 401 Total reward: -0.057 Training loss: 0.0422 Explore P: 0.1094\n",
      "Episode: 402 Total reward: -0.057 Training loss: 0.0841 Explore P: 0.1089\n",
      "Episode: 403 Total reward: -0.056 Training loss: 0.0354 Explore P: 0.1083\n",
      "Episode: 404 Total reward: -0.057 Training loss: 0.0469 Explore P: 0.1077\n",
      "Episode: 405 Total reward: -0.056 Training loss: 0.0661 Explore P: 0.1072\n",
      "Episode: 406 Total reward: 0.962 Training loss: 0.0721 Explore P: 0.1068\n",
      "Episode: 407 Total reward: -0.056 Training loss: 0.0568 Explore P: 0.1063\n",
      "Episode: 408 Total reward: -0.059000000000000004 Training loss: 0.0865 Explore P: 0.1057\n",
      "Episode: 409 Total reward: -0.057 Training loss: 0.0676 Explore P: 0.1052\n",
      "Episode: 410 Total reward: 0.957 Training loss: 0.2768 Explore P: 0.1048\n",
      "Episode: 411 Total reward: -0.059000000000000004 Training loss: 0.0454 Explore P: 0.1042\n",
      "Episode: 412 Total reward: -0.061000000000000006 Training loss: 0.0887 Explore P: 0.1036\n",
      "Episode: 413 Total reward: -0.056 Training loss: 0.2036 Explore P: 0.1031\n",
      "Episode: 414 Total reward: -0.058 Training loss: 0.9856 Explore P: 0.1026\n",
      "Episode: 415 Total reward: -0.056 Training loss: 0.6387 Explore P: 0.1021\n",
      "Episode: 416 Total reward: -0.058 Training loss: 0.1133 Explore P: 0.1015\n",
      "Episode: 417 Total reward: -0.056 Training loss: 0.1607 Explore P: 0.1010\n",
      "Episode: 418 Total reward: -0.056 Training loss: 0.1405 Explore P: 0.1005\n",
      "Episode: 419 Total reward: -0.058 Training loss: 0.0690 Explore P: 0.1000\n",
      "Episode: 420 Total reward: -0.056 Training loss: 0.0433 Explore P: 0.0995\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 421 Total reward: -0.057 Training loss: 0.1655 Explore P: 0.0990\n",
      "Episode: 422 Total reward: 0.959 Training loss: 0.0351 Explore P: 0.0986\n",
      "Episode: 423 Total reward: -0.059000000000000004 Training loss: 0.0451 Explore P: 0.0981\n",
      "Episode: 424 Total reward: -0.063 Training loss: 0.1148 Explore P: 0.0975\n",
      "Episode: 425 Total reward: -0.060000000000000005 Training loss: 0.0881 Explore P: 0.0970\n",
      "Model Saved\n",
      "Episode: 426 Total reward: -0.059000000000000004 Training loss: 0.0498 Explore P: 0.0965\n",
      "Episode: 427 Total reward: -0.065 Training loss: 0.0815 Explore P: 0.0959\n",
      "Episode: 428 Total reward: -0.059000000000000004 Training loss: 0.0362 Explore P: 0.0954\n",
      "Episode: 429 Total reward: -0.059000000000000004 Training loss: 0.1232 Explore P: 0.0949\n",
      "Episode: 430 Total reward: -0.057 Training loss: 0.0490 Explore P: 0.0945\n",
      "Episode: 431 Total reward: -0.056 Training loss: 0.1222 Explore P: 0.0940\n",
      "Episode: 432 Total reward: -0.056 Training loss: 0.1169 Explore P: 0.0935\n"
     ]
    },
    {
     "ename": "ViZDoomUnexpectedExitException",
     "evalue": "Controlled ViZDoom instance exited unexpectedly.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mViZDoomUnexpectedExitException\u001b[0m            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-85-962f6512f923>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     34\u001b[0m                 \u001b[1;31m#Choose Action\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m                 \u001b[0maction\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepsilon_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mchoose_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexplore_start\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexplore_stop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdecay_rate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdecay_step\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpossible_actions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m                 \u001b[0mreward\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m                 \u001b[0mepisode_rewards\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreward\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mViZDoomUnexpectedExitException\u001b[0m: Controlled ViZDoom instance exited unexpectedly."
     ]
    }
   ],
   "source": [
    "# Saver will help us to save our model\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "if training == True:\n",
    "    with tf.Session() as sess:\n",
    "        \n",
    "        decay_step = 0\n",
    "        \n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        game.init()\n",
    "\n",
    "        for episode in range(total_episodes):\n",
    "            \n",
    "            episode_rewards = []\n",
    "            first_state = True\n",
    "            \n",
    "            game.new_episode()    \n",
    "            \n",
    "            while not game.is_episode_finished():\n",
    "                \n",
    "                if first_state:\n",
    "                    state = game.get_state().screen_buffer\n",
    "                    state, stacked_frames = stack_frames(stacked_frames, state, True)\n",
    "                    first_state = False\n",
    "                \n",
    "                #Load State\n",
    "                next_state = game.get_state().screen_buffer\n",
    "                next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)\n",
    "                memory.add((state, action, reward, next_state, done))\n",
    "                state = next_state\n",
    "                \n",
    "                decay_step +=1\n",
    "                \n",
    "                #Choose Action\n",
    "                action, epsilon_ = choose_action(explore_start, explore_stop, decay_rate, decay_step, state, possible_actions)\n",
    "                reward = game.make_action(action)\n",
    "                episode_rewards.append(reward)\n",
    "\n",
    "                ### LEARNING PART            \n",
    "                # Obtain random mini-batch from memory\n",
    "                batch = memory.sample(batch_size)\n",
    "                states_mb = np.array([each[0] for each in batch], ndmin=3)\n",
    "                actions_mb = np.array([each[1] for each in batch])\n",
    "                rewards_mb = np.array([each[2] for each in batch]) \n",
    "                next_states_mb = np.array([each[3] for each in batch], ndmin=3)\n",
    "                dones_mb = np.array([each[4] for each in batch])\n",
    "\n",
    "                target_Qs_batch = []\n",
    "\n",
    "                # Get Q values for next_state\n",
    "                Qs_next_state = sess.run(DQNetwork.output, feed_dict = {DQNetwork.inputs_: next_states_mb})\n",
    "                \n",
    "                # Set Q_target = r if the episode ends at s+1, otherwise set Q_target = r + gamma*maxQ(s', a')\n",
    "                for i in range(0, len(batch)):\n",
    "                    terminal = dones_mb[i]\n",
    "\n",
    "                    # If we are in a terminal state, only equals reward\n",
    "                    if terminal:\n",
    "                        target_Qs_batch.append(rewards_mb[i])\n",
    "                        \n",
    "                    else:\n",
    "                        target = rewards_mb[i] + gamma * np.max(Qs_next_state[i])\n",
    "                        target_Qs_batch.append(target)\n",
    "                        \n",
    "\n",
    "                targets_mb = np.array([each for each in target_Qs_batch])\n",
    "\n",
    "                loss, _ = sess.run([DQNetwork.loss, DQNetwork.optimizer],\n",
    "                                    feed_dict={DQNetwork.inputs_: states_mb,\n",
    "                                               DQNetwork.target_Q: targets_mb,\n",
    "                                               DQNetwork.actions_: actions_mb})\n",
    "\n",
    "                # Write TF Summaries\n",
    "                summary = sess.run(write_op, feed_dict={DQNetwork.inputs_: states_mb,\n",
    "                                                   DQNetwork.target_Q: targets_mb,\n",
    "                                                   DQNetwork.actions_: actions_mb})\n",
    "                writer.add_summary(summary, episode)\n",
    "                writer.flush()\n",
    "                \n",
    "            # the episode ends so no next state\n",
    "            next_state = np.zeros((3,84,84), dtype=np.int)\n",
    "            next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)\n",
    "\n",
    "            # Set step = max_steps to end the episode\n",
    "            step = max_steps\n",
    "\n",
    "            # Get the total reward of the episode\n",
    "            total_reward = np.sum(episode_rewards)\n",
    "\n",
    "            print('Episode: {}'.format(episode),\n",
    "                      'Total reward: {}'.format(total_reward),\n",
    "                      'Training loss: {:.4f}'.format(loss),\n",
    "                      'Explore P: {:.4f}'.format(epsilon_))\n",
    "\n",
    "            memory.add((state, action, reward, next_state, done))\n",
    "            \n",
    "            # Save model every 5 episodes\n",
    "            if episode % 25 == 0:\n",
    "                save_path = saver.save(sess, \"./models/model_level_\" + str(level) + \"_episode_\" + str(episode) + \".ckpt\")\n",
    "                print(\"Model Saved\")\n",
    "                \n",
    "game.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./models/model_level_-4_episode_375.ckpt\n",
      "Episode 0\n",
      "Score:  -0.3000000000000002\n",
      "Episode 1\n"
     ]
    },
    {
     "ename": "ViZDoomUnexpectedExitException",
     "evalue": "Controlled ViZDoom instance exited unexpectedly.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mViZDoomUnexpectedExitException\u001b[0m            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-89-fb4e620604ce>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     29\u001b[0m             \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpossible_actions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mchoice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 31\u001b[1;33m             \u001b[0mgame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     32\u001b[0m             \u001b[0mscore\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_total_reward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mViZDoomUnexpectedExitException\u001b[0m: Controlled ViZDoom instance exited unexpectedly."
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    \n",
    "    game, possible_actions, action_names = create_environment(level = level)\n",
    "    \n",
    "    totalScore = 0\n",
    "    \n",
    "    # Load the model\n",
    "    saver.restore(sess, \"./models/model_level_\" + str(level) + \"_episode_375.ckpt\")\n",
    "    game.init()\n",
    "    for i in range(50):\n",
    "        \n",
    "        print('Episode {}'.format(i))\n",
    "        \n",
    "        game.new_episode()\n",
    "        \n",
    "        state = game.get_state().screen_buffer\n",
    "        state, stacked_frames = stack_frames(stacked_frames, state, True)\n",
    "            \n",
    "        while not game.is_episode_finished():\n",
    "            # Take the biggest Q value (= the best action)\n",
    "            Qs = sess.run(DQNetwork.output, feed_dict = {DQNetwork.inputs_: state.reshape((1, *state.shape))})\n",
    "            \n",
    "            next_state = game.get_state().screen_buffer\n",
    "            next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)\n",
    "            state = next_state\n",
    "            \n",
    "            # Take the biggest Q value (= the best action)\n",
    "            choice = np.argmax(Qs)\n",
    "            action = possible_actions[int(choice)]\n",
    "            \n",
    "            game.make_action(action)\n",
    "            score = game.get_total_reward()\n",
    "                \n",
    "        score = game.get_total_reward()\n",
    "        print(\"Score: \", score)\n",
    "    game.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Train Back-up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Saver will help us to save our model\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "if training == True:\n",
    "    with tf.Session() as sess:\n",
    "        # Initialize the variables\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "        # Initialize the decay rate (that will use to reduce epsilon) \n",
    "        decay_step = 0\n",
    "\n",
    "        # Init the game\n",
    "        game.init()\n",
    "\n",
    "        for episode in range(total_episodes):\n",
    "            # Set step to 0\n",
    "            step = 0\n",
    "            \n",
    "            # Initialize the rewards of the episode\n",
    "            episode_rewards = []\n",
    "            \n",
    "            # Make a new episode and observe the first state\n",
    "            game.new_episode()\n",
    "            state = game.get_state().screen_buffer\n",
    "            \n",
    "            # Remember that stack frame function also call our preprocess function.\n",
    "            state, stacked_frames = stack_frames(stacked_frames, state, True)\n",
    "\n",
    "            while step < max_steps:\n",
    "                step += 1\n",
    "                \n",
    "                # Increase decay_step\n",
    "                decay_step +=1\n",
    "                \n",
    "                # Predict the action to take and take it\n",
    "                action, epsilon_ = choose_action(explore_start, explore_stop, decay_rate, decay_step, state, possible_actions)\n",
    "\n",
    "                # Do the action\n",
    "                reward = game.make_action(action)\n",
    "\n",
    "                # Look if the episode is finished\n",
    "                done = game.is_episode_finished()\n",
    "                \n",
    "                # Add the reward to total reward\n",
    "                episode_rewards.append(reward)\n",
    "\n",
    "                # If the game is finished\n",
    "                if done:\n",
    "                    # the episode ends so no next state\n",
    "                    next_state = np.zeros((3,84,84), dtype=np.int)\n",
    "                    next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)\n",
    "                    \n",
    "                    # Set step = max_steps to end the episode\n",
    "                    step = max_steps\n",
    "\n",
    "                    # Get the total reward of the episode\n",
    "                    total_reward = np.sum(episode_rewards)\n",
    "\n",
    "                    print('Episode: {}'.format(episode),\n",
    "                              'Total reward: {}'.format(total_reward),\n",
    "                              'Training loss: {:.4f}'.format(loss),\n",
    "                              'Explore P: {:.4f}'.format(epsilon_))\n",
    "\n",
    "                    memory.add((state, action, reward, next_state, done))\n",
    "\n",
    "                else:\n",
    "                    # Get the next state\n",
    "                    next_state = game.get_state().screen_buffer\n",
    "                    \n",
    "                    # Stack the frame of the next_state\n",
    "                    next_state, stacked_frames = stack_frames(stacked_frames, next_state, False)\n",
    "                    \n",
    "\n",
    "                    # Add experience to memory\n",
    "                    memory.add((state, action, reward, next_state, done))\n",
    "                    \n",
    "                    # st+1 is now our current state\n",
    "                    state = next_state\n",
    "\n",
    "\n",
    "                ### LEARNING PART            \n",
    "                # Obtain random mini-batch from memory\n",
    "                batch = memory.sample(batch_size)\n",
    "                states_mb = np.array([each[0] for each in batch], ndmin=3)\n",
    "                actions_mb = np.array([each[1] for each in batch])\n",
    "                rewards_mb = np.array([each[2] for each in batch]) \n",
    "                next_states_mb = np.array([each[3] for each in batch], ndmin=3)\n",
    "                dones_mb = np.array([each[4] for each in batch])\n",
    "\n",
    "                target_Qs_batch = []\n",
    "\n",
    "                 # Get Q values for next_state \n",
    "                Qs_next_state = sess.run(DQNetwork.output, feed_dict = {DQNetwork.inputs_: next_states_mb})\n",
    "                \n",
    "                # Set Q_target = r if the episode ends at s+1, otherwise set Q_target = r + gamma*maxQ(s', a')\n",
    "                for i in range(0, len(batch)):\n",
    "                    terminal = dones_mb[i]\n",
    "\n",
    "                    # If we are in a terminal state, only equals reward\n",
    "                    if terminal:\n",
    "                        target_Qs_batch.append(rewards_mb[i])\n",
    "                        \n",
    "                    else:\n",
    "                        target = rewards_mb[i] + gamma * np.max(Qs_next_state[i])\n",
    "                        target_Qs_batch.append(target)\n",
    "                        \n",
    "\n",
    "                targets_mb = np.array([each for each in target_Qs_batch])\n",
    "\n",
    "                loss, _ = sess.run([DQNetwork.loss, DQNetwork.optimizer],\n",
    "                                    feed_dict={DQNetwork.inputs_: states_mb,\n",
    "                                               DQNetwork.target_Q: targets_mb,\n",
    "                                               DQNetwork.actions_: actions_mb})\n",
    "\n",
    "                # Write TF Summaries\n",
    "                summary = sess.run(write_op, feed_dict={DQNetwork.inputs_: states_mb,\n",
    "                                                   DQNetwork.target_Q: targets_mb,\n",
    "                                                   DQNetwork.actions_: actions_mb})\n",
    "                writer.add_summary(summary, episode)\n",
    "                writer.flush()\n",
    "\n",
    "            # Save model every 5 episodes\n",
    "            if episode % 25 == 0:\n",
    "                save_path = saver.save(sess, \"./models/model_level_\" + str(level) + \"_episode_\" + str(episode) + \".ckpt\")\n",
    "                print(\"Model Saved\")\n",
    "                \n",
    "game.close()"
   ]
  }
 ],
 "metadata": {
  "gist": {
   "data": {
    "description": "ai learns to play doom",
    "public": true
   },
   "id": ""
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}