Deadly_Corr.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8cee3a09",
   "metadata": {},
   "source": [
    "# setup Vizdoom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ce3ed8e6",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting vizdoom\n",
      "  Downloading vizdoom-1.1.13-cp38-cp38-win_amd64.whl (15.4 MB)\n",
      "Requirement already satisfied: numpy in c:\\users\\andre\\anaconda3\\lib\\site-packages (from vizdoom) (1.20.1)\n",
      "Installing collected packages: vizdoom\n",
      "Successfully installed vizdoom-1.1.13\n"
     ]
    }
   ],
   "source": [
    "!pip install vizdoom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "945c6707",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import vizdoom for game env\n",
    "from vizdoom import * \n",
    "# Import random for action sampling\n",
    "import random\n",
    "# Import time for sleeping\n",
    "import time \n",
    "# Import numpy for identity matrix(action have only 3 value)\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acebef6b",
   "metadata": {},
   "source": [
    "# Setup the gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "71d6c145",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gym in c:\\users\\andre\\anaconda3\\lib\\site-packages (0.24.0)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from gym) (1.6.0)\n",
      "Requirement already satisfied: gym-notices>=0.0.4 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from gym) (0.0.6)\n",
      "Requirement already satisfied: numpy>=1.18.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from gym) (1.20.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.8.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from gym) (4.11.3)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from importlib-metadata>=4.8.0->gym) (3.4.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "01114d3a",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\andre\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting opencv-python\n",
      "  Downloading opencv_python-4.6.0.66-cp36-abi3-win_amd64.whl (35.6 MB)\n",
      "Requirement already satisfied: numpy>=1.17.3 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from opencv-python) (1.20.1)\n",
      "Installing collected packages: opencv-python\n",
      "Successfully installed opencv-python-4.6.0.66\n"
     ]
    }
   ],
   "source": [
    "!pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "62bd9637",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import env base\n",
    "from gym import Env\n",
    "#import gym space\n",
    "from gym.spaces import Discrete,Box\n",
    "#import opencv\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a9fbf4dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Vizdoom OpenAI Gym Environment\n",
    "class VizDoomGym(Env): \n",
    "    # Function that is called when we start the env\n",
    "    def __init__(self, render=False,config='github/VizDoom/scenarios/deadly_corridor_s1.cfg'): \n",
    "        # Inherit from Env\n",
    "        super().__init__()\n",
    "        # Setup the game \n",
    "        self.game = DoomGame()\n",
    "        self.game.load_config(config)\n",
    "        \n",
    "        # Render frame logic\n",
    "        if render == False: \n",
    "            self.game.set_window_visible(False)\n",
    "        else:\n",
    "            self.game.set_window_visible(True)\n",
    "        \n",
    "        # Start the game \n",
    "        self.game.init()\n",
    "        \n",
    "        # Create the action space and observation space\n",
    "        self.observation_space = Box(low=0, high=255, shape=(100,160,1), dtype=np.uint8) \n",
    "        self.action_space = Discrete(7)\n",
    "        \n",
    "        # Game variables: HEALTH DAMAGE_TAKEN HITCOUNT SELECTED_WEAPON_AMMO \n",
    "        #for reward shaping\n",
    "        self.damage_taken=0\n",
    "        self.hitcount=0\n",
    "        self.ammo=52\n",
    "        \n",
    "    # This is how we take a step in the environment\n",
    "    def step(self, action):\n",
    "        # Specify action and take step \n",
    "        actions = np.identity(7)\n",
    "        movement_reward = self.game.make_action(actions[action], 4) \n",
    "        reward=0\n",
    "        \n",
    "        # Get all the other stuff we need to retun \n",
    "        if self.game.get_state(): \n",
    "            state = self.game.get_state().screen_buffer\n",
    "            state = self.grayscale(state)\n",
    "            #reward shaping \n",
    "            game_variables=self.game.get_state().game_variables\n",
    "            health,damage_taken,hitcount,ammo=game_variables\n",
    "            #delta var\n",
    "            damage_taken_delta= -damage_taken+self.damage_taken\n",
    "            self.damage_taken=damage_taken\n",
    "            hitcount_delta=hitcount-self.hitcount\n",
    "            self.hitcount=hitcount\n",
    "            ammo_delta=ammo-self.ammo\n",
    "            self.ammo=ammo \n",
    "            \n",
    "            reward=movement_reward+damage_taken_delta*20 + hitcount_delta*350 + ammo_delta*5            \n",
    "            info = ammo\n",
    "        else: \n",
    "            state = np.zeros(self.observation_space.shape)\n",
    "            info = 0 \n",
    "        \n",
    "        info = {\"info\":info}\n",
    "        done = self.game.is_episode_finished()\n",
    "        \n",
    "        return state, reward, done, info \n",
    "    \n",
    "    # Define how to render the game or environment \n",
    "    def render(): \n",
    "        pass\n",
    "    \n",
    "    # What happens when we start a new game \n",
    "    def reset(self): \n",
    "        self.game.new_episode()\n",
    "        state = self.game.get_state().screen_buffer\n",
    "        return self.grayscale(state)\n",
    "    \n",
    "    # Grayscale the game frame and resize it \n",
    "    def grayscale(self, observation):\n",
    "        gray = cv2.cvtColor(np.moveaxis(observation, 0, -1), cv2.COLOR_BGR2GRAY)\n",
    "        resize = cv2.resize(gray, (160,100), interpolation=cv2.INTER_CUBIC)\n",
    "        state = np.reshape(resize, (100,160,1))\n",
    "        return state\n",
    "    \n",
    "    # Call to close down the game\n",
    "    def close(self): \n",
    "        self.game.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "0ec70b5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "env=VizDoomGym(config='github/VizDoom/scenarios/deadly_corridor.cfg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "c1d55fde",
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0fdc0729",
   "metadata": {},
   "outputs": [],
   "source": [
    "#check for a valid env\n",
    "from stable_baselines3.common import env_checker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5fd44f3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "env_checker.check_env(env)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a9968b2",
   "metadata": {},
   "source": [
    "# Setup callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9fe306a6",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cu113\n",
      "Requirement already satisfied: torch in c:\\users\\andre\\anaconda3\\lib\\site-packages (1.12.1)\n",
      "Collecting torchvision\n",
      "  Downloading https://download.pytorch.org/whl/cu113/torchvision-0.13.1%2Bcu113-cp38-cp38-win_amd64.whl (4.7 MB)\n",
      "Collecting torchaudio\n",
      "  Downloading https://download.pytorch.org/whl/cu113/torchaudio-0.12.1%2Bcu113-cp38-cp38-win_amd64.whl (1.2 MB)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\andre\\anaconda3\\lib\\site-packages (from torch) (3.7.4.3)\n",
      "Requirement already satisfied: requests in c:\\users\\andre\\anaconda3\\lib\\site-packages (from torchvision) (2.25.1)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from torchvision) (8.2.0)\n",
      "Requirement already satisfied: numpy in c:\\users\\andre\\anaconda3\\lib\\site-packages (from torchvision) (1.20.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests->torchvision) (2020.12.5)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests->torchvision) (4.0.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests->torchvision) (1.26.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests->torchvision) (2.10)\n",
      "Installing collected packages: torchvision, torchaudio\n",
      "Successfully installed torchaudio-0.12.1+cu113 torchvision-0.13.1+cu113\n"
     ]
    }
   ],
   "source": [
    "!pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "70dd44af",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: stable-baselines3[extra] in c:\\users\\andre\\anaconda3\\lib\\site-packages (1.6.0)\n",
      "Requirement already satisfied: torch>=1.11 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (1.12.1)\n",
      "Requirement already satisfied: numpy in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (1.20.1)\n",
      "Requirement already satisfied: pandas in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (1.2.4)\n",
      "Requirement already satisfied: gym==0.21 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (0.21.0)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (3.3.4)\n",
      "Requirement already satisfied: cloudpickle in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (1.6.0)\n",
      "Requirement already satisfied: tensorboard>=2.2.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (2.8.0)\n",
      "Requirement already satisfied: psutil in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (5.8.0)\n",
      "Requirement already satisfied: autorom[accept-rom-license]~=0.4.2 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (0.4.2)\n",
      "Requirement already satisfied: protobuf~=3.19.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (3.19.4)\n",
      "Requirement already satisfied: opencv-python in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (4.6.0.66)\n",
      "Requirement already satisfied: ale-py==0.7.4 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (0.7.4)\n",
      "Requirement already satisfied: pillow in c:\\users\\andre\\anaconda3\\lib\\site-packages (from stable-baselines3[extra]) (8.2.0)\n",
      "Requirement already satisfied: importlib-resources in c:\\users\\andre\\anaconda3\\lib\\site-packages (from ale-py==0.7.4->stable-baselines3[extra]) (5.7.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.10.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from ale-py==0.7.4->stable-baselines3[extra]) (4.11.3)\n",
      "Requirement already satisfied: tqdm in c:\\users\\andre\\anaconda3\\lib\\site-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (4.59.0)\n",
      "Requirement already satisfied: click in c:\\users\\andre\\anaconda3\\lib\\site-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (7.1.2)\n",
      "Requirement already satisfied: requests in c:\\users\\andre\\anaconda3\\lib\\site-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.25.1)\n",
      "Requirement already satisfied: AutoROM.accept-rom-license in c:\\users\\andre\\anaconda3\\lib\\site-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (0.4.2)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from importlib-metadata>=4.10.0->ale-py==0.7.4->stable-baselines3[extra]) (3.4.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.6.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (52.0.0.post20210125)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.8.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.6)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.3.6)\n",
      "Requirement already satisfied: absl-py>=0.4 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.0.0)\n",
      "Requirement already satisfied: wheel>=0.26 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.36.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (2.6.6)\n",
      "Requirement already satisfied: grpcio>=1.24.3 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.44.0)\n",
      "Requirement already satisfied: six in c:\\users\\andre\\anaconda3\\lib\\site-packages (from absl-py>=0.4->tensorboard>=2.2.0->stable-baselines3[extra]) (1.15.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.2.8)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (5.0.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.8)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (1.3.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (1.26.4)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (4.0.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2020.12.5)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.10)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (3.2.0)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\andre\\anaconda3\\lib\\site-packages (from torch>=1.11->stable-baselines3[extra]) (3.7.4.3)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (1.3.1)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (2.8.1)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (0.10.0)\n",
      "Requirement already satisfied: pytz>=2017.3 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from pandas->stable-baselines3[extra]) (2021.1)\n"
     ]
    }
   ],
   "source": [
    "#so we can use ppo \n",
    "!pip install stable-baselines3[extra]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "61d81ddb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import os for file nav\n",
    "import os \n",
    "# Import callback class from sb3\n",
    "from stable_baselines3.common.callbacks import BaseCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "11e97b7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainAndLoggingCallback(BaseCallback):\n",
    "\n",
    "    def __init__(self, check_freq, save_path, verbose=1):\n",
    "        super(TrainAndLoggingCallback, self).__init__(verbose)\n",
    "        self.check_freq = check_freq\n",
    "        self.save_path = save_path\n",
    "\n",
    "    def _init_callback(self):\n",
    "        if self.save_path is not None:\n",
    "            os.makedirs(self.save_path, exist_ok=True)\n",
    "\n",
    "    def _on_step(self):\n",
    "        if self.n_calls % self.check_freq == 0:\n",
    "            model_path = os.path.join(self.save_path, 'best_model_{}'.format(self.n_calls))\n",
    "            self.model.save(model_path)\n",
    "\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e8105e51",
   "metadata": {},
   "outputs": [],
   "source": [
    "CHECKPOINT_DIR = './train/train_deadly'\n",
    "LOG_DIR = './logs/log_deadly'\n",
    "CHECKPOINT_DIR_ONLINE = './train/train_deadly_online'\n",
    "LOG_DIR_ONLINE = './logs/log_deadly_online'\n",
    "CHECKPOINT_DIR2 = './train/train_deadly2'\n",
    "LOG_DIR2 = './logs/log_deadly2'\n",
    "CHECKPOINT_DIR3 = './train/train_deadly3'\n",
    "LOG_DIR3 = './logs/log_deadly3'\n",
    "CHECKPOINT_DIR4 = './train/train_deadly4'\n",
    "LOG_DIR4 = './logs/log_deadly4'\n",
    "CHECKPOINT_DIR5 = './train/train_deadly5'\n",
    "LOG_DIR5 = './logs/log_deadly5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5be7aee6",
   "metadata": {},
   "outputs": [],
   "source": [
    "callback= TrainAndLoggingCallback(check_freq=10000,save_path=CHECKPOINT_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ead4af8d",
   "metadata": {},
   "source": [
    "# Train with PPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d6117060",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import ppo\n",
    "from stable_baselines3 import PPO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "16fb8f3c",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow in c:\\users\\andre\\anaconda3\\lib\\site-packages (2.8.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: setuptools in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (52.0.0.post20210125)\n",
      "Requirement already satisfied: flatbuffers>=1.12 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (2.0)\n",
      "Requirement already satisfied: keras<2.9,>=2.8.0rc0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (2.8.0)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (0.25.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: six>=1.12.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (1.15.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (1.12.1)\n",
      "Requirement already satisfied: numpy>=1.20 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (1.20.1)\n",
      "Requirement already satisfied: gast>=0.2.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (0.5.3)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (1.1.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (1.1.2)\n",
      "Requirement already satisfied: absl-py>=0.4.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (1.0.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (3.7.4.3)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (3.19.4)\n",
      "Requirement already satisfied: libclang>=9.0.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (13.0.0)\n",
      "Requirement already satisfied: tf-estimator-nightly==2.8.0.dev2021122109 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (2.8.0.dev2021122109)\n",
      "Requirement already satisfied: h5py>=2.9.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (2.10.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (1.44.0)\n",
      "Requirement already satisfied: tensorboard<2.9,>=2.8 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (2.8.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from astunparse>=1.6.0->tensorflow) (0.36.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (3.3.6)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (1.8.1)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (2.6.6)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (0.6.1)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (2.25.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (1.0.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from tensorboard<2.9,>=2.8->tensorflow) (0.4.6)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow) (4.8)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow) (0.2.8)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow) (5.0.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow) (4.11.3)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow) (3.4.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow) (0.4.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (1.26.4)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (4.0.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (2020.12.5)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow) (2.10)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\andre\\anaconda3\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow) (3.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1d4f2e8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#non render env\n",
    "env= VizDoomGym(config='github/VizDoom/scenarios/deadly_corridor_s1.cfg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "06186280",
   "metadata": {},
   "outputs": [],
   "source": [
    "callback= TrainAndLoggingCallback(check_freq=10000,save_path=CHECKPOINT_DIR_ONLINE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "86714955",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n",
      "Wrapping the env with a `Monitor` wrapper\n",
      "Wrapping the env in a DummyVecEnv.\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    }
   ],
   "source": [
    "#cnnpolicy for img\n",
    "#verbose give some info\n",
    "#n_steps tootal of time frame passed for 1 traing round, so higher = more info passed\n",
    "#model= PPO('CnnPolicy',env,tensorboard_log=LOG_DIR_ONLINE,verbose=1,learning_rate=0.00001,n_steps=8192)\n",
    "model= PPO('CnnPolicy',env,tensorboard_log=LOG_DIR_ONLINE,verbose=1,learning_rate=0.00001,n_steps=8192, clip_range=.1, gamma=.95, gae_lambda=.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "9be907fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env with a `Monitor` wrapper\n",
      "Wrapping the env in a DummyVecEnv.\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    }
   ],
   "source": [
    "model= PPO.load('./train/train_deadly_online/best_model_590kpp14')\n",
    "model.set_env(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "c0743000",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to ./logs/log_deadly_online\\PPO_17\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 134      |\n",
      "|    ep_rew_mean     | 896      |\n",
      "| time/              |          |\n",
      "|    fps             | 30       |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 268      |\n",
      "|    total_timesteps | 8192     |\n",
      "---------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 128         |\n",
      "|    ep_rew_mean          | 918         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 19          |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 840         |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007732519 |\n",
      "|    clip_fraction        | 0.259       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -1.11       |\n",
      "|    explained_variance   | 0.436       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.43e+04    |\n",
      "|    n_updates            | 730         |\n",
      "|    policy_gradient_loss | 0.00407     |\n",
      "|    value_loss           | 3.44e+04    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 130          |\n",
      "|    ep_rew_mean          | 922          |\n",
      "| time/                   |              |\n",
      "|    fps                  | 16           |\n",
      "|    iterations           | 3            |\n",
      "|    time_elapsed         | 1535         |\n",
      "|    total_timesteps      | 24576        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0052195922 |\n",
      "|    clip_fraction        | 0.232        |\n",
      "|    clip_range           | 0.1          |\n",
      "|    entropy_loss         | -1.08        |\n",
      "|    explained_variance   | 0.447        |\n",
      "|    learning_rate        | 1e-05        |\n",
      "|    loss                 | 1.44e+04     |\n",
      "|    n_updates            | 740          |\n",
      "|    policy_gradient_loss | 0.00388      |\n",
      "|    value_loss           | 3.54e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 124         |\n",
      "|    ep_rew_mean          | 923         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 14          |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 2325        |\n",
      "|    total_timesteps      | 32768       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.004918718 |\n",
      "|    clip_fraction        | 0.239       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -1.1        |\n",
      "|    explained_variance   | 0.414       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.46e+04    |\n",
      "|    n_updates            | 750         |\n",
      "|    policy_gradient_loss | 0.00423     |\n",
      "|    value_loss           | 3.35e+04    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 135          |\n",
      "|    ep_rew_mean          | 1.01e+03     |\n",
      "| time/                   |              |\n",
      "|    fps                  | 13           |\n",
      "|    iterations           | 5            |\n",
      "|    time_elapsed         | 3126         |\n",
      "|    total_timesteps      | 40960        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0071443934 |\n",
      "|    clip_fraction        | 0.277        |\n",
      "|    clip_range           | 0.1          |\n",
      "|    entropy_loss         | -1.06        |\n",
      "|    explained_variance   | 0.447        |\n",
      "|    learning_rate        | 1e-05        |\n",
      "|    loss                 | 1.77e+04     |\n",
      "|    n_updates            | 760          |\n",
      "|    policy_gradient_loss | 0.00475      |\n",
      "|    value_loss           | 3.37e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 138         |\n",
      "|    ep_rew_mean          | 1.01e+03    |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 6           |\n",
      "|    time_elapsed         | 3861        |\n",
      "|    total_timesteps      | 49152       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.006563802 |\n",
      "|    clip_fraction        | 0.276       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -1.07       |\n",
      "|    explained_variance   | 0.435       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 8.61e+03    |\n",
      "|    n_updates            | 770         |\n",
      "|    policy_gradient_loss | 0.00968     |\n",
      "|    value_loss           | 2.72e+04    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 141          |\n",
      "|    ep_rew_mean          | 1.02e+03     |\n",
      "| time/                   |              |\n",
      "|    fps                  | 12           |\n",
      "|    iterations           | 7            |\n",
      "|    time_elapsed         | 4501         |\n",
      "|    total_timesteps      | 57344        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0071461275 |\n",
      "|    clip_fraction        | 0.259        |\n",
      "|    clip_range           | 0.1          |\n",
      "|    entropy_loss         | -1.05        |\n",
      "|    explained_variance   | 0.416        |\n",
      "|    learning_rate        | 1e-05        |\n",
      "|    loss                 | 1.31e+04     |\n",
      "|    n_updates            | 780          |\n",
      "|    policy_gradient_loss | 0.00362      |\n",
      "|    value_loss           | 3.19e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 141         |\n",
      "|    ep_rew_mean          | 1.07e+03    |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 8           |\n",
      "|    time_elapsed         | 5143        |\n",
      "|    total_timesteps      | 65536       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008932078 |\n",
      "|    clip_fraction        | 0.272       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -1.01       |\n",
      "|    explained_variance   | 0.44        |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.29e+04    |\n",
      "|    n_updates            | 790         |\n",
      "|    policy_gradient_loss | 0.0045      |\n",
      "|    value_loss           | 2.69e+04    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 168         |\n",
      "|    ep_rew_mean          | 992         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 9           |\n",
      "|    time_elapsed         | 5763        |\n",
      "|    total_timesteps      | 73728       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.005846967 |\n",
      "|    clip_fraction        | 0.233       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -1          |\n",
      "|    explained_variance   | 0.439       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.53e+04    |\n",
      "|    n_updates            | 800         |\n",
      "|    policy_gradient_loss | 0.00527     |\n",
      "|    value_loss           | 2.86e+04    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 164          |\n",
      "|    ep_rew_mean          | 930          |\n",
      "| time/                   |              |\n",
      "|    fps                  | 12           |\n",
      "|    iterations           | 10           |\n",
      "|    time_elapsed         | 6390         |\n",
      "|    total_timesteps      | 81920        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0075277137 |\n",
      "|    clip_fraction        | 0.259        |\n",
      "|    clip_range           | 0.1          |\n",
      "|    entropy_loss         | -1.07        |\n",
      "|    explained_variance   | 0.425        |\n",
      "|    learning_rate        | 1e-05        |\n",
      "|    loss                 | 2.47e+04     |\n",
      "|    n_updates            | 810          |\n",
      "|    policy_gradient_loss | 0.00724      |\n",
      "|    value_loss           | 2.22e+04     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 128          |\n",
      "|    ep_rew_mean          | 977          |\n",
      "| time/                   |              |\n",
      "|    fps                  | 12           |\n",
      "|    iterations           | 11           |\n",
      "|    time_elapsed         | 6962         |\n",
      "|    total_timesteps      | 90112        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0059568407 |\n",
      "|    clip_fraction        | 0.236        |\n",
      "|    clip_range           | 0.1          |\n",
      "|    entropy_loss         | -1.02        |\n",
      "|    explained_variance   | 0.466        |\n",
      "|    learning_rate        | 1e-05        |\n",
      "|    loss                 | 1.57e+04     |\n",
      "|    n_updates            | 820          |\n",
      "|    policy_gradient_loss | 0.003        |\n",
      "|    value_loss           | 3.09e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 118         |\n",
      "|    ep_rew_mean          | 982         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 12          |\n",
      "|    time_elapsed         | 7601        |\n",
      "|    total_timesteps      | 98304       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.005527543 |\n",
      "|    clip_fraction        | 0.237       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -1          |\n",
      "|    explained_variance   | 0.43        |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.09e+04    |\n",
      "|    n_updates            | 830         |\n",
      "|    policy_gradient_loss | 0.00637     |\n",
      "|    value_loss           | 3.1e+04     |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 114         |\n",
      "|    ep_rew_mean          | 954         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 12          |\n",
      "|    iterations           | 13          |\n",
      "|    time_elapsed         | 8290        |\n",
      "|    total_timesteps      | 106496      |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007245814 |\n",
      "|    clip_fraction        | 0.267       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.996      |\n",
      "|    explained_variance   | 0.445       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.2e+04     |\n",
      "|    n_updates            | 840         |\n",
      "|    policy_gradient_loss | 0.00587     |\n",
      "|    value_loss           | 3.35e+04    |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 105        |\n",
      "|    ep_rew_mean          | 991        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 12         |\n",
      "|    iterations           | 14         |\n",
      "|    time_elapsed         | 8945       |\n",
      "|    total_timesteps      | 114688     |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.00639184 |\n",
      "|    clip_fraction        | 0.255      |\n",
      "|    clip_range           | 0.1        |\n",
      "|    entropy_loss         | -0.973     |\n",
      "|    explained_variance   | 0.516      |\n",
      "|    learning_rate        | 1e-05      |\n",
      "|    loss                 | 2.03e+04   |\n",
      "|    n_updates            | 850        |\n",
      "|    policy_gradient_loss | 0.00558    |\n",
      "|    value_loss           | 3.56e+04   |\n",
      "----------------------------------------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-40-86e7747bdf6d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m600000\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\stable_baselines3\\ppo\\ppo.py\u001b[0m in \u001b[0;36mlearn\u001b[1;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[0;32m    308\u001b[0m     ) -> \"PPO\":\n\u001b[0;32m    309\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 310\u001b[1;33m         return super().learn(\n\u001b[0m\u001b[0;32m    311\u001b[0m             \u001b[0mtotal_timesteps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    312\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\stable_baselines3\\common\\on_policy_algorithm.py\u001b[0m in \u001b[0;36mlearn\u001b[1;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[0;32m    265\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    266\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 267\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    268\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    269\u001b[0m         \u001b[0mcallback\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_training_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\stable_baselines3\\ppo\\ppo.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    268\u001b[0m                 \u001b[1;31m# Optimization step\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    269\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 270\u001b[1;33m                 \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    271\u001b[0m                 \u001b[1;31m# Clip grad norm\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    272\u001b[0m                 \u001b[0mth\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_grad_norm\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    394\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    395\u001b[0m                 inputs=inputs)\n\u001b[1;32m--> 396\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    397\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    398\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    171\u001b[0m     \u001b[1;31m# some Python versions print out the first line of a multi-line function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    172\u001b[0m     \u001b[1;31m# calls in the traceback and some print out the last line\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 173\u001b[1;33m     Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[0;32m    174\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    175\u001b[0m         allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.learn(total_timesteps=600000,callback=callback)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "750e843b",
   "metadata": {},
   "source": [
    "# Difficulty 1 Test effetuati da me\n",
    "ricorda che se ep_lengh troppo bassa l'agente potrebbe star correndo dritto che buono per dif 1 ma per il resto è overfitted\n",
    "### Test con 40000 timesteps\n",
    "####  ppo8 n_steps 4096 l_rate .0001 best model 110k\n",
    "len sembra bassa dai dati.Infatti dai test è palese che l'agente vada dritto ignorando i nemici\n",
    "#### ppo9 n_step 8192 l_rate .0001 best model 150k\n",
    "prova gia a sparare ai nemici ma non va molto avanti\n",
    "#### ppo10 n-step 8192 l_rate .00001 best model 190k\n",
    "cerca di uccidere i due nemici ma poi non va avanti, provare a riprendere ppo10 e portartalo a piu timesteps\n",
    "### Partendo da PP010 best_model con 80000 timesteps\n",
    "#### ppo11 stesse stat best model 50k o 80k\n",
    "visto che intorno a 55k timesteps il tempo diventa troppo basso bisogna testare 2 modelli: il migliore per reward e quello che i dati danno con buona reward e un buon tempo.Dai test in qualche modo sono peggiori di ppo10\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8049843b",
   "metadata": {},
   "source": [
    "# Test con iperparametri trovati online\n",
    "#### ppo3 addestrato con 400k timesteps nst model 430k va troppo giu il tempo medio quindi ri addestrare\n",
    "#### pp4 a partire da 10k best model di ppo3 miglriore ma dopo circa 200k va anche lui troppo basso\n",
    "#### pp7 a partire da 180k di pp4 va sempre troppo giu il tempo bisogna riprovare da capo\n",
    "#### pp10 aumentata reward hitcount 300\n",
    "#### pp12 aumentata anche dmg taken a 50 secondo me migliore riprovare con ancora di \n",
    "#### ppo13 dmg taken 100, troppo alto l'agente si mette in fondo mappa e gira su se stesso (come non chiedere)\n",
    "#### ppo14 20,350,5  (350k)avanza e uccide i nemi a sinistra finalmente un buon set di perparametri,(450k) entra seconda zona facile ma poi si ferma a volte torna anche indietro,(490) uccide subito un nemico va avanti e uccide un nemico della seconda zona,(550k) ci stiamo avviacinando molto capita che uccide quasi tutti e arriva nella zona finale spesso,(590k) molto aggressivo nel andare avanti e una vittoria durante il testing \n",
    "#### ppo15 riprendo da 590k pp14,previsto solo altri 100k ~ passi di addestramento(da rifare mi sa che uso funzione sbagliata)\n",
    "#### ppo16 come 15 ma con ppo.load.partono con +590k, non sembra che parta da pp14 ma neanche da zero non capisco .... errore stupido ho messo modeltest=ppo.load al posto di model......(cio rende ancora piu strano che sembri non partire da zero)\n",
    "#### ppo17 stavolta model=ppo.load adesso ha continuato il training ,su molti modelli vince almeno 1 volta su 5 \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40d69479",
   "metadata": {},
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1b145a3",
   "metadata": {},
   "source": [
    "# curriculum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a077ca35",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env with a `Monitor` wrapper\n",
      "Wrapping the env in a DummyVecEnv.\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Logging to ./logs/log_deadly_online\\PPO_18\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 81.1     |\n",
      "|    ep_rew_mean     | 805      |\n",
      "| time/              |          |\n",
      "|    fps             | 27       |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 297      |\n",
      "|    total_timesteps | 8192     |\n",
      "---------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 78.7        |\n",
      "|    ep_rew_mean          | 850         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 19          |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 819         |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.024473336 |\n",
      "|    clip_fraction        | 0.388       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.928      |\n",
      "|    explained_variance   | 0.35        |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 3.67e+04    |\n",
      "|    n_updates            | 860         |\n",
      "|    policy_gradient_loss | 0.0166      |\n",
      "|    value_loss           | 6.06e+04    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 98.4        |\n",
      "|    ep_rew_mean          | 810         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 18          |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 1363        |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007265257 |\n",
      "|    clip_fraction        | 0.251       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.92       |\n",
      "|    explained_variance   | 0.335       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 2.62e+04    |\n",
      "|    n_updates            | 870         |\n",
      "|    policy_gradient_loss | 0.00761     |\n",
      "|    value_loss           | 5.98e+04    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 103         |\n",
      "|    ep_rew_mean          | 899         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 17          |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 1831        |\n",
      "|    total_timesteps      | 32768       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008892763 |\n",
      "|    clip_fraction        | 0.305       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.956      |\n",
      "|    explained_variance   | 0.311       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.41e+04    |\n",
      "|    n_updates            | 880         |\n",
      "|    policy_gradient_loss | 0.0059      |\n",
      "|    value_loss           | 4.68e+04    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 95          |\n",
      "|    ep_rew_mean          | 906         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 18          |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 2272        |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007748942 |\n",
      "|    clip_fraction        | 0.291       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.963      |\n",
      "|    explained_variance   | 0.309       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.52e+04    |\n",
      "|    n_updates            | 890         |\n",
      "|    policy_gradient_loss | 0.00753     |\n",
      "|    value_loss           | 4.39e+04    |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x123bd81fe20>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = VizDoomGym(config='github/VizDoom/scenarios/deadly_corridor_s2.cfg')\n",
    "model= PPO.load('./train/train_deadly_online/best_model_700k')\n",
    "callback= TrainAndLoggingCallback(check_freq=10000,save_path=CHECKPOINT_DIR2)\n",
    "model.set_env(env)\n",
    "model.learn(total_timesteps=40000, callback=callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "64c1735e",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env with a `Monitor` wrapper\n",
      "Wrapping the env in a DummyVecEnv.\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Logging to ./logs/log_deadly_online\\PPO_19\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 104      |\n",
      "|    ep_rew_mean     | 965      |\n",
      "| time/              |          |\n",
      "|    fps             | 39       |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 208      |\n",
      "|    total_timesteps | 8192     |\n",
      "---------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 106          |\n",
      "|    ep_rew_mean          | 929          |\n",
      "| time/                   |              |\n",
      "|    fps                  | 24           |\n",
      "|    iterations           | 2            |\n",
      "|    time_elapsed         | 671          |\n",
      "|    total_timesteps      | 16384        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0071155545 |\n",
      "|    clip_fraction        | 0.29         |\n",
      "|    clip_range           | 0.1          |\n",
      "|    entropy_loss         | -0.964       |\n",
      "|    explained_variance   | 0.271        |\n",
      "|    learning_rate        | 1e-05        |\n",
      "|    loss                 | 3.71e+04     |\n",
      "|    n_updates            | 900          |\n",
      "|    policy_gradient_loss | 0.00906      |\n",
      "|    value_loss           | 4.24e+04     |\n",
      "------------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 93.3        |\n",
      "|    ep_rew_mean          | 843         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 21          |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 1135        |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008816425 |\n",
      "|    clip_fraction        | 0.315       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.921      |\n",
      "|    explained_variance   | 0.321       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 2.31e+04    |\n",
      "|    n_updates            | 910         |\n",
      "|    policy_gradient_loss | 0.00728     |\n",
      "|    value_loss           | 4.65e+04    |\n",
      "-----------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 100          |\n",
      "|    ep_rew_mean          | 857          |\n",
      "| time/                   |              |\n",
      "|    fps                  | 20           |\n",
      "|    iterations           | 4            |\n",
      "|    time_elapsed         | 1633         |\n",
      "|    total_timesteps      | 32768        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0053482046 |\n",
      "|    clip_fraction        | 0.25         |\n",
      "|    clip_range           | 0.1          |\n",
      "|    entropy_loss         | -0.918       |\n",
      "|    explained_variance   | 0.35         |\n",
      "|    learning_rate        | 1e-05        |\n",
      "|    loss                 | 1.69e+04     |\n",
      "|    n_updates            | 920          |\n",
      "|    policy_gradient_loss | 0.00601      |\n",
      "|    value_loss           | 4.55e+04     |\n",
      "------------------------------------------\n",
      "------------------------------------------\n",
      "| rollout/                |              |\n",
      "|    ep_len_mean          | 94.4         |\n",
      "|    ep_rew_mean          | 898          |\n",
      "| time/                   |              |\n",
      "|    fps                  | 19           |\n",
      "|    iterations           | 5            |\n",
      "|    time_elapsed         | 2127         |\n",
      "|    total_timesteps      | 40960        |\n",
      "| train/                  |              |\n",
      "|    approx_kl            | 0.0076242136 |\n",
      "|    clip_fraction        | 0.274        |\n",
      "|    clip_range           | 0.1          |\n",
      "|    entropy_loss         | -0.915       |\n",
      "|    explained_variance   | 0.381        |\n",
      "|    learning_rate        | 1e-05        |\n",
      "|    loss                 | 1.88e+04     |\n",
      "|    n_updates            | 930          |\n",
      "|    policy_gradient_loss | 0.00521      |\n",
      "|    value_loss           | 4.34e+04     |\n",
      "------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x123b75e79d0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = VizDoomGym(config='github/VizDoom/scenarios/deadly_corridor_s3.cfg')\n",
    "model= PPO.load('./train/train_deadly2/best_model_40000')\n",
    "callback= TrainAndLoggingCallback(check_freq=10000,save_path=CHECKPOINT_DIR3)\n",
    "model.set_env(env)\n",
    "\n",
    "model.learn(total_timesteps=40000, callback=callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3c743911",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env with a `Monitor` wrapper\n",
      "Wrapping the env in a DummyVecEnv.\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Logging to ./logs/log_deadly_online\\PPO_20\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 104      |\n",
      "|    ep_rew_mean     | 879      |\n",
      "| time/              |          |\n",
      "|    fps             | 26       |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 311      |\n",
      "|    total_timesteps | 8192     |\n",
      "---------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 96.1        |\n",
      "|    ep_rew_mean          | 931         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 18          |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 864         |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.005402882 |\n",
      "|    clip_fraction        | 0.261       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.892      |\n",
      "|    explained_variance   | 0.352       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 4.17e+04    |\n",
      "|    n_updates            | 940         |\n",
      "|    policy_gradient_loss | 0.00669     |\n",
      "|    value_loss           | 4.38e+04    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 101         |\n",
      "|    ep_rew_mean          | 847         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 17          |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 1391        |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008228039 |\n",
      "|    clip_fraction        | 0.252       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.862      |\n",
      "|    explained_variance   | 0.397       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.68e+04    |\n",
      "|    n_updates            | 950         |\n",
      "|    policy_gradient_loss | 0.00658     |\n",
      "|    value_loss           | 4.62e+04    |\n",
      "-----------------------------------------\n",
      "----------------------------------------\n",
      "| rollout/                |            |\n",
      "|    ep_len_mean          | 90.2       |\n",
      "|    ep_rew_mean          | 943        |\n",
      "| time/                   |            |\n",
      "|    fps                  | 17         |\n",
      "|    iterations           | 4          |\n",
      "|    time_elapsed         | 1896       |\n",
      "|    total_timesteps      | 32768      |\n",
      "| train/                  |            |\n",
      "|    approx_kl            | 0.00722342 |\n",
      "|    clip_fraction        | 0.238      |\n",
      "|    clip_range           | 0.1        |\n",
      "|    entropy_loss         | -0.855     |\n",
      "|    explained_variance   | 0.461      |\n",
      "|    learning_rate        | 1e-05      |\n",
      "|    loss                 | 2.15e+04   |\n",
      "|    n_updates            | 960        |\n",
      "|    policy_gradient_loss | 0.00723    |\n",
      "|    value_loss           | 4.54e+04   |\n",
      "----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 96.3        |\n",
      "|    ep_rew_mean          | 900         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 16          |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 2433        |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.006128302 |\n",
      "|    clip_fraction        | 0.239       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.818      |\n",
      "|    explained_variance   | 0.457       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 3.83e+04    |\n",
      "|    n_updates            | 970         |\n",
      "|    policy_gradient_loss | 0.00679     |\n",
      "|    value_loss           | 5.21e+04    |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x186ffbc4e50>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = VizDoomGym(config='github/VizDoom/scenarios/deadly_corridor_s4.cfg')\n",
    "model=PPO.load('./train/train_deadly3/best_model_40000')\n",
    "callback= TrainAndLoggingCallback(check_freq=10000,save_path=CHECKPOINT_DIR4)\n",
    "model.set_env(env)\n",
    "\n",
    "model.learn(total_timesteps=40000, callback=callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "89208c9f",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wrapping the env with a `Monitor` wrapper\n",
      "Wrapping the env in a DummyVecEnv.\n",
      "Wrapping the env in a VecTransposeImage.\n",
      "Logging to ./logs/log_deadly_online\\PPO_21\n",
      "---------------------------------\n",
      "| rollout/           |          |\n",
      "|    ep_len_mean     | 35.8     |\n",
      "|    ep_rew_mean     | 375      |\n",
      "| time/              |          |\n",
      "|    fps             | 31       |\n",
      "|    iterations      | 1        |\n",
      "|    time_elapsed    | 262      |\n",
      "|    total_timesteps | 8192     |\n",
      "---------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 32.9        |\n",
      "|    ep_rew_mean          | 342         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 20          |\n",
      "|    iterations           | 2           |\n",
      "|    time_elapsed         | 789         |\n",
      "|    total_timesteps      | 16384       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.044699937 |\n",
      "|    clip_fraction        | 0.529       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.874      |\n",
      "|    explained_variance   | -0.177      |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 2.24e+04    |\n",
      "|    n_updates            | 980         |\n",
      "|    policy_gradient_loss | 0.0371      |\n",
      "|    value_loss           | 7.95e+04    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 32.7        |\n",
      "|    ep_rew_mean          | 351         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 17          |\n",
      "|    iterations           | 3           |\n",
      "|    time_elapsed         | 1376        |\n",
      "|    total_timesteps      | 24576       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.011891191 |\n",
      "|    clip_fraction        | 0.228       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.757      |\n",
      "|    explained_variance   | 0.264       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 3.71e+04    |\n",
      "|    n_updates            | 990         |\n",
      "|    policy_gradient_loss | 0.00879     |\n",
      "|    value_loss           | 7.39e+04    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 30.7        |\n",
      "|    ep_rew_mean          | 328         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 16          |\n",
      "|    iterations           | 4           |\n",
      "|    time_elapsed         | 1952        |\n",
      "|    total_timesteps      | 32768       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.007896241 |\n",
      "|    clip_fraction        | 0.207       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.733      |\n",
      "|    explained_variance   | 0.326       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 1.88e+04    |\n",
      "|    n_updates            | 1000        |\n",
      "|    policy_gradient_loss | 0.00676     |\n",
      "|    value_loss           | 7.04e+04    |\n",
      "-----------------------------------------\n",
      "-----------------------------------------\n",
      "| rollout/                |             |\n",
      "|    ep_len_mean          | 31.9        |\n",
      "|    ep_rew_mean          | 332         |\n",
      "| time/                   |             |\n",
      "|    fps                  | 16          |\n",
      "|    iterations           | 5           |\n",
      "|    time_elapsed         | 2526        |\n",
      "|    total_timesteps      | 40960       |\n",
      "| train/                  |             |\n",
      "|    approx_kl            | 0.008090166 |\n",
      "|    clip_fraction        | 0.209       |\n",
      "|    clip_range           | 0.1         |\n",
      "|    entropy_loss         | -0.701      |\n",
      "|    explained_variance   | 0.374       |\n",
      "|    learning_rate        | 1e-05       |\n",
      "|    loss                 | 4.33e+04    |\n",
      "|    n_updates            | 1010        |\n",
      "|    policy_gradient_loss | 0.00795     |\n",
      "|    value_loss           | 6.8e+04     |\n",
      "-----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x18686f0fd00>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = VizDoomGym(config='github/VizDoom/scenarios/deadly_corridor.cfg')\n",
    "model=PPO.load('./train/train_deadly4/best_model_40000')\n",
    "callback= TrainAndLoggingCallback(check_freq=10000,save_path=CHECKPOINT_DIR5)\n",
    "model.set_env(env)\n",
    "\n",
    "model.learn(total_timesteps=40000, callback=callback)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b72e52f6",
   "metadata": {},
   "source": [
    "# Test the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "acef6d25",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import eval policy\n",
    "from stable_baselines3.common.evaluation import evaluate_policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "96b4f6c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "modeltest= PPO.load('./train/train_deadly_online/best_model_110000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f3daaddd",
   "metadata": {},
   "outputs": [],
   "source": [
    "modeltest= PPO.load('./train/train_deadly5/best_model_40000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "11338f4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "env=VizDoomGym(render=True,config='github/VizDoom/scenarios/deadly_corridor_s2.cfg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dd79b1fb",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\andre\\anaconda3\\lib\\site-packages\\stable_baselines3\\common\\evaluation.py:65: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "#mean reward for 10 games \n",
    "mean_rew, _ = evaluate_policy(modeltest,env,n_eval_episodes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4b3aeb7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Reward for episode 0 is -93.43565368652344\n",
      "Total Reward for episode 1 is 162.76583862304688\n",
      "Total Reward for episode 2 is 2050.3453979492188\n",
      "Total Reward for episode 3 is 53.37300109863281\n",
      "Total Reward for episode 4 is 1141.3853454589844\n"
     ]
    }
   ],
   "source": [
    "for episode in range(5): \n",
    "    obs = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done: \n",
    "        action, _ = modeltest.predict(obs)\n",
    "        obs, reward, done, info = env.step(action)\n",
    "        time.sleep(0.20)\n",
    "        total_reward += reward\n",
    "    print('Total Reward for episode {} is {}'.format(episode, total_reward))\n",
    "    time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "115c788c",
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ba5f1da",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}