muzero.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "muzero",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "1bvBjwVcXYgr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Lint as: python3\n",
        "\"\"\"Pseudocode description of the MuZero algorithm.\"\"\"\n",
        "# pylint: disable=unused-argument\n",
        "# pylint: disable=missing-docstring\n",
        "# pylint: disable=assignment-from-no-return\n",
        "\n",
        "from __future__ import absolute_import\n",
        "from __future__ import division\n",
        "from __future__ import print_function\n",
        "\n",
        "import collections\n",
        "import math\n",
        "import typing\n",
        "from typing import Dict, List, Optional\n",
        "import enum\n",
        "\n",
        "import numpy\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "import threading\n",
        "\n",
        "##########################\n",
        "####### Helpers ##########\n",
        "\n",
        "MAXIMUM_FLOAT_VALUE = float('inf')\n",
        "\n",
        "KnownBounds = collections.namedtuple('KnownBounds', ['min', 'max'])\n",
        "\n",
        "# noinspection PyArgumentList\n",
        "Winner = enum.Enum(\"Winner\", \"black white draw\")\n",
        "\n",
        "# noinspection PyArgumentList\n",
        "Player = enum.Enum(\"Player\", \"black white\")\n",
        "\n",
        "num_filters = 2\n",
        "num_blocks = 8\n",
        "\n",
        "class MinMaxStats(object):\n",
        "  \"\"\"A class that holds the min-max values of the tree.\"\"\"\n",
        "\n",
        "  def __init__(self, known_bounds: Optional[KnownBounds]):\n",
        "    self.maximum = known_bounds.max if known_bounds else -MAXIMUM_FLOAT_VALUE\n",
        "    self.minimum = known_bounds.min if known_bounds else MAXIMUM_FLOAT_VALUE\n",
        "\n",
        "  def update(self, value: float):\n",
        "    self.maximum = max(self.maximum, value)\n",
        "    self.minimum = min(self.minimum, value)\n",
        "\n",
        "  def normalize(self, value: float) -> float:\n",
        "    if self.maximum > self.minimum:\n",
        "      # We normalize only when we have set the maximum and minimum values.\n",
        "      return (value - self.minimum) / (self.maximum - self.minimum)\n",
        "    return value\n",
        "\n",
        "\n",
        "class MuZeroConfig(object):\n",
        "\n",
        "  def __init__(self,\n",
        "               action_space_size: int,\n",
        "               max_moves: int,\n",
        "               discount: float,\n",
        "               dirichlet_alpha: float,\n",
        "               num_simulations: int,\n",
        "               batch_size: int,\n",
        "               td_steps: int,\n",
        "               num_actors: int,\n",
        "               lr_init: float,\n",
        "               lr_decay_steps: float,\n",
        "               visit_softmax_temperature_fn,\n",
        "               known_bounds: Optional[KnownBounds] = None):\n",
        "    ### Self-Play\n",
        "    self.action_space_size = action_space_size\n",
        "    self.num_actors = num_actors\n",
        "\n",
        "    self.visit_softmax_temperature_fn = visit_softmax_temperature_fn\n",
        "    self.max_moves = max_moves\n",
        "    self.num_simulations = num_simulations\n",
        "    self.discount = discount\n",
        "\n",
        "    # Root prior exploration noise.\n",
        "    self.root_dirichlet_alpha = dirichlet_alpha\n",
        "    self.root_exploration_fraction = 0.25\n",
        "\n",
        "    # UCB formula\n",
        "    self.pb_c_base = 19652\n",
        "    self.pb_c_init = 1.25\n",
        "\n",
        "    # If we already have some information about which values occur in the\n",
        "    # environment, we can use them to initialize the rescaling.\n",
        "    # This is not strictly necessary, but establishes identical behaviour to\n",
        "    # AlphaZero in board games.\n",
        "    self.known_bounds = known_bounds\n",
        "\n",
        "    ### Training\n",
        "    self.training_steps = int(1e6)\n",
        "    self.checkpoint_interval = int(100)\n",
        "    self.window_size = int(1e6)\n",
        "    self.batch_size = batch_size\n",
        "    self.num_unroll_steps = 4\n",
        "    self.td_steps = td_steps\n",
        "\n",
        "    self.weight_decay = 1e-4\n",
        "    self.momentum = 0.9\n",
        "\n",
        "    # Exponential learning rate schedule\n",
        "    self.lr_init = lr_init\n",
        "    self.lr_decay_rate = 0.1\n",
        "    self.lr_decay_steps = lr_decay_steps\n",
        "\n",
        "  def new_game(self):\n",
        "    return Game(self.action_space_size, self.discount)\n",
        "\n",
        "\n",
        "def make_board_game_config(action_space_size: int, max_moves: int,\n",
        "                           dirichlet_alpha: float,\n",
        "                           lr_init: float) -> MuZeroConfig:\n",
        "\n",
        "  def visit_softmax_temperature(num_moves, training_steps):\n",
        "    if num_moves < 30:\n",
        "      return 1.0\n",
        "    else:\n",
        "      return 0.0  # Play according to the max.\n",
        "\n",
        "  return MuZeroConfig(\n",
        "      action_space_size=action_space_size,\n",
        "      max_moves=max_moves,\n",
        "      discount=1.0,\n",
        "      dirichlet_alpha=dirichlet_alpha,\n",
        "      num_simulations=10,\n",
        "      batch_size=64,\n",
        "      td_steps=max_moves,  # Always use Monte Carlo return.\n",
        "      num_actors=1,\n",
        "      lr_init=lr_init,\n",
        "      lr_decay_steps=400e3,\n",
        "      visit_softmax_temperature_fn=visit_softmax_temperature,\n",
        "      known_bounds=KnownBounds(-1, 1))\n",
        "\n",
        "def make_connect4_config() -> MuZeroConfig:\n",
        "  return make_board_game_config(\n",
        "      action_space_size=7, max_moves=20, dirichlet_alpha=0.03, lr_init=0.01)\n",
        "\n",
        "class Action(object):\n",
        "\n",
        "  def __init__(self, index: int):\n",
        "    self.index = index\n",
        "\n",
        "  def __hash__(self):\n",
        "    return self.index\n",
        "\n",
        "  def __eq__(self, other):\n",
        "    return self.index == other\n",
        "\n",
        "  def __gt__(self, other):\n",
        "    return self.index > other\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_vGSYHVMX-T_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "class Node(object):\n",
        "\n",
        "  def __init__(self, prior: float):\n",
        "    self.visit_count = 0\n",
        "    self.to_play = -1\n",
        "    self.prior = prior\n",
        "    self.value_sum = 0\n",
        "    self.children = {}\n",
        "    self.hidden_state = None\n",
        "    self.reward = 0\n",
        "\n",
        "  def expanded(self) -> bool:\n",
        "    return len(self.children) > 0\n",
        "\n",
        "  def value(self) -> float:\n",
        "    if self.visit_count == 0:\n",
        "      return 0\n",
        "    return self.value_sum / self.visit_count\n",
        "\n",
        "\n",
        "class ActionHistory(object):\n",
        "  \"\"\"Simple history container used inside the search.\n",
        "\n",
        "  Only used to keep track of the actions executed.\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, history: List[Action], action_space_size: int):\n",
        "    self.history = list(history)\n",
        "    self.action_space_size = action_space_size\n",
        "\n",
        "  def clone(self):\n",
        "    return ActionHistory(self.history, self.action_space_size)\n",
        "\n",
        "  def add_action(self, action: Action):\n",
        "    self.history.append(action)\n",
        "\n",
        "  def last_action(self) -> Action:\n",
        "    return self.history[-1]\n",
        "\n",
        "  def action_space(self) -> List[Action]:\n",
        "    return [i for i in range(self.action_space_size)]\n",
        "\n",
        "  def to_play(self) -> Player:\n",
        "    if len(self.history) % 2 == 0:\n",
        "      return Player.white\n",
        "    else:\n",
        "      return Player.black\n",
        "\n",
        "class Environment(object):\n",
        "  \"\"\"The environment MuZero is interacting with.\"\"\"\n",
        "\n",
        "  def __init__(self):\n",
        "      self.board = None\n",
        "      self.turn = 0\n",
        "      self.done = False\n",
        "      self.winner = None  # type: Winner\n",
        "      self.resigned = False\n",
        "\n",
        "  def reset(self):\n",
        "      self.board = []\n",
        "      for i in range(6):\n",
        "          self.board.append([])\n",
        "          for j in range(7): # pylint: disable=unused-variable\n",
        "              self.board[i].append(' ')\n",
        "      self.turn = 0\n",
        "      self.done = False\n",
        "      self.winner = None\n",
        "      self.resigned = False\n",
        "      return self\n",
        "\n",
        "  def update(self, board):\n",
        "      self.board = numpy.copy(board)\n",
        "      self.turn = self.turn_n()\n",
        "      self.done = False\n",
        "      self.winner = None\n",
        "      self.resigned = False\n",
        "      return self\n",
        "\n",
        "  def turn_n(self):\n",
        "      turn = 0\n",
        "      for i in range(6):\n",
        "          for j in range(7):\n",
        "              if self.board[i][j] != ' ':\n",
        "                  turn += 1\n",
        "\n",
        "      return turn\n",
        "\n",
        "  def player_turn(self):\n",
        "      if self.turn % 2 == 0:\n",
        "          return Player.white\n",
        "      else:\n",
        "          return Player.black\n",
        "\n",
        "  def step(self, action):\n",
        "      for i in range(6):\n",
        "          if self.board[i][action] == ' ':\n",
        "              self.board[i][action] = ('X' if self.player_turn() == Player.white else 'O')\n",
        "              break\n",
        "\n",
        "      self.turn += 1\n",
        "\n",
        "      self.check_for_fours()\n",
        "\n",
        "      if self.turn >= 42:\n",
        "          self.done = True\n",
        "          if self.winner is None:\n",
        "              self.winner = Winner.draw\n",
        "\n",
        "      r = 0\n",
        "      if self.done:\n",
        "        if self.turn % 2 == 0:\n",
        "          if Winner.white:\n",
        "            r = 1\n",
        "          elif Winner.black:\n",
        "            r = -1\n",
        "        else:\n",
        "          if Winner.black:\n",
        "            r = 1\n",
        "          elif Winner.white:\n",
        "            r = -1\n",
        "\n",
        "      return r\n",
        "\n",
        "  def legal_moves(self):\n",
        "      legal = [0, 0, 0, 0, 0, 0, 0]\n",
        "      for j in range(7):\n",
        "          for i in range(6):\n",
        "              if self.board[i][j] == ' ':\n",
        "                  legal[j] = 1\n",
        "                  break\n",
        "      return legal\n",
        "\n",
        "  def legal_actions(self):\n",
        "      legal = []\n",
        "      for j in range(7):\n",
        "          for i in range(6):\n",
        "              if self.board[i][j] == ' ':\n",
        "                  legal.append(j)\n",
        "                  break\n",
        "      return legal\n",
        "\n",
        "  def check_for_fours(self):\n",
        "      for i in range(6):\n",
        "          for j in range(7):\n",
        "              if self.board[i][j] != ' ':\n",
        "                  # check if a vertical four-in-a-row starts at (i, j)\n",
        "                  if self.vertical_check(i, j):\n",
        "                      self.done = True\n",
        "                      return\n",
        "\n",
        "                  # check if a horizontal four-in-a-row starts at (i, j)\n",
        "                  if self.horizontal_check(i, j):\n",
        "                      self.done = True\n",
        "                      return\n",
        "\n",
        "                  # check if a diagonal (either way) four-in-a-row starts at (i, j)\n",
        "                  diag_fours = self.diagonal_check(i, j)\n",
        "                  if diag_fours:\n",
        "                      self.done = True\n",
        "                      return\n",
        "\n",
        "  def vertical_check(self, row, col):\n",
        "      # print(\"checking vert\")\n",
        "      four_in_a_row = False\n",
        "      consecutive_count = 0\n",
        "\n",
        "      for i in range(row, 6):\n",
        "          if self.board[i][col].lower() == self.board[row][col].lower():\n",
        "              consecutive_count += 1\n",
        "          else:\n",
        "              break\n",
        "\n",
        "      if consecutive_count >= 4:\n",
        "          four_in_a_row = True\n",
        "          if 'x' == self.board[row][col].lower():\n",
        "              self.winner = Winner.white\n",
        "          else:\n",
        "              self.winner = Winner.black\n",
        "\n",
        "      return four_in_a_row\n",
        "\n",
        "  def horizontal_check(self, row, col):\n",
        "      four_in_a_row = False\n",
        "      consecutive_count = 0\n",
        "\n",
        "      for j in range(col, 7):\n",
        "          if self.board[row][j].lower() == self.board[row][col].lower():\n",
        "              consecutive_count += 1\n",
        "          else:\n",
        "              break\n",
        "\n",
        "      if consecutive_count >= 4:\n",
        "          four_in_a_row = True\n",
        "          if 'x' == self.board[row][col].lower():\n",
        "              self.winner = Winner.white\n",
        "          else:\n",
        "              self.winner = Winner.black\n",
        "\n",
        "      return four_in_a_row\n",
        "\n",
        "  def diagonal_check(self, row, col):\n",
        "      four_in_a_row = False\n",
        "      count = 0\n",
        "\n",
        "      consecutive_count = 0\n",
        "      j = col\n",
        "      for i in range(row, 6):\n",
        "          if j > 6:\n",
        "              break\n",
        "          elif self.board[i][j].lower() == self.board[row][col].lower():\n",
        "              consecutive_count += 1\n",
        "          else:\n",
        "              break\n",
        "          j += 1\n",
        "\n",
        "      if consecutive_count >= 4:\n",
        "          count += 1\n",
        "          if 'x' == self.board[row][col].lower():\n",
        "              self.winner = Winner.white\n",
        "          else:\n",
        "              self.winner = Winner.black\n",
        "\n",
        "      consecutive_count = 0\n",
        "      j = col\n",
        "      for i in range(row, -1, -1):\n",
        "          if j > 6:\n",
        "              break\n",
        "          elif self.board[i][j].lower() == self.board[row][col].lower():\n",
        "              consecutive_count += 1\n",
        "          else:\n",
        "              break\n",
        "          j += 1\n",
        "\n",
        "      if consecutive_count >= 4:\n",
        "          count += 1\n",
        "          if 'x' == self.board[row][col].lower():\n",
        "              self.winner = Winner.white\n",
        "          else:\n",
        "              self.winner = Winner.black\n",
        "\n",
        "      if count > 0:\n",
        "          four_in_a_row = True\n",
        "\n",
        "      return four_in_a_row\n",
        "\n",
        "  def black_and_white_plane(self):\n",
        "      board_white = numpy.copy(self.board)\n",
        "      board_black = numpy.copy(self.board)\n",
        "      for i in range(6):\n",
        "          for j in range(7):\n",
        "              if self.board[i][j] == ' ':\n",
        "                  board_white[i][j] = 0\n",
        "                  board_black[i][j] = 0\n",
        "              elif self.board[i][j] == 'X':\n",
        "                  board_white[i][j] = 1\n",
        "                  board_black[i][j] = 0\n",
        "              else:\n",
        "                  board_white[i][j] = 0\n",
        "                  board_black[i][j] = 1\n",
        "\n",
        "      return numpy.array(board_white), numpy.array(board_black)\n",
        "\n",
        "  def render(self):\n",
        "      print(\"\\nRound: \" + str(self.turn))\n",
        "\n",
        "      for i in range(5, -1, -1):\n",
        "          print(\"\\t\", end=\"\")\n",
        "          for j in range(7):\n",
        "              print(\"| \" + str(self.board[i][j]), end=\" \")\n",
        "          print(\"|\")\n",
        "      print(\"\\t  _   _   _   _   _   _   _ \")\n",
        "      print(\"\\t  1   2   3   4   5   6   7 \")\n",
        "\n",
        "      if self.done:\n",
        "          print(\"Game Over!\")\n",
        "          if self.winner == Winner.white:\n",
        "              print(\"X is the winner\")\n",
        "          elif self.winner == Winner.black:\n",
        "              print(\"O is the winner\")\n",
        "          else:\n",
        "              print(\"Game was a draw\")\n",
        "\n",
        "  @property\n",
        "  def observation(self):\n",
        "      return ''.join(''.join(x for x in y) for y in self.board)\n",
        "\n",
        "\n",
        "class Game(object):\n",
        "  \"\"\"A single episode of interaction with the environment.\"\"\"\n",
        "\n",
        "  def __init__(self, action_space_size: int, discount: float):\n",
        "    self.environment = Environment().reset()  # Game specific environment.\n",
        "    self.history = []\n",
        "    self.rewards = []\n",
        "    self.child_visits = []\n",
        "    self.root_values = []\n",
        "    self.action_space_size = action_space_size\n",
        "    self.discount = discount\n",
        "\n",
        "  def terminal(self) -> bool:\n",
        "    # Game specific termination rules.\n",
        "    return self.environment.done\n",
        "\n",
        "  def legal_actions(self) -> List[Action]:\n",
        "    # Game specific calculation of legal actions.\n",
        "    return self.environment.legal_actions()\n",
        "\n",
        "  def apply(self, action: Action):\n",
        "    reward = self.environment.step(action)\n",
        "    reward = reward if self.environment.turn % 2 != 0 and reward == 1 else -reward\n",
        "    self.rewards.append(reward)\n",
        "    self.history.append(action)\n",
        "\n",
        "  def store_search_statistics(self, root: Node):\n",
        "    sum_visits = sum(child.visit_count for child in root.children.values())\n",
        "    action_space = (Action(index) for index in range(self.action_space_size))\n",
        "    self.child_visits.append([\n",
        "        root.children[a].visit_count / sum_visits if a in root.children else 0\n",
        "        for a in action_space\n",
        "    ])\n",
        "    self.root_values.append(root.value())\n",
        "\n",
        "  def make_image(self, state_index: int):\n",
        "    # Game specific feature planes.    \n",
        "    o = Environment().reset()\n",
        "\n",
        "    for current_index in range(0, state_index):\n",
        "      o.step(self.history[current_index])\n",
        "\n",
        "    black_ary, white_ary = o.black_and_white_plane()\n",
        "    state = [black_ary, white_ary] if o.player_turn() == Player.black else [white_ary, black_ary]\n",
        "    return numpy.array(state)\n",
        "\n",
        "  def make_target(self, state_index: int, num_unroll_steps: int, td_steps: int,\n",
        "                  to_play: Player):\n",
        "    # The value target is the discounted root value of the search tree N steps\n",
        "    # into the future, plus the discounted sum of all rewards until then.\n",
        "    targets = []\n",
        "    for current_index in range(state_index, state_index + num_unroll_steps + 1):\n",
        "      bootstrap_index = current_index + td_steps\n",
        "      if bootstrap_index < len(self.root_values):\n",
        "        value = self.root_values[bootstrap_index] * self.discount**td_steps\n",
        "      else:\n",
        "        value = 0\n",
        "\n",
        "      for i, reward in enumerate(self.rewards[current_index:bootstrap_index]):\n",
        "        value += reward * self.discount**i  # pytype: disable=unsupported-operands\n",
        "\n",
        "      if current_index < len(self.root_values):\n",
        "        targets.append((value, self.rewards[current_index],\n",
        "                        self.child_visits[current_index]))\n",
        "      else:\n",
        "        # States past the end of games are treated as absorbing states.\n",
        "        targets.append((0, 0, []))\n",
        "    return targets\n",
        "\n",
        "  def to_play(self) -> Player:\n",
        "    return self.environment.player_turn\n",
        "\n",
        "  def action_history(self) -> ActionHistory:\n",
        "    return ActionHistory(self.history, self.action_space_size)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k-gvlZwpYGVn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "class ReplayBuffer(object):\n",
        "\n",
        "  def __init__(self, config: MuZeroConfig):\n",
        "    self.window_size = config.window_size\n",
        "    self.batch_size = config.batch_size\n",
        "    self.buffer = []\n",
        "\n",
        "  def save_game(self, game):    \n",
        "    if len(self.buffer) > self.window_size:\n",
        "      self.buffer.pop(0)\n",
        "    self.buffer.append(game)\n",
        "\n",
        "  def sample_batch(self, num_unroll_steps: int, td_steps: int):\n",
        "    games = [self.sample_game() for _ in range(self.batch_size)]\n",
        "    game_pos = [(g, self.sample_position(g)) for g in games]\n",
        "    return [(g.make_image(i), g.history[i:i + num_unroll_steps],\n",
        "             g.make_target(i, num_unroll_steps, td_steps, g.to_play()))\n",
        "            for (g, i) in game_pos]\n",
        "\n",
        "  def sample_game(self) -> Game:\n",
        "    # Sample game from buffer either uniformly or according to some priority.\n",
        "    return numpy.random.choice(self.buffer)\n",
        "\n",
        "  def sample_position(self, game) -> int:\n",
        "    # Sample position from game either uniformly or according to some priority.\n",
        "    return numpy.random.choice(game.history)\n",
        "\n",
        "# Nets\n",
        "class NetworkOutput(typing.NamedTuple):\n",
        "  value: float\n",
        "  reward: float\n",
        "  policy_logits: Dict[Action, float]\n",
        "  hidden_state: List[float]\n",
        "\n",
        "\n",
        "class Conv(nn.Module):\n",
        "    def __init__(self, filters0, filters1, kernel_size, bn=False):\n",
        "        super().__init__()\n",
        "        self.conv = nn.Conv2d(filters0, filters1, kernel_size, stride=1, padding=kernel_size//2, bias=False)\n",
        "        self.bn = None\n",
        "        if bn:\n",
        "            self.bn = nn.BatchNorm2d(filters1)\n",
        "\n",
        "    def forward(self, x):\n",
        "        h = self.conv(x)\n",
        "        if self.bn is not None:\n",
        "            h = self.bn(h)\n",
        "        return h\n",
        "\n",
        "class ResidualBlock(nn.Module):\n",
        "    def __init__(self, filters):\n",
        "        super().__init__()\n",
        "        self.conv = Conv(filters, filters, 3, True)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return F.relu(x + (self.conv(x)))\n",
        "\n",
        "class Representation(nn.Module):\n",
        "    ''' Conversion from observation to inner abstract state '''\n",
        "    def __init__(self, input_shape):\n",
        "        super().__init__()\n",
        "        self.input_shape = input_shape\n",
        "        self.board_size = self.input_shape[1] * self.input_shape[2]\n",
        "\n",
        "        self.layer0 = Conv(self.input_shape[0], num_filters, 3, bn=True)\n",
        "        self.blocks = nn.ModuleList([ResidualBlock(num_filters) for _ in range(num_blocks)])\n",
        "\n",
        "    def forward(self, x):\n",
        "        h = F.relu(self.layer0(x))\n",
        "        for block in self.blocks:\n",
        "            h = block(h)\n",
        "        return h\n",
        "\n",
        "class Prediction(nn.Module):\n",
        "    ''' Policy and value prediction from inner abstract state '''\n",
        "    def __init__(self, action_shape):\n",
        "        super().__init__()\n",
        "        self.board_size = 42\n",
        "        self.action_size = action_shape\n",
        "\n",
        "        self.conv_p1 = Conv(num_filters, 4, 1, bn=True)\n",
        "        self.conv_p2 = Conv(4, 1, 1)\n",
        "\n",
        "        self.conv_v = Conv(num_filters, 4, 1, bn=True)\n",
        "        self.fc_v = nn.Linear(self.board_size * 4, 1, bias=False)\n",
        "\n",
        "    def forward(self, rp):\n",
        "        h_p = F.relu(self.conv_p1(rp))\n",
        "        h_p = self.conv_p2(h_p).view(-1, self.action_size)\n",
        "\n",
        "        h_v = F.relu(self.conv_v(rp))\n",
        "        h_v = self.fc_v(h_v.view(-1, self.board_size * 4))\n",
        "\n",
        "        # range of value is -1 ~ 1\n",
        "        return F.softmax(h_p, dim=-1), torch.tanh(h_v)\n",
        "\n",
        "class Dynamics(nn.Module):\n",
        "    '''Abstruct state transition'''\n",
        "    def __init__(self, rp_shape, act_shape):\n",
        "        super().__init__()\n",
        "        self.rp_shape = rp_shape\n",
        "        self.layer0 = Conv(rp_shape[0] + act_shape[0], num_filters, 3, bn=True)\n",
        "        self.blocks = nn.ModuleList([ResidualBlock(num_filters) for _ in range(num_blocks)])\n",
        "\n",
        "    def forward(self, rp, a):\n",
        "        h = torch.cat([rp, a], dim=1)\n",
        "        h = self.layer0(h)\n",
        "        for block in self.blocks:\n",
        "            h = block(h)\n",
        "        return h\n",
        "\n",
        "class Network(nn.Module):\n",
        "\n",
        "  def __init__(self, action_space_size: int):\n",
        "    super().__init__()\n",
        "    self.steps = 0\n",
        "    self.action_space_size = action_space_size\n",
        "    input_shape = (2, 6, 7)\n",
        "    rp_shape = (num_filters, *input_shape[1:])\n",
        "    self.representation = Representation(input_shape).to(device)\n",
        "    self.prediction = Prediction(action_space_size).to(device)\n",
        "    self.dynamics = Dynamics(rp_shape, (2, 6, 7)).to(device)\n",
        "    self.eval()\n",
        "  \n",
        "  def predict_initial_inference(self, x):    \n",
        "    assert x.ndim in (3, 4)\n",
        "    assert x.shape == (2, 6, 7) or x.shape[1:] == (2, 6, 7)\n",
        "    orig_x = x\n",
        "    if x.ndim == 3:\n",
        "        x = x.reshape(1, 2, 6, 7)\n",
        "    \n",
        "    x = torch.Tensor(x).to(device)\n",
        "    h = self.representation(x)\n",
        "    policy, value = self.prediction(h)\n",
        "    \n",
        "    if orig_x.ndim == 3:\n",
        "        return h[0], policy[0], value[0]\n",
        "    else:\n",
        "        return h, policy, value\n",
        "\n",
        "  def predict_recurrent_inference(self, x, a):\n",
        "\n",
        "    if x.ndim == 3:\n",
        "      x = x.reshape(1, 2, 6, 7)\n",
        "\n",
        "    a = numpy.full((1, 2, 6, 7), a)\n",
        "\n",
        "    g = self.dynamics(x, torch.Tensor(a).to(device))\n",
        "    policy, value = self.prediction(g)\n",
        "    \n",
        "    return g[0], policy[0], value[0]\n",
        "\n",
        "  def initial_inference(self, image) -> NetworkOutput:\n",
        "    # representation + prediction function\n",
        "    h, p, v = self.predict_initial_inference(image.astype(numpy.float32))\n",
        "    return NetworkOutput(v, 0, p, h)\n",
        "\n",
        "  def recurrent_inference(self, hidden_state, action) -> NetworkOutput:\n",
        "    # dynamics + prediction function\n",
        "    g, p, v = self.predict_recurrent_inference(hidden_state, action)\n",
        "    return NetworkOutput(v, 0, p, g) \n",
        "\n",
        "  def training_steps(self) -> int:\n",
        "    # How many steps / batches the network has been trained for.\n",
        "    return self.steps\n",
        "\n",
        "\n",
        "class SharedStorage(object):\n",
        "\n",
        "  def __init__(self):\n",
        "    self._networks = {}\n",
        "\n",
        "  def latest_network(self) -> Network:\n",
        "    if self._networks:\n",
        "      return self._networks[max(self._networks.keys())]\n",
        "    else:\n",
        "      # policy -> uniform, value -> 0, reward -> 0\n",
        "      return make_uniform_network()\n",
        "\n",
        "  def old_network(self) -> Network:\n",
        "    if self._networks:\n",
        "      return self._networks[min(self._networks.keys())]\n",
        "    else:\n",
        "      # policy -> uniform, value -> 0, reward -> 0\n",
        "      return make_uniform_network()\n",
        "\n",
        "  def save_network(self, step: int, network: Network):\n",
        "    self._networks[step] = network\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "50NBNHaqYPP1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "################################################################################\n",
        "############################# Testing the latest net ###########################\n",
        "################################################################################\n",
        "\n",
        "# Battle against random agents\n",
        "def vs_random(network, n=100):\n",
        "    results = {}\n",
        "    for i in range(n):\n",
        "        first_turn = i % 2 == 0\n",
        "        turn = first_turn\n",
        "        game = config.new_game()\n",
        "        r = 0\n",
        "        while not game.terminal():\n",
        "            if turn:\n",
        "              root = Node(0)\n",
        "              current_observation = game.make_image(-1)\n",
        "              expand_node(root, game.to_play(), game.legal_actions(),\n",
        "                          network.initial_inference(current_observation))\n",
        "              add_exploration_noise(config, root)\n",
        "              run_mcts(config, root, game.action_history(), network)\n",
        "              action = select_action(config, len(game.history), root, network)\n",
        "            else:\n",
        "              action = numpy.random.choice(game.legal_actions())\n",
        "            game.apply(action)\n",
        "            turn = not turn\n",
        "        if ((game.environment.winner == Winner.white and first_turn) \n",
        "            or (game.environment.winner == Winner.black and not first_turn)):\n",
        "          r = 1\n",
        "        elif ((game.environment.winner == Winner.black and first_turn) \n",
        "            or (game.environment.winner == Winner.white and not first_turn)):\n",
        "          r = -1\n",
        "        results[r] = results.get(r, 0) + 1\n",
        "    return results\n",
        "\n",
        "def random_vs_random(n=100):\n",
        "    results = {}\n",
        "    for i in range(n):\n",
        "        first_turn = i % 2 == 0\n",
        "        turn = first_turn\n",
        "        game = config.new_game()\n",
        "        r = 0\n",
        "        while not game.terminal():\n",
        "            action = numpy.random.choice(game.legal_actions())\n",
        "            game.apply(action)\n",
        "            turn = not turn\n",
        "        if ((game.environment.winner == Winner.white and first_turn) \n",
        "            or (game.environment.winner == Winner.black and not first_turn)):\n",
        "          r = 1\n",
        "        elif ((game.environment.winner == Winner.black and first_turn) \n",
        "            or (game.environment.winner == Winner.white and not first_turn)):\n",
        "          r = -1\n",
        "        results[r] = results.get(r, 0) + 1\n",
        "    return results\n",
        "\n",
        "def latest_vs_older(last, old, n=100):\n",
        "    results = {}\n",
        "    for i in range(n):\n",
        "        first_turn = i % 2 == 0\n",
        "        turn = first_turn\n",
        "        game = config.new_game()\n",
        "        r = 0\n",
        "        while not game.terminal():\n",
        "            if turn:\n",
        "              root = Node(0)\n",
        "              current_observation = game.make_image(-1)\n",
        "              expand_node(root, game.to_play(), game.legal_actions(),\n",
        "                          last.initial_inference(current_observation))\n",
        "              add_exploration_noise(config, root)\n",
        "              run_mcts(config, root, game.action_history(), last)\n",
        "              action = select_action(config, len(game.history), root, last)\n",
        "            else:\n",
        "              root = Node(0)\n",
        "              current_observation = game.make_image(-1)\n",
        "              expand_node(root, game.to_play(), game.legal_actions(),\n",
        "                          old.initial_inference(current_observation))\n",
        "              add_exploration_noise(config, root)\n",
        "              run_mcts(config, root, game.action_history(), old)\n",
        "              action = select_action(config, len(game.history), root, old)\n",
        "            game.apply(action)\n",
        "            turn = not turn\n",
        "        if ((game.environment.winner == Winner.white and first_turn) \n",
        "            or (game.environment.winner == Winner.black and not first_turn)):\n",
        "          r = 1\n",
        "        elif ((game.environment.winner == Winner.black and first_turn) \n",
        "            or (game.environment.winner == Winner.white and not first_turn)):\n",
        "          r = -1\n",
        "        results[r] = results.get(r, 0) + 1\n",
        "    return results\n",
        "\n",
        "##### End Helpers ########\n",
        "##########################\n",
        "\n",
        "\n",
        "# MuZero training is split into two independent parts: Network training and\n",
        "# self-play data generation.\n",
        "# These two parts only communicate by transferring the latest network checkpoint\n",
        "# from the training to the self-play, and the finished games from the self-play\n",
        "# to the training.\n",
        "def muzero(config: MuZeroConfig):\n",
        "  storage = SharedStorage()\n",
        "  replay_buffer = ReplayBuffer(config)\n",
        "\n",
        "  # Start n concurrent actor threads\n",
        "  threads = list()\n",
        "  for _ in range(config.num_actors):\n",
        "    t = threading.Thread(target=launch_job, args=(run_selfplay, config, storage, replay_buffer))\n",
        "    threads.append(t)\n",
        "\n",
        "  # Start all threads\n",
        "  for x in threads:\n",
        "    x.start() \n",
        "\n",
        "  train_network(config, storage, replay_buffer)\n",
        "\n",
        "  return storage.latest_network()\n",
        "\n",
        "\n",
        "##################################\n",
        "####### Part 1: Self-Play ########\n",
        "\n",
        "\n",
        "# Each self-play job is independent of all others; it takes the latest network\n",
        "# snapshot, produces a game and makes it available to the training job by\n",
        "# writing it to a shared replay buffer.\n",
        "def run_selfplay(config: MuZeroConfig, storage: SharedStorage,\n",
        "                 replay_buffer: ReplayBuffer):\n",
        "  while True:\n",
        "    network = storage.latest_network()\n",
        "    game = play_game(config, network)\n",
        "    replay_buffer.save_game(game)\n",
        "\n",
        "\n",
        "# Each game is produced by starting at the initial board position, then\n",
        "# repeatedly executing a Monte Carlo Tree Search to generate moves until the end\n",
        "# of the game is reached.\n",
        "def play_game(config: MuZeroConfig, network: Network) -> Game:\n",
        "  game = config.new_game()  \n",
        "  while not game.terminal() and len(game.history) < config.max_moves:\n",
        "    # At the root of the search tree we use the representation function to\n",
        "    # obtain a hidden state given the current observation.\n",
        "    root = Node(0)\n",
        "    current_observation = game.make_image(-1)\n",
        "    expand_node(root, game.to_play(), game.legal_actions(),\n",
        "                network.initial_inference(current_observation))\n",
        "    add_exploration_noise(config, root)\n",
        "\n",
        "    # We then run a Monte Carlo Tree Search using only action sequences and the\n",
        "    # model learned by the network.\n",
        "    run_mcts(config, root, game.action_history(), network)\n",
        "    action = select_action(config, len(game.history), root, network)\n",
        "    game.apply(action)\n",
        "    game.store_search_statistics(root)\n",
        "  return game\n",
        "\n",
        "\n",
        "# Core Monte Carlo Tree Search algorithm.\n",
        "# To decide on an action, we run N simulations, always starting at the root of\n",
        "# the search tree and traversing the tree according to the UCB formula until we\n",
        "# reach a leaf node.\n",
        "def run_mcts(config: MuZeroConfig, root: Node, action_history: ActionHistory,\n",
        "             network: Network):\n",
        "  min_max_stats = MinMaxStats(config.known_bounds)\n",
        "\n",
        "  for _ in range(config.num_simulations):\n",
        "    history = action_history.clone()\n",
        "    node = root\n",
        "    search_path = [node]\n",
        "\n",
        "    while node.expanded():\n",
        "      action, node = select_child(config, node, min_max_stats)\n",
        "      history.add_action(action)\n",
        "      search_path.append(node)\n",
        "\n",
        "    # Inside the search tree we use the dynamics function to obtain the next\n",
        "    # hidden state given an action and the previous hidden state.\n",
        "    parent = search_path[-2]\n",
        "    network_output = network.recurrent_inference(parent.hidden_state,\n",
        "                                                 history.last_action())\n",
        "    expand_node(node, history.to_play(), history.action_space(), network_output)\n",
        "\n",
        "    backpropagate(search_path, network_output.value, history.to_play(),\n",
        "                  config.discount, min_max_stats)\n",
        "\n",
        "\n",
        "def select_action(config: MuZeroConfig, num_moves: int, node: Node,\n",
        "                  network: Network):\n",
        "  visit_counts = [\n",
        "      (child.visit_count, action) for action, child in node.children.items()\n",
        "  ]\n",
        "  t = config.visit_softmax_temperature_fn(\n",
        "      num_moves=num_moves, training_steps=network.training_steps())\n",
        "  _, action = softmax_sample(visit_counts, t)\n",
        "  return action\n",
        "\n",
        "\n",
        "# Select the child with the highest UCB score.\n",
        "def select_child(config: MuZeroConfig, node: Node,\n",
        "                 min_max_stats: MinMaxStats):\n",
        "  _, action, child = max(\n",
        "      (ucb_score(config, node, child, min_max_stats), action,\n",
        "       child) for action, child in node.children.items())\n",
        "  return action, child\n",
        "\n",
        "\n",
        "# The score for a node is based on its value, plus an exploration bonus based on\n",
        "# the prior.\n",
        "def ucb_score(config: MuZeroConfig, parent: Node, child: Node,\n",
        "              min_max_stats: MinMaxStats) -> float:\n",
        "  pb_c = math.log((parent.visit_count + config.pb_c_base + 1) /\n",
        "                  config.pb_c_base) + config.pb_c_init\n",
        "  pb_c *= math.sqrt(parent.visit_count) / (child.visit_count + 1)\n",
        "\n",
        "  prior_score = pb_c * child.prior\n",
        "  value_score = min_max_stats.normalize(child.value())\n",
        "  return prior_score + value_score\n",
        "\n",
        "\n",
        "# We expand a node using the value, reward and policy prediction obtained from\n",
        "# the neural network.\n",
        "def expand_node(node: Node, to_play: Player, actions: List[Action],\n",
        "                network_output: NetworkOutput):\n",
        "  node.to_play = to_play\n",
        "  node.hidden_state = network_output.hidden_state\n",
        "  node.reward = network_output.reward\n",
        "  policy = {a: math.exp(network_output.policy_logits[a]) for a in actions}\n",
        "  policy_sum = sum(policy.values())\n",
        "  for action, p in policy.items():\n",
        "    node.children[action] = Node(p / policy_sum)\n",
        "\n",
        "\n",
        "# At the end of a simulation, we propagate the evaluation all the way up the\n",
        "# tree to the root.\n",
        "def backpropagate(search_path: List[Node], value: float, to_play: Player,\n",
        "                  discount: float, min_max_stats: MinMaxStats):\n",
        "  for node in search_path:\n",
        "    node.value_sum += value if node.to_play == to_play else -value\n",
        "    node.visit_count += 1\n",
        "    min_max_stats.update(node.value())\n",
        "\n",
        "    value = node.reward + discount * value\n",
        "\n",
        "\n",
        "# At the start of each search, we add dirichlet noise to the prior of the root\n",
        "# to encourage the search to explore new actions.\n",
        "def add_exploration_noise(config: MuZeroConfig, node: Node):\n",
        "  actions = list(node.children.keys())\n",
        "  noise = numpy.random.dirichlet([config.root_dirichlet_alpha] * len(actions))\n",
        "  frac = config.root_exploration_fraction\n",
        "  for a, n in zip(actions, noise):\n",
        "    node.children[a].prior = node.children[a].prior * (1 - frac) + n * frac\n",
        "\n",
        "\n",
        "######### End Self-Play ##########\n",
        "##################################\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nPcv4eiEYTyY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "##################################\n",
        "####### Part 2: Training #########\n",
        "def train_network(config: MuZeroConfig, storage: SharedStorage,\n",
        "                  replay_buffer: ReplayBuffer):\n",
        "\n",
        "  network = Network(config.action_space_size).to(device)\n",
        "\n",
        "  while True:\n",
        "\n",
        "    optimizer = optim.SGD(network.parameters(), lr=0.01, weight_decay=config.lr_decay_rate,\n",
        "                          momentum=config.momentum)\n",
        "\n",
        "    while not len(replay_buffer.buffer) > 0:\n",
        "      pass\n",
        "    \n",
        "    for i in range(config.training_steps):\n",
        "      if i % config.checkpoint_interval == 0 and i > 0:\n",
        "        storage.save_network(i, network)     \n",
        "        # Test against random agent    \n",
        "        vs_random_once = vs_random(network)\n",
        "        print('network_vs_random = ', sorted(vs_random_once.items()), end='\\n')\n",
        "        vs_older = latest_vs_older(storage.latest_network(), storage.old_network())\n",
        "        print('lastnet_vs_older = ', sorted(vs_older.items()), end='\\n') \n",
        "      batch = replay_buffer.sample_batch(config.num_unroll_steps, config.td_steps)\n",
        "      update_weights(batch, network, optimizer)\n",
        "    storage.save_network(config.training_steps, network)\n",
        "\n",
        "def update_weights(batch, network, optimizer):    \n",
        "\n",
        "  network.train()    \n",
        "\n",
        "  p_loss, v_loss = 0, 0\n",
        "\n",
        "  for image, actions, targets in batch:\n",
        "    # Initial step, from the real observation.\n",
        "    value, reward, policy_logits, hidden_state = network.initial_inference(image)\n",
        "    predictions = [(1.0, value, reward, policy_logits)]\n",
        "\n",
        "    # Recurrent steps, from action and previous hidden state.\n",
        "    for action in actions:\n",
        "      value, reward, policy_logits, hidden_state = network.recurrent_inference(hidden_state, action)\n",
        "      predictions.append((1.0 / len(actions), value, reward, policy_logits))\n",
        "\n",
        "    for prediction, target in zip(predictions, targets):\n",
        "      if(len(target[2]) > 0):\n",
        "        _ , value, reward, policy_logits = prediction\n",
        "        target_value, target_reward, target_policy = target\n",
        "\n",
        "        p_loss += torch.sum(-torch.Tensor(numpy.array(target_policy)).to(device) * torch.log(policy_logits))\n",
        "        v_loss += torch.sum((torch.Tensor([target_value]).to(device) - value) ** 2)\n",
        "  \n",
        "  optimizer.zero_grad()    \n",
        "  total_loss = (p_loss + v_loss)\n",
        "  total_loss.backward()\n",
        "  optimizer.step()\n",
        "  network.steps += 1\n",
        "  print('p_loss %f v_loss %f' % (p_loss / len(batch), v_loss / len(batch)))\n",
        "\n",
        "######### End Training ###########\n",
        "##################################\n",
        "\n",
        "################################################################################\n",
        "############################# End of pseudocode ################################\n",
        "################################################################################\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rMHQkGRaYVug",
        "colab_type": "code",
        "outputId": "e3c0c499-e793-475c-f985-67b3a4f72d54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "print(device)\n",
        "\n",
        "# Stubs to make the typechecker happy.\n",
        "def softmax_sample(distribution, temperature: float):\n",
        "  if temperature == 0:\n",
        "    temperature = 1\n",
        "  distribution = numpy.array(distribution)**(1/temperature)\n",
        "  p_sum = distribution.sum()\n",
        "  sample_temp = distribution/p_sum\n",
        "  return 0, numpy.argmax(numpy.random.multinomial(1, sample_temp, 1))\n",
        "\n",
        "def launch_job(f, *args):\n",
        "  f(*args)\n",
        "\n",
        "def make_uniform_network():\n",
        "  return Network(make_connect4_config().action_space_size).to(device)\n",
        "\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "irYbFinAZwnR",
        "colab_type": "code",
        "outputId": "f63e2cd0-199f-48a2-e771-6f9b3d0edcd4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "config = make_connect4_config()\n",
        "vs_random_once = random_vs_random()\n",
        "print('random_vs_random = ', sorted(vs_random_once.items()), end='\\n')\n",
        "network = muzero(config)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "random_vs_random =  [(-1, 47), (1, 53)]\n",
            "p_loss 9.893979 v_loss 0.601927\n",
            "p_loss 8.691347 v_loss 3.955910\n",
            "p_loss 8.554790 v_loss 4.140779\n",
            "p_loss 8.814151 v_loss 6.870962\n",
            "p_loss 9.364299 v_loss 2.571944\n",
            "p_loss 9.304607 v_loss 5.093513\n",
            "p_loss 9.411983 v_loss 5.341243\n",
            "p_loss 9.229336 v_loss 5.062500\n",
            "p_loss 9.419596 v_loss 5.937500\n",
            "p_loss 9.608006 v_loss 6.640625\n",
            "p_loss 10.105663 v_loss 7.640625\n",
            "p_loss 9.597007 v_loss 7.640625\n",
            "p_loss 9.869622 v_loss 7.015625\n",
            "p_loss 9.549665 v_loss 7.109375\n",
            "p_loss 9.376898 v_loss 6.093750\n",
            "p_loss 9.610895 v_loss 5.859375\n",
            "p_loss 9.609826 v_loss 5.078125\n",
            "p_loss 9.732680 v_loss 7.875000\n",
            "p_loss 9.679747 v_loss 6.250000\n",
            "p_loss 9.736465 v_loss 6.484375\n",
            "p_loss 9.609228 v_loss 6.000000\n",
            "p_loss 9.397275 v_loss 6.484375\n",
            "p_loss 9.608634 v_loss 5.859375\n",
            "p_loss 9.546373 v_loss 5.953125\n",
            "p_loss 9.638430 v_loss 5.796875\n",
            "p_loss 9.276681 v_loss 5.468750\n",
            "p_loss 9.638519 v_loss 6.796875\n",
            "p_loss 9.609526 v_loss 5.859375\n",
            "p_loss 9.485651 v_loss 6.156250\n",
            "p_loss 9.641564 v_loss 5.843750\n",
            "p_loss 9.727946 v_loss 6.890625\n",
            "p_loss 9.610948 v_loss 6.406250\n",
            "p_loss 9.604395 v_loss 5.937500\n",
            "p_loss 9.605557 v_loss 6.328125\n",
            "p_loss 9.486109 v_loss 8.437500\n",
            "p_loss 9.623996 v_loss 6.953125\n",
            "p_loss 9.729959 v_loss 6.218750\n",
            "p_loss 9.729631 v_loss 6.015625\n",
            "p_loss 9.731546 v_loss 4.296875\n",
            "p_loss 9.729529 v_loss 6.234375\n",
            "p_loss 9.729661 v_loss 7.031250\n",
            "p_loss 9.609757 v_loss 5.078125\n",
            "p_loss 9.730211 v_loss 7.265625\n",
            "p_loss 9.243921 v_loss 5.140625\n",
            "p_loss 9.638432 v_loss 5.390625\n",
            "p_loss 9.728609 v_loss 8.125000\n",
            "p_loss 9.608781 v_loss 5.984375\n",
            "p_loss 9.731835 v_loss 6.953125\n",
            "p_loss 9.670328 v_loss 4.937500\n",
            "p_loss 9.727421 v_loss 5.546875\n",
            "p_loss 9.485489 v_loss 6.968750\n",
            "p_loss 9.518372 v_loss 5.796875\n",
            "p_loss 9.669447 v_loss 6.468750\n",
            "p_loss 9.730165 v_loss 6.578125\n",
            "p_loss 9.729876 v_loss 4.984375\n",
            "p_loss 9.547157 v_loss 6.781250\n",
            "p_loss 9.729643 v_loss 6.640625\n",
            "p_loss 9.486866 v_loss 5.953125\n",
            "p_loss 9.729808 v_loss 6.609375\n",
            "p_loss 9.671455 v_loss 6.234375\n",
            "p_loss 9.729378 v_loss 6.500000\n",
            "p_loss 9.517458 v_loss 7.109375\n",
            "p_loss 9.730806 v_loss 5.531250\n",
            "p_loss 9.730116 v_loss 5.953125\n",
            "p_loss 9.456341 v_loss 5.375000\n",
            "p_loss 9.608233 v_loss 5.296875\n",
            "p_loss 9.668807 v_loss 5.546875\n",
            "p_loss 9.668777 v_loss 5.468750\n",
            "p_loss 9.729571 v_loss 5.546875\n",
            "p_loss 9.638371 v_loss 6.015625\n",
            "p_loss 9.729585 v_loss 6.531250\n",
            "p_loss 9.577559 v_loss 5.000000\n",
            "p_loss 9.668775 v_loss 6.687500\n",
            "p_loss 9.668775 v_loss 5.078125\n",
            "p_loss 9.729585 v_loss 6.171875\n",
            "p_loss 9.729585 v_loss 5.828125\n",
            "p_loss 9.729585 v_loss 6.390625\n",
            "p_loss 9.607965 v_loss 6.312500\n",
            "p_loss 9.516749 v_loss 5.984375\n",
            "p_loss 9.729585 v_loss 6.078125\n",
            "p_loss 9.638370 v_loss 5.953125\n",
            "p_loss 9.638370 v_loss 7.250000\n",
            "p_loss 9.729585 v_loss 5.843750\n",
            "p_loss 9.729585 v_loss 6.468750\n",
            "p_loss 9.729585 v_loss 5.906250\n",
            "p_loss 9.729585 v_loss 5.359375\n",
            "p_loss 9.607965 v_loss 6.484375\n",
            "p_loss 9.516749 v_loss 5.687500\n",
            "p_loss 9.729585 v_loss 5.937500\n",
            "p_loss 9.607965 v_loss 5.140625\n",
            "p_loss 9.638370 v_loss 5.531250\n",
            "p_loss 9.729585 v_loss 5.625000\n",
            "p_loss 9.607965 v_loss 5.531250\n",
            "p_loss 9.607965 v_loss 6.015625\n",
            "p_loss 9.638370 v_loss 6.687500\n",
            "p_loss 9.729585 v_loss 5.078125\n",
            "p_loss 9.364724 v_loss 6.203125\n",
            "p_loss 9.607965 v_loss 4.984375\n",
            "p_loss 9.638370 v_loss 5.421875\n",
            "p_loss 9.607965 v_loss 5.718750\n",
            "network_vs_random =  [(-1, 29), (1, 71)]\n",
            "lastnet_vs_older =  [(-1, 47), (0, 8), (1, 45)]\n",
            "p_loss 9.729585 v_loss 6.296875\n",
            "p_loss 9.638370 v_loss 5.218750\n",
            "p_loss 9.638370 v_loss 5.765625\n",
            "p_loss 9.668775 v_loss 6.218750\n",
            "p_loss 9.729585 v_loss 5.937500\n",
            "p_loss 9.729585 v_loss 5.125000\n",
            "p_loss 9.729585 v_loss 5.703125\n",
            "p_loss 9.729585 v_loss 5.296875\n",
            "p_loss 9.729585 v_loss 6.093750\n",
            "p_loss 9.729585 v_loss 5.687500\n",
            "p_loss 9.668775 v_loss 4.953125\n",
            "p_loss 9.607965 v_loss 5.609375\n",
            "p_loss 9.638370 v_loss 5.593750\n",
            "p_loss 9.729585 v_loss 6.953125\n",
            "p_loss 9.607965 v_loss 5.203125\n",
            "p_loss 9.729585 v_loss 5.937500\n",
            "p_loss 9.547154 v_loss 4.765625\n",
            "p_loss 9.547154 v_loss 6.500000\n",
            "p_loss 9.729585 v_loss 6.812500\n",
            "p_loss 9.547154 v_loss 4.812500\n",
            "p_loss 9.729585 v_loss 6.109375\n",
            "p_loss 9.729585 v_loss 5.828125\n",
            "p_loss 9.547154 v_loss 6.359375\n",
            "p_loss 9.638370 v_loss 6.218750\n",
            "p_loss 9.729585 v_loss 4.921875\n",
            "p_loss 9.729585 v_loss 5.625000\n",
            "p_loss 9.547154 v_loss 5.546875\n",
            "p_loss 9.729585 v_loss 5.921875\n",
            "p_loss 9.638370 v_loss 6.531250\n",
            "p_loss 9.486344 v_loss 6.500000\n",
            "p_loss 9.729585 v_loss 6.265625\n",
            "p_loss 9.577559 v_loss 5.593750\n",
            "p_loss 9.638370 v_loss 5.562500\n",
            "p_loss 9.607965 v_loss 4.984375\n",
            "p_loss 9.668775 v_loss 5.453125\n",
            "p_loss 9.607965 v_loss 5.187500\n",
            "p_loss 9.729585 v_loss 5.156250\n",
            "p_loss 9.729585 v_loss 6.093750\n",
            "p_loss 9.638370 v_loss 6.593750\n",
            "p_loss 9.729585 v_loss 5.687500\n",
            "p_loss 9.516749 v_loss 5.500000\n",
            "p_loss 9.638370 v_loss 5.656250\n",
            "p_loss 9.729585 v_loss 6.171875\n",
            "p_loss 9.607965 v_loss 5.125000\n",
            "p_loss 9.638370 v_loss 6.218750\n",
            "p_loss 9.638370 v_loss 4.625000\n",
            "p_loss 9.547154 v_loss 5.890625\n",
            "p_loss 9.638370 v_loss 5.687500\n",
            "p_loss 9.607965 v_loss 5.312500\n",
            "p_loss 9.486344 v_loss 5.546875\n",
            "p_loss 9.577559 v_loss 5.515625\n",
            "p_loss 9.638370 v_loss 7.046875\n",
            "p_loss 9.638370 v_loss 5.796875\n",
            "p_loss 9.516749 v_loss 6.281250\n",
            "p_loss 9.516749 v_loss 4.750000\n",
            "p_loss 9.425534 v_loss 6.109375\n",
            "p_loss 9.607965 v_loss 6.078125\n",
            "p_loss 9.607965 v_loss 5.906250\n",
            "p_loss 9.729585 v_loss 6.265625\n",
            "p_loss 9.729585 v_loss 5.828125\n",
            "p_loss 9.668775 v_loss 5.765625\n",
            "p_loss 9.547154 v_loss 6.296875\n",
            "p_loss 9.668775 v_loss 6.093750\n",
            "p_loss 9.729585 v_loss 6.062500\n",
            "p_loss 9.729585 v_loss 5.328125\n",
            "p_loss 9.638370 v_loss 5.640625\n",
            "p_loss 9.486344 v_loss 6.453125\n",
            "p_loss 9.729585 v_loss 5.937500\n",
            "p_loss 9.364724 v_loss 6.312500\n",
            "p_loss 9.729585 v_loss 5.361357\n",
            "p_loss 9.668775 v_loss 5.234375\n",
            "p_loss 9.638370 v_loss 5.374981\n",
            "p_loss 9.729585 v_loss 5.156250\n",
            "p_loss 9.547154 v_loss 6.265625\n",
            "p_loss 9.668775 v_loss 6.236503\n",
            "p_loss 9.607965 v_loss 5.257741\n",
            "p_loss 9.729585 v_loss 5.281235\n",
            "p_loss 9.729585 v_loss 5.218750\n",
            "p_loss 9.638370 v_loss 5.296875\n",
            "p_loss 9.729585 v_loss 5.909706\n",
            "p_loss 9.729585 v_loss 5.406250\n",
            "p_loss 9.668775 v_loss 5.796867\n",
            "p_loss 9.638370 v_loss 7.033606\n",
            "p_loss 9.638370 v_loss 6.044357\n",
            "p_loss 9.638370 v_loss 6.156266\n",
            "p_loss 9.516749 v_loss 4.984373\n",
            "p_loss 9.668775 v_loss 5.312500\n",
            "p_loss 9.668775 v_loss 5.546875\n",
            "p_loss 9.729585 v_loss 5.312500\n",
            "p_loss 9.668775 v_loss 5.383249\n",
            "p_loss 9.729585 v_loss 5.770036\n",
            "p_loss 9.729585 v_loss 6.033074\n",
            "p_loss 9.638370 v_loss 5.437500\n",
            "p_loss 9.668775 v_loss 4.976887\n",
            "p_loss 9.638370 v_loss 5.343680\n",
            "p_loss 9.729585 v_loss 4.828125\n",
            "p_loss 9.607965 v_loss 5.000000\n",
            "p_loss 9.729585 v_loss 4.437500\n",
            "p_loss 9.668775 v_loss 5.765625\n",
            "p_loss 9.547154 v_loss 6.156250\n",
            "network_vs_random =  [(-1, 33), (1, 67)]\n",
            "lastnet_vs_older =  [(-1, 47), (0, 4), (1, 49)]\n",
            "p_loss 9.729585 v_loss 6.500000\n",
            "p_loss 9.516749 v_loss 5.671875\n",
            "p_loss 9.729585 v_loss 6.483990\n",
            "p_loss 9.577559 v_loss 6.233738\n",
            "p_loss 9.668775 v_loss 5.312500\n",
            "p_loss 9.607965 v_loss 5.750000\n",
            "p_loss 9.607965 v_loss 5.156230\n",
            "p_loss 9.729585 v_loss 5.234306\n",
            "p_loss 9.729585 v_loss 6.562348\n",
            "p_loss 9.668775 v_loss 5.384754\n",
            "p_loss 9.729585 v_loss 6.562500\n",
            "p_loss 9.607965 v_loss 5.062500\n",
            "p_loss 9.729585 v_loss 6.406250\n",
            "p_loss 9.729585 v_loss 6.250000\n",
            "p_loss 9.547154 v_loss 6.171875\n",
            "p_loss 9.729585 v_loss 6.484375\n",
            "p_loss 9.577559 v_loss 5.718750\n",
            "p_loss 9.729585 v_loss 5.781250\n",
            "p_loss 9.638370 v_loss 5.515625\n",
            "p_loss 9.668775 v_loss 4.765625\n",
            "p_loss 9.577559 v_loss 5.468750\n",
            "p_loss 9.729585 v_loss 5.781250\n",
            "p_loss 9.577559 v_loss 4.656250\n",
            "p_loss 9.638370 v_loss 4.343750\n",
            "p_loss 9.729585 v_loss 5.703125\n",
            "p_loss 9.577559 v_loss 5.671875\n",
            "p_loss 9.486344 v_loss 6.562500\n",
            "p_loss 9.547154 v_loss 6.046875\n",
            "p_loss 9.668775 v_loss 4.687500\n",
            "p_loss 9.699180 v_loss 6.250000\n",
            "p_loss 9.729585 v_loss 4.921875\n",
            "p_loss 9.638370 v_loss 5.359375\n",
            "p_loss 9.729585 v_loss 5.078125\n",
            "p_loss 9.699180 v_loss 5.546875\n",
            "p_loss 9.729585 v_loss 4.687500\n",
            "p_loss 9.729585 v_loss 5.546875\n",
            "p_loss 9.729585 v_loss 5.700053\n",
            "p_loss 9.729585 v_loss 5.859375\n",
            "p_loss 9.729585 v_loss 6.015625\n",
            "p_loss 9.729585 v_loss 5.234369\n",
            "p_loss 9.729585 v_loss 5.078125\n",
            "p_loss 9.729585 v_loss 5.546875\n",
            "p_loss 9.729585 v_loss 5.546875\n",
            "p_loss 9.729585 v_loss 6.093750\n",
            "p_loss 9.729585 v_loss 5.468750\n",
            "p_loss 9.729585 v_loss 6.171875\n",
            "p_loss 9.638370 v_loss 6.015625\n",
            "p_loss 9.729585 v_loss 5.625000\n",
            "p_loss 9.729585 v_loss 5.156250\n",
            "p_loss 9.729585 v_loss 5.312500\n",
            "p_loss 9.638370 v_loss 5.750000\n",
            "p_loss 9.547154 v_loss 5.390625\n",
            "p_loss 9.607965 v_loss 5.765625\n",
            "p_loss 9.577559 v_loss 6.062500\n",
            "p_loss 9.638370 v_loss 5.359375\n",
            "p_loss 9.729585 v_loss 6.484375\n",
            "p_loss 9.729585 v_loss 5.859375\n",
            "p_loss 9.729585 v_loss 5.937500\n",
            "p_loss 9.729585 v_loss 5.000000\n",
            "p_loss 9.668775 v_loss 5.859375\n",
            "p_loss 9.729585 v_loss 6.484375\n",
            "p_loss 9.638370 v_loss 6.015625\n",
            "p_loss 9.729585 v_loss 6.953125\n",
            "p_loss 9.516749 v_loss 5.359375\n",
            "p_loss 9.729585 v_loss 4.687500\n",
            "p_loss 9.729585 v_loss 7.421875\n",
            "p_loss 9.699180 v_loss 5.703125\n",
            "p_loss 9.516749 v_loss 5.937500\n",
            "p_loss 9.729585 v_loss 6.171875\n",
            "p_loss 9.729585 v_loss 5.390625\n",
            "p_loss 9.699180 v_loss 5.468750\n",
            "p_loss 9.729585 v_loss 6.093750\n",
            "p_loss 9.729585 v_loss 5.546875\n",
            "p_loss 9.729585 v_loss 5.859375\n",
            "p_loss 9.547154 v_loss 4.453125\n",
            "p_loss 9.729585 v_loss 6.406250\n",
            "p_loss 9.638370 v_loss 5.312500\n",
            "p_loss 9.699180 v_loss 5.625000\n",
            "p_loss 9.638370 v_loss 6.484375\n",
            "p_loss 9.212699 v_loss 5.703125\n",
            "p_loss 9.607965 v_loss 5.812500\n",
            "p_loss 9.607965 v_loss 4.921875\n",
            "p_loss 9.607965 v_loss 5.312500\n",
            "p_loss 9.668775 v_loss 6.796875\n",
            "p_loss 9.729585 v_loss 5.390625\n",
            "p_loss 9.729585 v_loss 5.781249\n",
            "p_loss 9.516749 v_loss 3.953125\n",
            "p_loss 9.729585 v_loss 5.390625\n",
            "p_loss 9.607965 v_loss 6.250000\n",
            "p_loss 9.729585 v_loss 6.406250\n",
            "p_loss 9.577559 v_loss 4.531250\n",
            "p_loss 9.729585 v_loss 5.781250\n",
            "p_loss 9.729585 v_loss 5.390625\n",
            "p_loss 9.577559 v_loss 5.203125\n",
            "p_loss 9.729585 v_loss 5.468750\n",
            "p_loss 9.577559 v_loss 5.250000\n",
            "p_loss 9.729585 v_loss 6.406250\n",
            "p_loss 9.729585 v_loss 6.250000\n",
            "p_loss 9.729585 v_loss 5.692367\n",
            "p_loss 9.638370 v_loss 6.062500\n",
            "network_vs_random =  [(-1, 27), (1, 73)]\n",
            "lastnet_vs_older =  [(-1, 45), (0, 7), (1, 48)]\n",
            "p_loss 9.607965 v_loss 6.093750\n",
            "p_loss 9.638370 v_loss 4.968750\n",
            "p_loss 9.729585 v_loss 5.937500\n",
            "p_loss 9.547154 v_loss 5.562500\n",
            "p_loss 9.638370 v_loss 4.578125\n",
            "p_loss 9.486344 v_loss 5.294950\n",
            "p_loss 9.729585 v_loss 4.687500\n",
            "p_loss 9.729585 v_loss 6.015625\n",
            "p_loss 9.607965 v_loss 5.781250\n",
            "p_loss 9.729585 v_loss 4.531236\n",
            "p_loss 9.729585 v_loss 6.078133\n",
            "p_loss 9.729585 v_loss 4.921767\n",
            "p_loss 9.638370 v_loss 5.509293\n",
            "p_loss 9.547154 v_loss 5.390625\n",
            "p_loss 9.729585 v_loss 4.921860\n",
            "p_loss 9.729585 v_loss 6.316791\n",
            "p_loss 9.607965 v_loss 4.531090\n",
            "p_loss 9.729585 v_loss 5.078125\n",
            "p_loss 9.668775 v_loss 5.078024\n",
            "p_loss 9.729585 v_loss 5.609815\n",
            "p_loss 9.729585 v_loss 5.468715\n",
            "p_loss 9.547154 v_loss 4.680043\n",
            "p_loss 9.729585 v_loss 5.859375\n",
            "p_loss 9.729585 v_loss 5.468747\n",
            "p_loss 9.729585 v_loss 6.249979\n",
            "p_loss 9.729585 v_loss 5.703061\n",
            "p_loss 9.638370 v_loss 6.093711\n",
            "p_loss 9.668775 v_loss 5.452218\n",
            "p_loss 9.638370 v_loss 5.906246\n",
            "p_loss 9.729585 v_loss 4.907439\n",
            "p_loss 9.638370 v_loss 6.000257\n",
            "p_loss 9.729585 v_loss 5.937452\n",
            "p_loss 9.607965 v_loss 5.853457\n",
            "p_loss 9.638370 v_loss 5.781186\n",
            "p_loss 9.729585 v_loss 6.249977\n",
            "p_loss 9.607965 v_loss 6.406226\n",
            "p_loss 9.607965 v_loss 5.859349\n",
            "p_loss 9.729585 v_loss 5.703093\n",
            "p_loss 9.729585 v_loss 5.372836\n",
            "p_loss 9.699180 v_loss 5.359443\n",
            "p_loss 9.729585 v_loss 6.483509\n",
            "p_loss 9.729585 v_loss 5.155165\n",
            "p_loss 9.607965 v_loss 4.603976\n",
            "p_loss 9.729585 v_loss 4.828409\n",
            "p_loss 9.729585 v_loss 6.212677\n",
            "p_loss 9.729585 v_loss 4.425554\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.729589 v_loss 1.171875\n",
            "p_loss 9.729590 v_loss 0.937500\n",
            "p_loss 9.729578 v_loss 1.328125\n",
            "p_loss 9.668768 v_loss 1.079524\n",
            "p_loss 9.577559 v_loss 1.796875\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.486344 v_loss 1.051934\n",
            "p_loss 9.699180 v_loss 1.968928\n",
            "p_loss 9.729585 v_loss 1.328125\n",
            "p_loss 9.729585 v_loss 1.019460\n",
            "p_loss 9.668775 v_loss 1.174096\n",
            "p_loss 9.699180 v_loss 1.234375\n",
            "p_loss 9.729585 v_loss 1.328125\n",
            "p_loss 9.547154 v_loss 1.312500\n",
            "p_loss 9.668775 v_loss 1.062500\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.638370 v_loss 1.125000\n",
            "p_loss 9.729585 v_loss 1.015625\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.668775 v_loss 1.296875\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.729585 v_loss 0.937500\n",
            "p_loss 9.607965 v_loss 1.265625\n",
            "p_loss 9.607965 v_loss 1.265625\n",
            "p_loss 9.638370 v_loss 0.734375\n",
            "p_loss 9.729585 v_loss 0.781250\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.668775 v_loss 1.296875\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.638370 v_loss 1.125000\n",
            "p_loss 9.729585 v_loss 0.546875\n",
            "p_loss 9.607965 v_loss 1.265625\n",
            "p_loss 9.638370 v_loss 1.203125\n",
            "p_loss 9.607965 v_loss 1.421875\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.729585 v_loss 1.015625\n",
            "p_loss 9.729585 v_loss 0.546875\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.729585 v_loss 1.015625\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.607965 v_loss 0.953125\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.699180 v_loss 0.843750\n",
            "p_loss 9.638370 v_loss 1.359375\n",
            "p_loss 9.607965 v_loss 0.875000\n",
            "p_loss 9.729585 v_loss 1.015625\n",
            "p_loss 9.729585 v_loss 1.640625\n",
            "p_loss 9.547154 v_loss 0.843750\n",
            "p_loss 9.638370 v_loss 1.046875\n",
            "p_loss 9.729585 v_loss 1.796875\n",
            "network_vs_random =  [(-1, 22), (1, 78)]\n",
            "lastnet_vs_older =  [(-1, 45), (0, 10), (1, 45)]\n",
            "p_loss 9.729585 v_loss 1.015625\n",
            "p_loss 9.699180 v_loss 1.078125\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.547154 v_loss 0.921875\n",
            "p_loss 9.516749 v_loss 0.593750\n",
            "p_loss 9.607965 v_loss 1.343750\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.577559 v_loss 1.250000\n",
            "p_loss 9.607965 v_loss 1.500000\n",
            "p_loss 9.638370 v_loss 0.968750\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.607965 v_loss 0.953125\n",
            "p_loss 9.638370 v_loss 1.125000\n",
            "p_loss 9.729585 v_loss 0.703125\n",
            "p_loss 9.638370 v_loss 0.968750\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.668775 v_loss 0.906250\n",
            "p_loss 9.577559 v_loss 1.015625\n",
            "p_loss 9.607965 v_loss 1.265625\n",
            "p_loss 9.607965 v_loss 0.953125\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.607965 v_loss 1.578125\n",
            "p_loss 9.607965 v_loss 1.265625\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.699180 v_loss 1.078125\n",
            "p_loss 9.699180 v_loss 0.921875\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.607965 v_loss 1.109375\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.729585 v_loss 0.937500\n",
            "p_loss 9.638370 v_loss 0.968750\n",
            "p_loss 9.729585 v_loss 1.406250\n",
            "p_loss 9.547154 v_loss 1.234375\n",
            "p_loss 9.638370 v_loss 0.968750\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.638370 v_loss 0.734375\n",
            "p_loss 9.638370 v_loss 1.515625\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.638370 v_loss 1.046875\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.516749 v_loss 0.828125\n",
            "p_loss 9.638370 v_loss 1.281250\n",
            "p_loss 9.729585 v_loss 1.562500\n",
            "p_loss 9.668775 v_loss 1.453125\n",
            "p_loss 9.516749 v_loss 0.828125\n",
            "p_loss 9.577559 v_loss 0.937500\n",
            "p_loss 9.516749 v_loss 1.218750\n",
            "p_loss 9.729585 v_loss 1.250000\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.607965 v_loss 1.031250\n",
            "p_loss 9.729585 v_loss 0.937500\n",
            "p_loss 9.607965 v_loss 1.343750\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.729585 v_loss 0.859375\n",
            "p_loss 9.638370 v_loss 0.734375\n",
            "p_loss 9.607965 v_loss 1.109375\n",
            "p_loss 9.638370 v_loss 1.906250\n",
            "p_loss 9.638370 v_loss 1.046875\n",
            "p_loss 9.638370 v_loss 1.125000\n",
            "p_loss 9.638370 v_loss 1.281250\n",
            "p_loss 9.668775 v_loss 1.296875\n",
            "p_loss 9.638370 v_loss 1.203125\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.668775 v_loss 0.984375\n",
            "p_loss 9.638370 v_loss 0.890625\n",
            "p_loss 9.638370 v_loss 0.968750\n",
            "p_loss 9.607965 v_loss 0.796875\n",
            "p_loss 9.638370 v_loss 1.046875\n",
            "p_loss 9.729585 v_loss 0.546875\n",
            "p_loss 9.638370 v_loss 1.125000\n",
            "p_loss 9.699180 v_loss 1.000000\n",
            "p_loss 9.668775 v_loss 1.140625\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.729585 v_loss 1.484375\n",
            "p_loss 9.729585 v_loss 1.171875\n",
            "p_loss 9.729585 v_loss 1.093750\n",
            "p_loss 9.486344 v_loss 0.968750\n",
            "p_loss 9.607965 v_loss 1.109375\n",
            "p_loss 9.638370 v_loss 0.578125\n",
            "p_loss 9.729585 v_loss 1.250000\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}