pix2pix_mono_seg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Segmentation Models: using `tf.keras` framework.\n",
      "{'img_shape': [512, 512], 'input_channels': 3, 'output_channels': 1}\n"
     ]
    }
   ],
   "source": [
    "# TBD 1 : logger 추가\n",
    "# TBD 2: flask github 참고, method, class, 파일의 맨 윗단 마다 pydoc 형식으로 달기\n",
    "# TBD 3: 축약어를 자제할것 (특히 변수)\n",
    "\n",
    "# -------------------------\n",
    "#   To-do\n",
    "# -------------------------\n",
    "\n",
    "# 0. add data-setter, receiver system use python queue.Queue() class\n",
    "# this will resolve i/o bottleneck\n",
    "# 1. add logger\n",
    "# 2. make image drawer overlay mask on image\n",
    "# 3. make iterable\n",
    "# 4. make verbose turn on and off\n",
    "# 5. write pydoc\n",
    "\n",
    "# python basic Module\n",
    "import os\n",
    "import sys\n",
    "import types\n",
    "import progressbar\n",
    "from datetime import datetime\n",
    "from shutil import copy\n",
    "from pickle import dump, load\n",
    "\n",
    "# math, image, plot Module\n",
    "import numpy as np\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt  # TBD\n",
    "\n",
    "# tensorflow Module\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import backend as keras_backend\n",
    "from tensorflow.keras.layers import GaussianNoise\n",
    "from tensorflow.keras.layers import Input, Concatenate\n",
    "from tensorflow.keras.layers import Conv2D\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.optimizers import Nadam\n",
    "from tensorflow.keras.initializers import RandomNormal\n",
    "from tensorflow.keras import losses\n",
    "\n",
    "# keras segmentaion third-party Moudle\n",
    "import segmentation_models as sm\n",
    "\n",
    "# custom Module\n",
    "from gan_module.data_loader.medical_segmentation_data_loader import DataLoader\n",
    "from gan_module.data_loader.manage_batch import BatchQueueManager\n",
    "\n",
    "from gan_module.model.build_model import build_generator, build_discriminator\n",
    "from gan_module.util.draw_images import ImageDrawer\n",
    "from gan_module import custom_loss\n",
    "from gan_module.custom_loss import f1_loss_for_training, f1_score, dice_loss_for_training\n",
    "from gan_module.util.manage_learning_rate import learning_rate_scheduler\n",
    "from gan_module.config import CONFIG\n",
    "\n",
    "custom_loss.AXIS = [1, 2, 3]\n",
    "\n",
    "USE_GPU = True\n",
    "\n",
    "if USE_GPU:\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "    gpu_devices = tf.config.experimental.list_physical_devices(\"GPU\")\n",
    "    for device in gpu_devices:\n",
    "        tf.config.experimental.set_memory_growth(device, True)\n",
    "else:\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = '-1'\n",
    "\n",
    "KERNEL_INITIALIZER = RandomNormal(mean=0.0, stddev=0.02)\n",
    "\n",
    "\n",
    "class Pix2PixSegmentation:\n",
    "    def __init__(\n",
    "        self,\n",
    "        generator_power=32,\n",
    "        discriminator_power=32,\n",
    "        generator_learning_rate=1e-4,\n",
    "        discriminator_learning_rate=1e-4,\n",
    "        find_error=False,\n",
    "        temp_weights_path=\".\",\n",
    "        draw_images=True,\n",
    "        on_memory=True,\n",
    "        code_test=False\n",
    "    ):\n",
    "        # Input shape\n",
    "        img_shape = CONFIG[\"img_shape\"]\n",
    "        input_channels = CONFIG[\"input_channels\"]\n",
    "        output_channels = CONFIG[\"output_channels\"]\n",
    "\n",
    "        input_img_shape = (*img_shape, input_channels)\n",
    "        output_img_shape = (*img_shape, output_channels)\n",
    "        # set parameter\n",
    "        self.start_epoch = None\n",
    "        self.history = {\"generator_loss\": [],\n",
    "                        \"f1_loss_train\": [], \"f1_score_train\": [],\n",
    "                        \"f1_loss_valid\": [], \"f1_score_valid\": []}\n",
    "        self.discriminator_loss_ratio = keras_backend.variable(0.5)\n",
    "        self.f1_loss_ratio = keras_backend.variable(100)\n",
    "        self.find_error = find_error\n",
    "        self.find_error_epoch = 30\n",
    "        self.error_list = []\n",
    "        self.temp_weights_path = temp_weights_path\n",
    "\n",
    "        # Configure data loader\n",
    "        self.dataset_name = \"tumor\"\n",
    "        self.data_loader = DataLoader(\n",
    "            dataset_name=self.dataset_name,\n",
    "            on_memory=on_memory, code_test=code_test\n",
    "        )\n",
    "\n",
    "        self.loaded_data_index = {\n",
    "            \"train\": np.arange(self.data_loader.data_length[\"train\"]),\n",
    "            \"valid\": np.arange(self.data_loader.data_length[\"valid\"])\n",
    "        }\n",
    "\n",
    "        # Configure Image Drawer\n",
    "        self.draw_images = draw_images\n",
    "        self.image_drawer = ImageDrawer(\n",
    "            dataset_name=self.dataset_name, data_loader=self.data_loader\n",
    "        )\n",
    "        self.discriminator_acc_previous = 0.5\n",
    "        self.discriminator_acces = np.array(\n",
    "            [0.5 for _ in range(self.data_loader.data_length[\"train\"])])\n",
    "        self.discriminator_acces_previous = self.discriminator_acces.copy()\n",
    "        self.generator_losses = np.array(\n",
    "            [1 for _ in range(self.data_loader.data_length[\"train\"])], dtype=np.float32)\n",
    "        self.generator_losses_previous = self.generator_losses.copy()\n",
    "        self.generator_loss_min = 500\n",
    "        self.generator_loss_previous = 100\n",
    "        self.generator_loss_max_previous = 1000\n",
    "        self.generator_loss_max_min = 1000\n",
    "        self.generator_loss_min_min = 1000\n",
    "        self.weight_save_stack = False\n",
    "        self.training_end_stack = 0\n",
    "        # Calculate output shape of D (PatchGAN)\n",
    "        patch = 2 ** 3\n",
    "        self.disc_patch = (patch, patch, 1)\n",
    "\n",
    "        # Number of filters in the first layer of G and D\n",
    "        self.generator_learning_rate = generator_learning_rate\n",
    "        self.discriminator_learning_rate = discriminator_learning_rate\n",
    "        generator_optimizer = Nadam(self.generator_learning_rate)\n",
    "        discriminator_optimizer = Nadam(self.discriminator_learning_rate)\n",
    "\n",
    "        # Build the generator\n",
    "        self.generator = build_generator(\n",
    "            input_img_shape=input_img_shape,\n",
    "            output_channels=output_channels,\n",
    "            generator_power=generator_power,\n",
    "            kernel_initializer=KERNEL_INITIALIZER,\n",
    "        )\n",
    "        # Build and compile the discriminator\n",
    "        self.discriminator = build_discriminator(\n",
    "            input_img_shape=input_img_shape,\n",
    "            output_img_shape=output_img_shape,\n",
    "            discriminator_power=discriminator_power,\n",
    "            kernel_initializer=KERNEL_INITIALIZER,\n",
    "        )\n",
    "        # self.discriminator = self.build_discriminator()\n",
    "        # 'mse' or tf.keras.losses.Huber() tf.keras.losses.LogCosh()\n",
    "        self.discriminator.compile(\n",
    "            loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1),\n",
    "            optimizer=discriminator_optimizer,\n",
    "            metrics=[\"accuracy\"],\n",
    "        )\n",
    "\n",
    "        # -------------------------\n",
    "        # Construct Computational\n",
    "        #   Graph of Generator\n",
    "        # -------------------------\n",
    "\n",
    "        # Input images and their conditioning images\n",
    "        original_img = Input(shape=input_img_shape)\n",
    "        masked_img = Input(shape=output_img_shape)\n",
    "        # generate image from original_img for target masked_img\n",
    "        model_masked_img = self.generator(original_img)\n",
    "\n",
    "        # For the combined model we will only train the generator\n",
    "        self.discriminator.trainable = False\n",
    "        # Discriminators determines validity of translated images / condition pairs\n",
    "        model_validity = self.discriminator([original_img, model_masked_img])\n",
    "        # give score by\n",
    "        # 1. how generator trick discriminator\n",
    "        # 2. how generator's image same as real photo in pixel\n",
    "        # 3. if you want change loss, see doc https://keras.io/api/losses/\n",
    "        # 4. 'mse', 'mae', tf.keras.losses.LogCosh(),  tf.keras.losses.Huber()\n",
    "        self.combined = Model(\n",
    "            inputs=[original_img, masked_img],\n",
    "            outputs=[model_validity, model_masked_img],\n",
    "        )\n",
    "\n",
    "        self.combined.compile(\n",
    "            loss=[\n",
    "                tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1),\n",
    "                dice_loss_for_training\n",
    "            ],\n",
    "            loss_weights=[self.discriminator_loss_ratio,\n",
    "                          self.f1_loss_ratio],\n",
    "            optimizer=generator_optimizer\n",
    "        )\n",
    "\n",
    "    def train(self, epochs, batch_size=1, sample_interval=50, epoch_shuffle_term=10):\n",
    "\n",
    "        start_time = datetime.now()\n",
    "\n",
    "        # Adversarial loss ground truths\n",
    "        self.training_end_stack = 0\n",
    "        self.batch_size = batch_size\n",
    "        valid_patch = np.ones(\n",
    "            (self.batch_size, *self.disc_patch), dtype=np.float32)\n",
    "        fake_patch = np.zeros(\n",
    "            (self.batch_size, *self.disc_patch), dtype=np.float32)\n",
    "        # TBD : move batch_queue_manager to __init__\n",
    "        self.batch_queue_manager = BatchQueueManager(self)\n",
    "\n",
    "        if self.start_epoch is None:\n",
    "            self.start_epoch = 0\n",
    "        for epoch in range(self.start_epoch, epochs):\n",
    "            bar = progressbar.ProgressBar(\n",
    "                maxval=self.data_loader.data_length[\"train\"]).start()\n",
    "            batch_i = 0\n",
    "\n",
    "            discriminator_losses = []\n",
    "            generator_loss_max_in_epoch = 0\n",
    "            generator_loss_min_in_epoch = 1000\n",
    "\n",
    "            # shffle data maybe\n",
    "            if epoch % epoch_shuffle_term == 0:\n",
    "                np.random.shuffle(self.loaded_data_index[\"train\"])\n",
    "\n",
    "            if self.discriminator_acc_previous < 0.5:\n",
    "                discriminator_learning = True\n",
    "                print(\"discriminator_learning is True\")\n",
    "            else:\n",
    "                discriminator_learning = False\n",
    "                print(\"discriminator_learning is False\")\n",
    "\n",
    "            while batch_i + self.batch_size <= self.data_loader.data_length[\"train\"]:\n",
    "                bar.update(batch_i)\n",
    "\n",
    "                batch_index = self.loaded_data_index[\"train\"][batch_i: batch_i +\n",
    "                                                              self.batch_size]\n",
    "\n",
    "                original_img, masked_img = self.batch_queue_manager.get_batch(\n",
    "                    data_mode=\"train\")\n",
    "                model_masked_img = self.generator.predict_on_batch(\n",
    "                    original_img)\n",
    "\n",
    "                # forTest\n",
    "                self.masked_img = masked_img\n",
    "                self.original_img = original_img\n",
    "                self.model_masked_img = model_masked_img\n",
    "                self.valid_path = valid_patch\n",
    "                self.fake_patch = fake_patch\n",
    "\n",
    "                generator_current_learning_rate = learning_rate_scheduler(\n",
    "                    self.generator_learning_rate,\n",
    "                    epoch,\n",
    "                )\n",
    "                discriminator_current_learning_rate = learning_rate_scheduler(\n",
    "                    self.discriminator_learning_rate,\n",
    "                    epoch,\n",
    "                ) * (1 - self.discriminator_acc_previous)\n",
    "                keras_backend.set_value(\n",
    "                    self.discriminator.optimizer.learning_rate,\n",
    "                    discriminator_current_learning_rate,\n",
    "                )\n",
    "                keras_backend.set_value(\n",
    "                    self.combined.optimizer.learning_rate, generator_current_learning_rate\n",
    "                )\n",
    "                keras_backend.set_value(\n",
    "                    self.discriminator_loss_ratio, keras_backend.variable(\n",
    "                        0.5) + 4.5 * (self.discriminator_acc_previous)\n",
    "                )\n",
    "                keras_backend.set_value(\n",
    "                    self.f1_loss_ratio, keras_backend.variable(\n",
    "                        100) - 4.5 * (self.discriminator_acc_previous)\n",
    "                )\n",
    "                # ---------------------\n",
    "                #  Train Discriminator\n",
    "                # ---------------------\n",
    "                # Train Discriminator for valid image if it failed to detect fake image\n",
    "\n",
    "                if discriminator_learning:\n",
    "                    self.discriminator.train_on_batch(\n",
    "                        [original_img, masked_img], valid_patch)\n",
    "\n",
    "                batch_discriminator_acc_previous = np.mean(\n",
    "                    self.discriminator_acces_previous[batch_index])\n",
    "\n",
    "                # -----------------\n",
    "                #  Train Generator\n",
    "                # -----------------\n",
    "\n",
    "                # Train the generators\n",
    "\n",
    "                generator_loss = self.combined.train_on_batch(\n",
    "                    [original_img, masked_img],\n",
    "                    [valid_patch, masked_img],\n",
    "                )\n",
    "                # train discriminator for fake image if it failed to detect fake image\n",
    "                if (batch_discriminator_acc_previous <= 0.5 or epoch == 0) and discriminator_learning:\n",
    "                    discriminator_loss = self.discriminator.train_on_batch(\n",
    "                        [original_img, model_masked_img], fake_patch)\n",
    "                else:\n",
    "                    discriminator_loss = self.discriminator.test_on_batch(\n",
    "                        [original_img, model_masked_img], fake_patch)\n",
    "\n",
    "                self.discriminator_acces[batch_index] = discriminator_loss[1]\n",
    "                self.generator_losses[batch_index] = generator_loss[0]\n",
    "                elapsed_time = datetime.now() - start_time\n",
    "                if batch_i % sample_interval == 0:\n",
    "                    # Plot the progress\n",
    "                    print(\n",
    "                        \"[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %f] time: %s\"\n",
    "                        % (\n",
    "                            epoch,\n",
    "                            epochs,\n",
    "                            batch_i,\n",
    "                            self.data_loader.data_length[\"train\"],\n",
    "                            discriminator_loss[0],\n",
    "                            100 * discriminator_loss[1],\n",
    "                            generator_loss[0],\n",
    "                            elapsed_time,\n",
    "                        )\n",
    "                    )\n",
    "\n",
    "                # If at save interval => save generated image samples\n",
    "                if batch_i % sample_interval == 0:\n",
    "                    self.image_drawer.sample_images(\n",
    "                        self.generator, epoch, batch_i)\n",
    "\n",
    "                discriminator_losses.append(discriminator_loss[0])\n",
    "                # loss 가 가장 높은 이미지를 저장 및 max_in_epoch 갱신\n",
    "                if generator_loss[0] > generator_loss_max_in_epoch:\n",
    "                    generator_loss_max_in_epoch = generator_loss[0]\n",
    "                    if self.draw_images or epoch % 25 == 0:\n",
    "                        model_masked_img = self.generator.predict_on_batch(\n",
    "                            original_img)\n",
    "                        self.image_drawer.draw_worst_and_best(\n",
    "                            original_img,\n",
    "                            model_masked_img,\n",
    "                            masked_img,\n",
    "                            epoch,\n",
    "                            worst=True,\n",
    "                        )\n",
    "\n",
    "                # loss 가 가장 낮은 이미지를 저장 및 max_in_epoch 갱신\n",
    "                if generator_loss_min_in_epoch > generator_loss[0]:\n",
    "                    generator_loss_min_in_epoch = generator_loss[0]\n",
    "                    if self.draw_images or epoch % 25 == 0:\n",
    "                        model_masked_img = self.generator.predict_on_batch(\n",
    "                            original_img)\n",
    "                        self.image_drawer.draw_worst_and_best(\n",
    "                            original_img,\n",
    "                            model_masked_img,\n",
    "                            masked_img,\n",
    "                            epoch,\n",
    "                            worst=False,\n",
    "                        )\n",
    "\n",
    "                # 한 배치 끝\n",
    "                batch_i += self.batch_size\n",
    "            # training batch 사이클 끝\n",
    "            print(f\"discriminator_acces : {np.mean(self.discriminator_acces)}\")\n",
    "            print(\n",
    "                f\"Mean generator_loss : {np.mean(self.generator_losses)}\")\n",
    "            print(f\"Max generator_loss : {np.max(self.generator_losses)}\")\n",
    "            print(f\"Min generator_loss : {np.min(self.generator_losses)}\")\n",
    "            print(\n",
    "                f\"generator loss decrease : {self.generator_loss_min - np.mean(self.generator_losses)}\"\n",
    "            )\n",
    "            print(\n",
    "                f\"generator loss decrease ratio : ({np.mean(self.generator_losses) / self.generator_loss_min})\"\n",
    "            )\n",
    "            print(\n",
    "                f\"Max generator loss decrease : {self.generator_loss_max_previous - np.max(self.generator_losses)}\"\n",
    "            )\n",
    "            print(\n",
    "                f\"current lowest generator loss : {self.generator_loss_min}\")\n",
    "            print(\n",
    "                f\"current Learning_rate = {generator_current_learning_rate}\")\n",
    "\n",
    "            if np.mean(self.generator_losses) / self.generator_loss_min < 1.1:\n",
    "                if self.generator_loss_min > np.mean(self.generator_losses):\n",
    "                    self.generator_loss_min = np.mean(self.generator_losses)\n",
    "                    self.generator_loss_max_min = generator_loss_max_in_epoch\n",
    "                    self.generator_loss_min_min = generator_loss_min_in_epoch\n",
    "                    self.weight_save_stack = True\n",
    "                    self.save_study_info()\n",
    "                    print(\"save weights\")\n",
    "\n",
    "                train_f1_loss_list = []\n",
    "                train_f1_score_list = []\n",
    "                for index in range(0, self.data_loader.data_length[\"train\"], self.batch_size):\n",
    "\n",
    "                    train_source_img, train_masked_img = self.batch_queue_manager.get_batch(\n",
    "                        data_mode=\"train\")\n",
    "\n",
    "                    train_model_masked_img = self.generator.predict_on_batch(\n",
    "                        train_source_img)\n",
    "\n",
    "                    train_f1_loss = f1_loss_for_training(\n",
    "                        train_masked_img, train_model_masked_img)\n",
    "                    train_f1_score = f1_score(\n",
    "                        train_masked_img, train_model_masked_img)\n",
    "                    train_f1_loss_list.append(train_f1_loss)\n",
    "                    train_f1_score_list.append(train_f1_score)\n",
    "\n",
    "                print(\n",
    "                    f\"train_f1_loss : {np.mean(train_f1_loss_list) * self.f1_loss_ratio}\")\n",
    "                print(f\"train_f1_score : {1 - np.mean(train_f1_loss_list)}\")\n",
    "                print(\n",
    "                    f\"train_f1_rounded_score : {np.mean(train_f1_score_list)}\")\n",
    "\n",
    "            if np.mean(self.generator_losses) / self.generator_loss_min < 1.1:\n",
    "                if self.generator_loss_min > np.mean(self.generator_losses):\n",
    "                    self.generator_loss_min = np.mean(self.generator_losses)\n",
    "                    self.generator_loss_max_min = generator_loss_max_in_epoch\n",
    "                    self.generator_loss_min_min = generator_loss_min_in_epoch\n",
    "                    self.weight_save_stack = True\n",
    "                    self.save_study_info()\n",
    "                    print(\"save weights\")\n",
    "\n",
    "                valid_f1_loss_list = []\n",
    "                valid_f1_score_list = []\n",
    "                for index in range(0, self.data_loader.data_length[\"valid\"], self.batch_size):\n",
    "\n",
    "                    valid_source_img, valid_masked_img = self.batch_queue_manager.get_batch(\n",
    "                        data_mode=\"valid\")\n",
    "\n",
    "                    valid_model_masked_img = self.generator.predict_on_batch(\n",
    "                        valid_source_img)\n",
    "\n",
    "                    valid_f1_loss = f1_loss_for_training(\n",
    "                        valid_masked_img, valid_model_masked_img)\n",
    "                    valid_f1_score = f1_score(\n",
    "                        valid_masked_img, valid_model_masked_img)\n",
    "                    valid_f1_loss_list.append(valid_f1_loss)\n",
    "                    valid_f1_score_list.append(valid_f1_score)\n",
    "\n",
    "                print(\n",
    "                    f\"valid_f1_loss : {np.mean(valid_f1_loss_list) * self.f1_loss_ratio}\")\n",
    "                print(f\"valid_f1_score : {1 - np.mean(valid_f1_loss_list)}\")\n",
    "                print(\n",
    "                    f\"valid_f1_rounded_score : {np.mean(valid_f1_score_list)}\")\n",
    "            else:\n",
    "                print(\"loss decrease.\")\n",
    "                self.load_best_weights()\n",
    "\n",
    "            # previous generator_loss 갱신\n",
    "            self.generator_loss_previous = np.mean(self.generator_losses)\n",
    "            self.generator_loss_max_previous = generator_loss_max_in_epoch\n",
    "\n",
    "            if epoch >= 10 and self.weight_save_stack:\n",
    "                copy(\n",
    "                    \"generator.h5\",\n",
    "                    \"./generator_weights/generator_\"\n",
    "                    + str(round(self.generator_loss_min, 5))\n",
    "                    + \"_\"\n",
    "                    + str(round(self.generator_loss_max_min, 5))\n",
    "                    + \".h5\",\n",
    "                )\n",
    "                self.weight_save_stack = False\n",
    "\n",
    "            self.discriminator_acc_previous = np.mean(self.discriminator_acces)\n",
    "            self.discriminator_acces_previous = self.discriminator_acces.copy()\n",
    "            self.generator_losses_previous = self.generator_losses.copy()\n",
    "            # TBD: add epoch bigger than history length\n",
    "            self.history[\"generator_loss\"].append(\n",
    "                np.mean(self.generator_losses))\n",
    "            self.history[\"f1_loss_train\"].append(\n",
    "                np.mean(train_f1_loss_list))\n",
    "            self.history[\"f1_score_train\"].append(\n",
    "                np.mean(train_f1_score_list))\n",
    "            self.history[\"f1_loss_valid\"].append(\n",
    "                np.mean(valid_f1_loss_list))\n",
    "            self.history[\"f1_score_valid\"].append(\n",
    "                np.mean(valid_f1_score_list))\n",
    "\n",
    "    def get_info_folderPath(self):\n",
    "        return (\n",
    "            str(round(self.generator_loss_min, 5))\n",
    "            + \"_\"\n",
    "            + str(round(self.generator_loss_max_min, 5))\n",
    "        )\n",
    "\n",
    "    def save_study_info(self, path=None):\n",
    "\n",
    "        if path is None:\n",
    "            path = self.temp_weights_path\n",
    "\n",
    "        generator_weigth_path = os.path.join(path, \"generator.h5\")\n",
    "        discriminator_weigth_path = os.path.join(path, \"discriminator.h5\")\n",
    "        combined_weigth_path = os.path.join(path, \"combined.h5\")\n",
    "\n",
    "        self.generator.save_weights(generator_weigth_path)\n",
    "        self.discriminator.save_weights(discriminator_weigth_path)\n",
    "        self.combined.save_weights(combined_weigth_path)\n",
    "\n",
    "        study_info = {}\n",
    "        study_info[\"start_epoch\"] = self.start_epoch\n",
    "        study_info[\"train_loaded_data_index\"] = self.loaded_data_index[\"train\"]\n",
    "        study_info[\"generator_loss_min\"] = self.generator_loss_min\n",
    "        study_info[\"generator_loss_max_min\"] = self.generator_loss_max_min\n",
    "        study_info[\"generator_loss_min_min\"] = self.generator_loss_min_min\n",
    "        study_info[\"generator_losses_previous\"] = self.generator_losses_previous\n",
    "        study_info[\"discriminator_acces\"] = self.discriminator_acces\n",
    "        study_info[\"history\"] = self.history\n",
    "        file = open(path + \"/study_info.pkl\", \"wb\")\n",
    "        dump(study_info, file)\n",
    "        file.close()\n",
    "\n",
    "    def load_study_info(self):\n",
    "\n",
    "        self.generator.load_weights(\"generator.h5\")\n",
    "        self.discriminator.load_weights(\"discriminator.h5\")\n",
    "        self.combined.load_weights(\"combined.h5\")\n",
    "\n",
    "        if os.path.isfile(\"study_info.pkl\"):\n",
    "            file = open(\"study_info.pkl\", \"rb\")\n",
    "            study_info = load(file)\n",
    "            file.close()\n",
    "            self.start_epoch = study_info[\"start_epoch\"]\n",
    "            self.loaded_data_index[\"train\"] = study_info[\"train_loaded_data_index\"]\n",
    "            self.generator_loss_min = study_info[\"generator_loss_min\"]\n",
    "            self.generator_loss_max_min = study_info[\"generator_loss_max_min\"]\n",
    "            self.generator_loss_min_min = study_info[\"generator_loss_min_min\"]\n",
    "            self.generator_losses_previous = study_info[\"generator_losses_previous\"]\n",
    "            self.discriminator_acces = study_info[\"discriminator_acces\"]\n",
    "            self.history = study_info[\"history\"]\n",
    "        else:\n",
    "            print(\"No info pkl file!\")\n",
    "\n",
    "    def load_best_weights(self):\n",
    "        self.generator.load_weights(self.temp_weights_path + \"/generator.h5\")\n",
    "        self.discriminator.load_weights(\n",
    "            self.temp_weights_path + \"/discriminator.h5\")\n",
    "        self.combined.load_weights(self.temp_weights_path + \"/combined.h5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "generator_lr = 1e-3\n",
    "discriminator_lr = 5e-4\n",
    "batch_size = 4\n",
    "g_lr = generator_lr * batch_size\n",
    "d_lr = discriminator_lr * batch_size\n",
    "gan = Pix2PixSegmentation(generator_power=4, discriminator_power=4, \n",
    "                          generator_learning_rate=g_lr, discriminator_learning_rate=d_lr,\n",
    "                          on_memory=False, code_test=False, draw_images=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_learning is True\n",
      "[Epoch 0/325] [Batch 0/6200] [D loss: 0.696086, acc:   0%] [G loss: 96.435379] time: 0:00:34.757660\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 0/325] [Batch 3100/6200] [D loss: 0.693173, acc:   0%] [G loss: 74.128418] time: 0:06:55.364024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 80.1045150756836\n",
      "Max generator_loss : 98.5821762084961\n",
      "Min generator_loss : 46.660160064697266\n",
      "generator loss decrease : 419.8954849243164\n",
      "generator loss decrease ratio : (0.1602090301513672)\n",
      "Max generator loss decrease : 901.4178237915039\n",
      "current lowest generator loss : 500\n",
      "current Learning_rate = 0.0004\n",
      "save weights\n",
      "train_f1_loss : 78.04361724853516\n",
      "train_f1_score : 0.2015998363494873\n",
      "train_f1_rounded_score : 0.29598209261894226\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 80.41770935058594\n",
      "valid_f1_score : 0.177312433719635\n",
      "valid_f1_rounded_score : 0.31955522298812866\n",
      "discriminator_learning is True\n",
      "[Epoch 1/325] [Batch 0/6200] [D loss: 0.693224, acc:   0%] [G loss: 81.904007] time: 0:14:56.811557\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 1/325] [Batch 3100/6200] [D loss: 0.693108, acc: 100%] [G loss: 69.347443] time: 0:21:14.060676\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.6967741935483871\n",
      "Mean generator_loss : 76.36406707763672\n",
      "Max generator_loss : 98.7520980834961\n",
      "Min generator_loss : 40.23741149902344\n",
      "generator loss decrease : 3.740447998046875\n",
      "generator loss decrease ratio : (0.9533054232597351)\n",
      "Max generator loss decrease : -0.169921875\n",
      "current lowest generator loss : 80.1045150756836\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 75.93363952636719\n",
      "train_f1_score : 0.2406635880470276\n",
      "train_f1_rounded_score : 0.5509488582611084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 77.89224243164062\n",
      "valid_f1_score : 0.221077561378479\n",
      "valid_f1_rounded_score : 0.5725177526473999\n",
      "discriminator_learning is True\n",
      "[Epoch 2/325] [Batch 0/6200] [D loss: 0.693106, acc: 100%] [G loss: 79.405128] time: 0:29:12.547798\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 2/325] [Batch 3100/6200] [D loss: 0.745805, acc:   0%] [G loss: 66.031563] time: 0:35:03.006687\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.30709677419354836\n",
      "Mean generator_loss : 73.77129364013672\n",
      "Max generator_loss : 97.82752990722656\n",
      "Min generator_loss : 37.063636779785156\n",
      "generator loss decrease : 2.5927734375\n",
      "generator loss decrease ratio : (0.9660472273826599)\n",
      "Max generator loss decrease : 0.9245681762695312\n",
      "current lowest generator loss : 76.36406707763672\n",
      "current Learning_rate = 0.0012\n",
      "save weights\n",
      "train_f1_loss : 69.96316528320312\n",
      "train_f1_score : 0.27772146463394165\n",
      "train_f1_rounded_score : 0.524686872959137\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 70.07485961914062\n",
      "valid_f1_score : 0.27656829357147217\n",
      "valid_f1_rounded_score : 0.5483386516571045\n",
      "discriminator_learning is True\n",
      "[Epoch 3/325] [Batch 0/6200] [D loss: 0.838564, acc:   0%] [G loss: 76.488289] time: 0:42:01.854749\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 3/325] [Batch 3100/6200] [D loss: 0.925551, acc:   0%] [G loss: 60.979778] time: 0:47:40.503968\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 70.14008331298828\n",
      "Max generator_loss : 97.8425064086914\n",
      "Min generator_loss : 33.297481536865234\n",
      "generator loss decrease : 3.6312103271484375\n",
      "generator loss decrease ratio : (0.9507774710655212)\n",
      "Max generator loss decrease : -0.01497650146484375\n",
      "current lowest generator loss : 73.77129364013672\n",
      "current Learning_rate = 0.0016\n",
      "save weights\n",
      "train_f1_loss : 68.44963073730469\n",
      "train_f1_score : 0.3059118390083313\n",
      "train_f1_rounded_score : 0.5572127103805542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 69.17716217041016\n",
      "valid_f1_score : 0.2985345721244812\n",
      "valid_f1_rounded_score : 0.5716159343719482\n",
      "discriminator_learning is True\n",
      "[Epoch 4/325] [Batch 0/6200] [D loss: 0.828772, acc:   0%] [G loss: 71.096924] time: 0:55:36.362018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 4/325] [Batch 3100/6200] [D loss: 0.748041, acc:   0%] [G loss: 52.802464] time: 1:01:51.687454\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 64.76318359375\n",
      "Max generator_loss : 97.97195434570312\n",
      "Min generator_loss : 25.980958938598633\n",
      "generator loss decrease : 5.376899719238281\n",
      "generator loss decrease ratio : (0.9233405590057373)\n",
      "Max generator loss decrease : -0.12944793701171875\n",
      "current lowest generator loss : 70.14008331298828\n",
      "current Learning_rate = 0.002\n",
      "save weights\n",
      "train_f1_loss : 63.033058166503906\n",
      "train_f1_score : 0.36966943740844727\n",
      "train_f1_rounded_score : 0.5296995043754578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 64.0001449584961\n",
      "valid_f1_score : 0.35999858379364014\n",
      "valid_f1_rounded_score : 0.5396730303764343\n",
      "discriminator_learning is True\n",
      "[Epoch 5/325] [Batch 0/6200] [D loss: 0.715920, acc:   0%] [G loss: 65.734962] time: 1:09:47.879737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 5/325] [Batch 3100/6200] [D loss: 0.701171, acc:   0%] [G loss: 47.310360] time: 1:16:03.327623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 58.292728424072266\n",
      "Max generator_loss : 96.68206787109375\n",
      "Min generator_loss : 20.492393493652344\n",
      "generator loss decrease : 6.470455169677734\n",
      "generator loss decrease ratio : (0.9000905156135559)\n",
      "Max generator loss decrease : 1.289886474609375\n",
      "current lowest generator loss : 64.76318359375\n",
      "current Learning_rate = 0.0024\n",
      "save weights\n",
      "train_f1_loss : 64.6059341430664\n",
      "train_f1_score : 0.3539406657218933\n",
      "train_f1_rounded_score : 0.45879897475242615\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 62.87589645385742\n",
      "valid_f1_score : 0.37124103307724\n",
      "valid_f1_rounded_score : 0.4748849868774414\n",
      "discriminator_learning is True\n",
      "[Epoch 6/325] [Batch 0/6200] [D loss: 0.696023, acc:   0%] [G loss: 60.928352] time: 1:23:58.258420\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 6/325] [Batch 3100/6200] [D loss: 0.694037, acc:   0%] [G loss: 41.000912] time: 1:30:13.275776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 51.34254455566406\n",
      "Max generator_loss : 91.82518768310547\n",
      "Min generator_loss : 19.40215301513672\n",
      "generator loss decrease : 6.950183868408203\n",
      "generator loss decrease ratio : (0.8807709813117981)\n",
      "Max generator loss decrease : 4.856880187988281\n",
      "current lowest generator loss : 58.292728424072266\n",
      "current Learning_rate = 0.0028\n",
      "save weights\n",
      "train_f1_loss : 47.299354553222656\n",
      "train_f1_score : 0.5270064473152161\n",
      "train_f1_rounded_score : 0.5644339323043823\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 48.89937210083008\n",
      "valid_f1_score : 0.5110062658786774\n",
      "valid_f1_rounded_score : 0.5483605861663818\n",
      "discriminator_learning is True\n",
      "[Epoch 7/325] [Batch 0/6200] [D loss: 0.693462, acc:   0%] [G loss: 50.840969] time: 1:38:10.364780\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 7/325] [Batch 3100/6200] [D loss: 0.693269, acc:   0%] [G loss: 39.647240] time: 1:44:27.458749\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.46\n",
      "Mean generator_loss : 45.251712799072266\n",
      "Max generator_loss : 90.436279296875\n",
      "Min generator_loss : 14.879942893981934\n",
      "generator loss decrease : 6.090831756591797\n",
      "generator loss decrease ratio : (0.8813686966896057)\n",
      "Max generator loss decrease : 1.3889083862304688\n",
      "current lowest generator loss : 51.34254455566406\n",
      "current Learning_rate = 0.0032\n",
      "save weights\n",
      "train_f1_loss : 43.62351608276367\n",
      "train_f1_score : 0.5637648403644562\n",
      "train_f1_rounded_score : 0.5729163885116577\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 39.559478759765625\n",
      "valid_f1_score : 0.6044051945209503\n",
      "valid_f1_rounded_score : 0.6153514981269836\n",
      "discriminator_learning is True\n",
      "[Epoch 8/325] [Batch 0/6200] [D loss: 0.692979, acc: 100%] [G loss: 48.246071] time: 1:52:23.160176\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 8/325] [Batch 3100/6200] [D loss: 0.693002, acc: 100%] [G loss: 40.060158] time: 1:58:36.783836\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.5432258064516129\n",
      "Mean generator_loss : 42.826873779296875\n",
      "Max generator_loss : 88.63002014160156\n",
      "Min generator_loss : 14.811006546020508\n",
      "generator loss decrease : 2.4248390197753906\n",
      "generator loss decrease ratio : (0.9464144110679626)\n",
      "Max generator loss decrease : 1.8062591552734375\n",
      "current lowest generator loss : 45.251712799072266\n",
      "current Learning_rate = 0.0036000000000000003\n",
      "save weights\n",
      "train_f1_loss : 37.7664680480957\n",
      "train_f1_score : 0.6143524348735809\n",
      "train_f1_rounded_score : 0.6188241243362427\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 36.874019622802734\n",
      "valid_f1_score : 0.6234655380249023\n",
      "valid_f1_rounded_score : 0.6293125152587891\n",
      "discriminator_learning is True\n",
      "[Epoch 9/325] [Batch 0/6200] [D loss: 1.259570, acc:   0%] [G loss: 44.695080] time: 2:05:41.126334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 9/325] [Batch 3100/6200] [D loss: 1.649020, acc:   0%] [G loss: 38.371384] time: 2:11:01.155190\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 40.371177673339844\n",
      "Max generator_loss : 84.03255462646484\n",
      "Min generator_loss : 15.000516891479492\n",
      "generator loss decrease : 2.4556961059570312\n",
      "generator loss decrease ratio : (0.9426599144935608)\n",
      "Max generator loss decrease : 4.597465515136719\n",
      "current lowest generator loss : 42.826873779296875\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 47.40535354614258\n",
      "train_f1_score : 0.5140677392482758\n",
      "train_f1_rounded_score : 0.5221951007843018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 48.49394226074219\n",
      "valid_f1_score : 0.5029090940952301\n",
      "valid_f1_rounded_score : 0.5112179517745972\n",
      "discriminator_learning is True\n",
      "[Epoch 10/325] [Batch 0/6200] [D loss: 1.070548, acc:   0%] [G loss: 43.766785] time: 2:18:48.450962\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 10/325] [Batch 3100/6200] [D loss: 0.753270, acc:   0%] [G loss: 43.805759] time: 2:25:00.981950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 39.52012634277344\n",
      "Max generator_loss : 84.11965942382812\n",
      "Min generator_loss : 15.081275939941406\n",
      "generator loss decrease : 0.8510513305664062\n",
      "generator loss decrease ratio : (0.9789193272590637)\n",
      "Max generator loss decrease : -0.08710479736328125\n",
      "current lowest generator loss : 40.371177673339844\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 42.11882019042969\n",
      "train_f1_score : 0.5788117945194244\n",
      "train_f1_rounded_score : 0.5848178267478943\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 39.64842987060547\n",
      "valid_f1_score : 0.6035157144069672\n",
      "valid_f1_rounded_score : 0.6100401878356934\n",
      "discriminator_learning is True\n",
      "[Epoch 11/325] [Batch 0/6200] [D loss: 0.703580, acc:   0%] [G loss: 45.336407] time: 2:32:52.916613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 11/325] [Batch 3100/6200] [D loss: 0.695031, acc:   0%] [G loss: 31.719860] time: 2:39:06.173367\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 39.00513458251953\n",
      "Max generator_loss : 84.43656158447266\n",
      "Min generator_loss : 14.916129112243652\n",
      "generator loss decrease : 0.5149917602539062\n",
      "generator loss decrease ratio : (0.9869688749313354)\n",
      "Max generator loss decrease : -0.31690216064453125\n",
      "current lowest generator loss : 39.52012634277344\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 43.800418853759766\n",
      "train_f1_score : 0.561995804309845\n",
      "train_f1_rounded_score : 0.5677030086517334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 45.808624267578125\n",
      "valid_f1_score : 0.5419137477874756\n",
      "valid_f1_rounded_score : 0.547684907913208\n",
      "discriminator_learning is True\n",
      "[Epoch 12/325] [Batch 0/6200] [D loss: 0.693533, acc:   0%] [G loss: 45.283264] time: 2:46:58.653650\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 12/325] [Batch 3100/6200] [D loss: 0.692916, acc: 100%] [G loss: 38.740192] time: 2:53:11.516238\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.5496774193548387\n",
      "Mean generator_loss : 38.383460998535156\n",
      "Max generator_loss : 83.28636932373047\n",
      "Min generator_loss : 13.874104499816895\n",
      "generator loss decrease : 0.621673583984375\n",
      "generator loss decrease ratio : (0.9840617775917053)\n",
      "Max generator loss decrease : 1.1501922607421875\n",
      "current lowest generator loss : 39.00513458251953\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 38.96549606323242\n",
      "train_f1_score : 0.6103450357913971\n",
      "train_f1_rounded_score : 0.6134210228919983\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 40.31135559082031\n",
      "valid_f1_score : 0.5968864262104034\n",
      "valid_f1_rounded_score : 0.6004001498222351\n",
      "discriminator_learning is True\n",
      "[Epoch 13/325] [Batch 0/6200] [D loss: 0.692947, acc: 100%] [G loss: 43.741280] time: 3:01:03.506901\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 13/325] [Batch 3100/6200] [D loss: 0.764139, acc:   0%] [G loss: 38.776878] time: 3:07:11.470181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.4541935483870968\n",
      "Mean generator_loss : 38.724212646484375\n",
      "Max generator_loss : 82.02825927734375\n",
      "Min generator_loss : 14.35615062713623\n",
      "generator loss decrease : -0.34075164794921875\n",
      "generator loss decrease ratio : (1.0088775157928467)\n",
      "Max generator loss decrease : 1.2581100463867188\n",
      "current lowest generator loss : 38.383460998535156\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 39.30451583862305\n",
      "train_f1_score : 0.5969861149787903\n",
      "train_f1_rounded_score : 0.6005509495735168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 40.13231658935547\n",
      "valid_f1_score : 0.5884981155395508\n",
      "valid_f1_rounded_score : 0.5923851728439331\n",
      "discriminator_learning is True\n",
      "[Epoch 14/325] [Batch 0/6200] [D loss: 1.296950, acc:   0%] [G loss: 42.391335] time: 3:14:10.080679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 14/325] [Batch 3100/6200] [D loss: 1.555124, acc:   0%] [G loss: 33.800102] time: 3:19:35.236576\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 37.95407485961914\n",
      "Max generator_loss : 83.03262329101562\n",
      "Min generator_loss : 14.927680015563965\n",
      "generator loss decrease : 0.4293861389160156\n",
      "generator loss decrease ratio : (0.9888132214546204)\n",
      "Max generator loss decrease : -1.004364013671875\n",
      "current lowest generator loss : 38.383460998535156\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 43.427913665771484\n",
      "train_f1_score : 0.5566595792770386\n",
      "train_f1_rounded_score : 0.5627040266990662\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 43.471675872802734\n",
      "valid_f1_score : 0.5562128126621246\n",
      "valid_f1_rounded_score : 0.5617725849151611\n",
      "discriminator_learning is True\n",
      "[Epoch 15/325] [Batch 0/6200] [D loss: 0.957874, acc:   0%] [G loss: 42.291332] time: 3:27:27.263407\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 15/325] [Batch 3100/6200] [D loss: 0.736056, acc:   0%] [G loss: 34.632275] time: 3:33:40.051864\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 37.33473205566406\n",
      "Max generator_loss : 83.15276336669922\n",
      "Min generator_loss : 13.477701187133789\n",
      "generator loss decrease : 0.6193428039550781\n",
      "generator loss decrease ratio : (0.9836817979812622)\n",
      "Max generator loss decrease : -0.12014007568359375\n",
      "current lowest generator loss : 37.95407485961914\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 37.8902473449707\n",
      "train_f1_score : 0.6210975348949432\n",
      "train_f1_rounded_score : 0.6215329766273499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.76277542114258\n",
      "valid_f1_score : 0.6523722410202026\n",
      "valid_f1_rounded_score : 0.6532456278800964\n",
      "discriminator_learning is True\n",
      "[Epoch 16/325] [Batch 0/6200] [D loss: 0.700633, acc:   0%] [G loss: 42.199085] time: 3:41:32.071433\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 16/325] [Batch 3100/6200] [D loss: 0.694515, acc:   0%] [G loss: 35.886139] time: 3:47:44.887750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 37.178802490234375\n",
      "Max generator_loss : 83.6144790649414\n",
      "Min generator_loss : 13.823569297790527\n",
      "generator loss decrease : 0.1559295654296875\n",
      "generator loss decrease ratio : (0.9958234429359436)\n",
      "Max generator loss decrease : -0.4617156982421875\n",
      "current lowest generator loss : 37.33473205566406\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 43.04697036743164\n",
      "train_f1_score : 0.5695302784442902\n",
      "train_f1_rounded_score : 0.5692545771598816\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 37.12748336791992\n",
      "valid_f1_score : 0.6287251710891724\n",
      "valid_f1_rounded_score : 0.6288843750953674\n",
      "discriminator_learning is True\n",
      "[Epoch 17/325] [Batch 0/6200] [D loss: 0.693442, acc:   0%] [G loss: 45.168522] time: 3:55:37.714950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 17/325] [Batch 3100/6200] [D loss: 0.692922, acc: 100%] [G loss: 36.484329] time: 4:01:50.283900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.6432258064516129\n",
      "Mean generator_loss : 37.16531753540039\n",
      "Max generator_loss : 88.2412109375\n",
      "Min generator_loss : 14.08203125\n",
      "generator loss decrease : 0.013484954833984375\n",
      "generator loss decrease ratio : (0.9996373057365417)\n",
      "Max generator loss decrease : -4.626731872558594\n",
      "current lowest generator loss : 37.178802490234375\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 36.16762161254883\n",
      "train_f1_score : 0.6383237838745117\n",
      "train_f1_rounded_score : 0.6399487257003784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 36.29045486450195\n",
      "valid_f1_score : 0.6370954513549805\n",
      "valid_f1_rounded_score : 0.6391713619232178\n",
      "discriminator_learning is True\n",
      "[Epoch 18/325] [Batch 0/6200] [D loss: 0.692954, acc: 100%] [G loss: 42.247780] time: 4:09:42.211742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 18/325] [Batch 3100/6200] [D loss: 0.850993, acc:   0%] [G loss: 33.610825] time: 4:15:39.565225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.36129032258064514\n",
      "Mean generator_loss : 38.011104583740234\n",
      "Max generator_loss : 81.04379272460938\n",
      "Min generator_loss : 14.578478813171387\n",
      "generator loss decrease : -0.8457870483398438\n",
      "generator loss decrease ratio : (1.0227574110031128)\n",
      "Max generator loss decrease : 7.197418212890625\n",
      "current lowest generator loss : 37.16531753540039\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 34.56146240234375\n",
      "train_f1_score : 0.6440832912921906\n",
      "train_f1_rounded_score : 0.6456901431083679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 35.429718017578125\n",
      "valid_f1_score : 0.6351419389247894\n",
      "valid_f1_rounded_score : 0.6372237205505371\n",
      "discriminator_learning is True\n",
      "[Epoch 19/325] [Batch 0/6200] [D loss: 1.238605, acc:   0%] [G loss: 39.923996] time: 4:22:38.191962\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 19/325] [Batch 3100/6200] [D loss: 1.261832, acc:   0%] [G loss: 31.834349] time: 4:28:12.656290\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 36.16638946533203\n",
      "Max generator_loss : 80.79341888427734\n",
      "Min generator_loss : 13.90713882446289\n",
      "generator loss decrease : 0.9989280700683594\n",
      "generator loss decrease ratio : (0.9731220602989197)\n",
      "Max generator loss decrease : 0.25037384033203125\n",
      "current lowest generator loss : 37.16531753540039\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 36.89646530151367\n",
      "train_f1_score : 0.6249375343322754\n",
      "train_f1_rounded_score : 0.626936137676239\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 37.90626525878906\n",
      "valid_f1_score : 0.6146726608276367\n",
      "valid_f1_rounded_score : 0.6167226433753967\n",
      "discriminator_learning is True\n",
      "[Epoch 20/325] [Batch 0/6200] [D loss: 0.848203, acc:   0%] [G loss: 40.786598] time: 4:36:04.186282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 20/325] [Batch 3100/6200] [D loss: 0.718741, acc:   0%] [G loss: 32.962872] time: 4:42:17.007384\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 36.312660217285156\n",
      "Max generator_loss : 82.67060852050781\n",
      "Min generator_loss : 13.7413911819458\n",
      "generator loss decrease : -0.146270751953125\n",
      "generator loss decrease ratio : (1.0040444135665894)\n",
      "Max generator loss decrease : -1.8771896362304688\n",
      "current lowest generator loss : 36.16638946533203\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 38.9951171875\n",
      "train_f1_score : 0.6100488305091858\n",
      "train_f1_rounded_score : 0.610956072807312\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 40.57874298095703\n",
      "valid_f1_score : 0.5942125618457794\n",
      "valid_f1_rounded_score : 0.595727801322937\n",
      "discriminator_learning is True\n",
      "[Epoch 21/325] [Batch 0/6200] [D loss: 0.697648, acc:   0%] [G loss: 40.171837] time: 4:50:08.620341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 21/325] [Batch 3100/6200] [D loss: 0.693991, acc:   0%] [G loss: 31.977863] time: 4:56:21.691616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 36.03045654296875\n",
      "Max generator_loss : 78.6500473022461\n",
      "Min generator_loss : 14.546175003051758\n",
      "generator loss decrease : 0.13593292236328125\n",
      "generator loss decrease ratio : (0.9962414503097534)\n",
      "Max generator loss decrease : 4.020561218261719\n",
      "current lowest generator loss : 36.16638946533203\n",
      "current Learning_rate = 0.004\n",
      "save weights\n",
      "train_f1_loss : 39.45551681518555\n",
      "train_f1_score : 0.6054448485374451\n",
      "train_f1_rounded_score : 0.6054269671440125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 38.75272750854492\n",
      "valid_f1_score : 0.6124727427959442\n",
      "valid_f1_rounded_score : 0.6127474308013916\n",
      "discriminator_learning is True\n",
      "[Epoch 22/325] [Batch 0/6200] [D loss: 0.693350, acc:   0%] [G loss: 40.932232] time: 5:04:13.928122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 22/325] [Batch 3100/6200] [D loss: 0.692933, acc: 100%] [G loss: 34.635178] time: 5:10:26.481428\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.7909677419354839\n",
      "Mean generator_loss : 36.22970962524414\n",
      "Max generator_loss : 83.3160171508789\n",
      "Min generator_loss : 13.936774253845215\n",
      "generator loss decrease : -0.19925308227539062\n",
      "generator loss decrease ratio : (1.0055301189422607)\n",
      "Max generator loss decrease : -4.6659698486328125\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 36.1075439453125\n",
      "train_f1_score : 0.6389245688915253\n",
      "train_f1_rounded_score : 0.6402270793914795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 38.288333892822266\n",
      "valid_f1_score : 0.6171166598796844\n",
      "valid_f1_rounded_score : 0.6186009049415588\n",
      "discriminator_learning is False\n",
      "[Epoch 23/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 37.661026] time: 5:18:17.737828\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 23/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 32.915390] time: 5:22:18.747564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 37.04884719848633\n",
      "Max generator_loss : 83.26500701904297\n",
      "Min generator_loss : 16.29317283630371\n",
      "generator loss decrease : -1.0183906555175781\n",
      "generator loss decrease ratio : (1.0282647609710693)\n",
      "Max generator loss decrease : 0.0510101318359375\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 33.21431350708008\n",
      "train_f1_score : 0.655598372220993\n",
      "train_f1_rounded_score : 0.6561228632926941\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 35.77934265136719\n",
      "valid_f1_score : 0.6290014088153839\n",
      "valid_f1_rounded_score : 0.6298751831054688\n",
      "discriminator_learning is False\n",
      "[Epoch 24/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 39.542015] time: 5:27:57.834310\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 24/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 32.634247] time: 5:31:58.621885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 37.22456741333008\n",
      "Max generator_loss : 77.750732421875\n",
      "Min generator_loss : 16.212448120117188\n",
      "generator loss decrease : -1.1941108703613281\n",
      "generator loss decrease ratio : (1.0331417322158813)\n",
      "Max generator loss decrease : 5.514274597167969\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 33.831905364990234\n",
      "train_f1_score : 0.6457391977310181\n",
      "train_f1_rounded_score : 0.6465058922767639\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 36.70716094970703\n",
      "valid_f1_score : 0.6156318187713623\n",
      "valid_f1_rounded_score : 0.6170197129249573\n",
      "discriminator_learning is False\n",
      "[Epoch 25/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 45.706264] time: 5:37:38.432444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 25/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 34.433723] time: 5:41:41.705081\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 37.82411575317383\n",
      "Max generator_loss : 80.66132354736328\n",
      "Min generator_loss : 16.583251953125\n",
      "generator loss decrease : -1.7936592102050781\n",
      "generator loss decrease ratio : (1.0497817993164062)\n",
      "Max generator loss decrease : -2.9105911254882812\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 34.24755859375\n",
      "train_f1_score : 0.6413868367671967\n",
      "train_f1_rounded_score : 0.6409979462623596\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.27727127075195\n",
      "valid_f1_score : 0.6410757005214691\n",
      "valid_f1_rounded_score : 0.6411426067352295\n",
      "discriminator_learning is False\n",
      "[Epoch 26/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 38.111309] time: 5:47:21.553271\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 26/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 33.966301] time: 5:51:22.090099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 37.47633743286133\n",
      "Max generator_loss : 88.20701599121094\n",
      "Min generator_loss : 16.15494728088379\n",
      "generator loss decrease : -1.4458808898925781\n",
      "generator loss decrease ratio : (1.0401294231414795)\n",
      "Max generator loss decrease : -7.545692443847656\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 48.6289176940918\n",
      "train_f1_score : 0.49079668521881104\n",
      "train_f1_rounded_score : 0.4892224073410034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 43.30237579345703\n",
      "valid_f1_score : 0.5465719699859619\n",
      "valid_f1_rounded_score : 0.5451379418373108\n",
      "discriminator_learning is False\n",
      "[Epoch 27/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 41.500488] time: 5:57:01.198413\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 27/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 32.789570] time: 6:01:01.742125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 37.13880920410156\n",
      "Max generator_loss : 81.90460205078125\n",
      "Min generator_loss : 16.484432220458984\n",
      "generator loss decrease : -1.1083526611328125\n",
      "generator loss decrease ratio : (1.0307615995407104)\n",
      "Max generator loss decrease : 6.3024139404296875\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 33.39391326904297\n",
      "train_f1_score : 0.6503255069255829\n",
      "train_f1_rounded_score : 0.652248740196228\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.55933380126953\n",
      "valid_f1_score : 0.6381221413612366\n",
      "valid_f1_rounded_score : 0.6398894190788269\n",
      "discriminator_learning is False\n",
      "[Epoch 28/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 41.985062] time: 6:06:42.216698\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 28/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 42.113537] time: 6:10:42.818616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 36.90007400512695\n",
      "Max generator_loss : 81.60433197021484\n",
      "Min generator_loss : 15.684598922729492\n",
      "generator loss decrease : -0.8696174621582031\n",
      "generator loss decrease ratio : (1.0241355895996094)\n",
      "Max generator loss decrease : 0.30027008056640625\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 36.71700668334961\n",
      "train_f1_score : 0.6155287325382233\n",
      "train_f1_rounded_score : 0.6159923076629639\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 35.233482360839844\n",
      "valid_f1_score : 0.6310630142688751\n",
      "valid_f1_rounded_score : 0.6324489116668701\n",
      "discriminator_learning is False\n",
      "[Epoch 29/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 39.888706] time: 6:16:21.850450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 29/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 37.885475] time: 6:20:22.419078\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 36.97524642944336\n",
      "Max generator_loss : 82.78749084472656\n",
      "Min generator_loss : 15.598101615905762\n",
      "generator loss decrease : -0.9447898864746094\n",
      "generator loss decrease ratio : (1.0262219905853271)\n",
      "Max generator loss decrease : -1.1831588745117188\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.004\n",
      "train_f1_loss : 38.3894157409668\n",
      "train_f1_score : 0.5980165898799896\n",
      "train_f1_rounded_score : 0.5989539623260498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 41.2287483215332\n",
      "valid_f1_score : 0.5682853758335114\n",
      "valid_f1_rounded_score : 0.5694082379341125\n",
      "discriminator_learning is False\n",
      "[Epoch 30/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 43.399101] time: 6:26:01.250081\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 30/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 30.082544] time: 6:30:01.842004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 35.34426498413086\n",
      "Max generator_loss : 81.47832489013672\n",
      "Min generator_loss : 15.262712478637695\n",
      "generator loss decrease : 0.6861915588378906\n",
      "generator loss decrease ratio : (0.9809552431106567)\n",
      "Max generator loss decrease : 1.3091659545898438\n",
      "current lowest generator loss : 36.03045654296875\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 31.07408905029297\n",
      "train_f1_score : 0.6746168732643127\n",
      "train_f1_rounded_score : 0.6745689511299133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.47639846801758\n",
      "valid_f1_score : 0.6494618058204651\n",
      "valid_f1_rounded_score : 0.6498699188232422\n",
      "discriminator_learning is False\n",
      "[Epoch 31/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 39.886452] time: 6:35:41.384823\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 31/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 30.088158] time: 6:39:41.822135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 34.77069091796875\n",
      "Max generator_loss : 79.92398834228516\n",
      "Min generator_loss : 15.07766342163086\n",
      "generator loss decrease : 0.5735740661621094\n",
      "generator loss decrease ratio : (0.983771800994873)\n",
      "Max generator loss decrease : 1.5543365478515625\n",
      "current lowest generator loss : 35.34426498413086\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 30.251312255859375\n",
      "train_f1_score : 0.6832323372364044\n",
      "train_f1_rounded_score : 0.6833679676055908\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.073219299316406\n",
      "valid_f1_score : 0.6536835730075836\n",
      "valid_f1_rounded_score : 0.6541653275489807\n",
      "discriminator_learning is False\n",
      "[Epoch 32/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 37.391300] time: 6:45:21.075490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 32/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.794830] time: 6:49:21.875263\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 34.5048713684082\n",
      "Max generator_loss : 76.26730346679688\n",
      "Min generator_loss : 15.08159065246582\n",
      "generator loss decrease : 0.2658195495605469\n",
      "generator loss decrease ratio : (0.9923550486564636)\n",
      "Max generator loss decrease : 3.6566848754882812\n",
      "current lowest generator loss : 34.77069091796875\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.945148468017578\n",
      "train_f1_score : 0.6864382326602936\n",
      "train_f1_rounded_score : 0.6866018176078796\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.45058059692383\n",
      "valid_f1_score : 0.6497321426868439\n",
      "valid_f1_rounded_score : 0.6502721309661865\n",
      "discriminator_learning is False\n",
      "[Epoch 33/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 36.567673] time: 6:55:01.325251\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 33/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.661089] time: 6:59:01.929240\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 34.263336181640625\n",
      "Max generator_loss : 78.19640350341797\n",
      "Min generator_loss : 15.02023696899414\n",
      "generator loss decrease : 0.24153518676757812\n",
      "generator loss decrease ratio : (0.9929999709129333)\n",
      "Max generator loss decrease : -1.9291000366210938\n",
      "current lowest generator loss : 34.5048713684082\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 30.52621841430664\n",
      "train_f1_score : 0.6803537309169769\n",
      "train_f1_rounded_score : 0.6802623271942139\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.07316970825195\n",
      "valid_f1_score : 0.6536840796470642\n",
      "valid_f1_rounded_score : 0.6538853049278259\n",
      "discriminator_learning is False\n",
      "[Epoch 34/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 34.296444] time: 7:04:41.585156\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 34/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.926821] time: 7:08:42.217420\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 34.21105194091797\n",
      "Max generator_loss : 76.86800384521484\n",
      "Min generator_loss : 15.074660301208496\n",
      "generator loss decrease : 0.05228424072265625\n",
      "generator loss decrease ratio : (0.9984740614891052)\n",
      "Max generator loss decrease : 1.328399658203125\n",
      "current lowest generator loss : 34.263336181640625\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 30.41402244567871\n",
      "train_f1_score : 0.6815285682678223\n",
      "train_f1_rounded_score : 0.6816068887710571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.40644073486328\n",
      "valid_f1_score : 0.6501943469047546\n",
      "valid_f1_rounded_score : 0.6506561636924744\n",
      "discriminator_learning is False\n",
      "[Epoch 35/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 36.154388] time: 7:14:21.812950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 35/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.773273] time: 7:18:22.487857\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 34.089202880859375\n",
      "Max generator_loss : 78.8848648071289\n",
      "Min generator_loss : 15.180295944213867\n",
      "generator loss decrease : 0.12184906005859375\n",
      "generator loss decrease ratio : (0.9964383244514465)\n",
      "Max generator loss decrease : -2.0168609619140625\n",
      "current lowest generator loss : 34.21105194091797\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.865232467651367\n",
      "train_f1_score : 0.6872750520706177\n",
      "train_f1_rounded_score : 0.6874017119407654\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.53235626220703\n",
      "valid_f1_score : 0.6488758623600006\n",
      "valid_f1_rounded_score : 0.6493855118751526\n",
      "discriminator_learning is False\n",
      "[Epoch 36/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 36.794327] time: 7:24:02.186940\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 36/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.648787] time: 7:28:02.739327\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.80520248413086\n",
      "Max generator_loss : 75.85619354248047\n",
      "Min generator_loss : 15.1238374710083\n",
      "generator loss decrease : 0.2840003967285156\n",
      "generator loss decrease ratio : (0.9916688799858093)\n",
      "Max generator loss decrease : 3.0286712646484375\n",
      "current lowest generator loss : 34.089202880859375\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 31.520715713500977\n",
      "train_f1_score : 0.6699401438236237\n",
      "train_f1_rounded_score : 0.6697607636451721\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.324275970458984\n",
      "valid_f1_score : 0.6405834853649139\n",
      "valid_f1_rounded_score : 0.6408020257949829\n",
      "discriminator_learning is False\n",
      "[Epoch 37/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 30.637445] time: 7:33:42.376562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 37/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.641790] time: 7:37:42.808936\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.657955169677734\n",
      "Max generator_loss : 73.74128723144531\n",
      "Min generator_loss : 15.23157024383545\n",
      "generator loss decrease : 0.147247314453125\n",
      "generator loss decrease ratio : (0.995644211769104)\n",
      "Max generator loss decrease : 2.1149063110351562\n",
      "current lowest generator loss : 33.80520248413086\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 30.92238426208496\n",
      "train_f1_score : 0.6762053966522217\n",
      "train_f1_rounded_score : 0.6759939789772034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.33024597167969\n",
      "valid_f1_score : 0.6509921848773956\n",
      "valid_f1_rounded_score : 0.6512269377708435\n",
      "discriminator_learning is False\n",
      "[Epoch 38/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 37.903549] time: 7:43:25.366388\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 38/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.382601] time: 7:47:27.641268\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.51018524169922\n",
      "Max generator_loss : 72.57577514648438\n",
      "Min generator_loss : 15.419007301330566\n",
      "generator loss decrease : 0.14776992797851562\n",
      "generator loss decrease ratio : (0.9956096410751343)\n",
      "Max generator loss decrease : 1.1655120849609375\n",
      "current lowest generator loss : 33.657955169677734\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.757244110107422\n",
      "train_f1_score : 0.6884058117866516\n",
      "train_f1_rounded_score : 0.6886438131332397\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.2352294921875\n",
      "valid_f1_score : 0.6415159106254578\n",
      "valid_f1_rounded_score : 0.6420823931694031\n",
      "discriminator_learning is False\n",
      "[Epoch 39/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 33.274033] time: 7:53:12.896574\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 39/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.236656] time: 7:57:17.419705\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.5582275390625\n",
      "Max generator_loss : 70.99626922607422\n",
      "Min generator_loss : 15.099252700805664\n",
      "generator loss decrease : -0.04804229736328125\n",
      "generator loss decrease ratio : (1.0014336109161377)\n",
      "Max generator loss decrease : 1.5795059204101562\n",
      "current lowest generator loss : 33.51018524169922\n",
      "current Learning_rate = 0.0008\n",
      "train_f1_loss : 30.607440948486328\n",
      "train_f1_score : 0.6795032322406769\n",
      "train_f1_rounded_score : 0.67937833070755\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.21225357055664\n",
      "valid_f1_score : 0.6417565047740936\n",
      "valid_f1_rounded_score : 0.6420432329177856\n",
      "discriminator_learning is False\n",
      "[Epoch 40/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 32.513817] time: 8:03:04.229223\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 40/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.438072] time: 8:07:07.446681\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.26895523071289\n",
      "Max generator_loss : 73.83418273925781\n",
      "Min generator_loss : 14.830170631408691\n",
      "generator loss decrease : 0.24123001098632812\n",
      "generator loss decrease ratio : (0.992801308631897)\n",
      "Max generator loss decrease : -2.8379135131835938\n",
      "current lowest generator loss : 33.51018524169922\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.7319278717041\n",
      "train_f1_score : 0.6886709034442902\n",
      "train_f1_rounded_score : 0.6887235641479492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.286739349365234\n",
      "valid_f1_score : 0.6409765481948853\n",
      "valid_f1_rounded_score : 0.6413033604621887\n",
      "discriminator_learning is False\n",
      "[Epoch 41/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 27.501108] time: 8:12:51.850121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 41/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.941799] time: 8:17:04.180364\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.28662872314453\n",
      "Max generator_loss : 70.74989318847656\n",
      "Min generator_loss : 15.033089637756348\n",
      "generator loss decrease : -0.017673492431640625\n",
      "generator loss decrease ratio : (1.0005311965942383)\n",
      "Max generator loss decrease : 3.08428955078125\n",
      "current lowest generator loss : 33.26895523071289\n",
      "current Learning_rate = 0.0008\n",
      "train_f1_loss : 32.29856491088867\n",
      "train_f1_score : 0.6617951393127441\n",
      "train_f1_rounded_score : 0.6614747643470764\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.39332962036133\n",
      "valid_f1_score : 0.6398604214191437\n",
      "valid_f1_rounded_score : 0.6400963068008423\n",
      "discriminator_learning is False\n",
      "[Epoch 42/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.145632] time: 8:23:34.910864\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 42/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.205303] time: 8:28:02.576862\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.087188720703125\n",
      "Max generator_loss : 70.86189270019531\n",
      "Min generator_loss : 15.041802406311035\n",
      "generator loss decrease : 0.18176651000976562\n",
      "generator loss decrease ratio : (0.9945364594459534)\n",
      "Max generator loss decrease : -0.11199951171875\n",
      "current lowest generator loss : 33.26895523071289\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.487390518188477\n",
      "train_f1_score : 0.6912315189838409\n",
      "train_f1_rounded_score : 0.6916226148605347\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.138885498046875\n",
      "valid_f1_score : 0.642524778842926\n",
      "valid_f1_rounded_score : 0.6432732939720154\n",
      "discriminator_learning is False\n",
      "[Epoch 43/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 34.681526] time: 8:34:37.927862\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 43/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.333834] time: 8:39:04.183156\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.0189094543457\n",
      "Max generator_loss : 69.75394439697266\n",
      "Min generator_loss : 14.69413948059082\n",
      "generator loss decrease : 0.06827926635742188\n",
      "generator loss decrease ratio : (0.9979363679885864)\n",
      "Max generator loss decrease : 1.1079483032226562\n",
      "current lowest generator loss : 33.087188720703125\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.49116325378418\n",
      "train_f1_score : 0.6911920011043549\n",
      "train_f1_rounded_score : 0.6916022896766663\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.927146911621094\n",
      "valid_f1_score : 0.6447419226169586\n",
      "valid_f1_rounded_score : 0.6454683542251587\n",
      "discriminator_learning is False\n",
      "[Epoch 44/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 34.941917] time: 8:45:36.952658\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 44/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.442535] time: 8:50:05.195146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 32.926021575927734\n",
      "Max generator_loss : 70.37303924560547\n",
      "Min generator_loss : 15.248333930969238\n",
      "generator loss decrease : 0.09288787841796875\n",
      "generator loss decrease ratio : (0.9971868395805359)\n",
      "Max generator loss decrease : -0.6190948486328125\n",
      "current lowest generator loss : 33.0189094543457\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 32.45112991333008\n",
      "train_f1_score : 0.6601975858211517\n",
      "train_f1_rounded_score : 0.659805417060852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 35.23633575439453\n",
      "valid_f1_score : 0.6310331523418427\n",
      "valid_f1_rounded_score : 0.631181001663208\n",
      "discriminator_learning is False\n",
      "[Epoch 45/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 32.507462] time: 8:56:47.447134\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 45/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.101137] time: 9:01:11.960145\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 32.845458984375\n",
      "Max generator_loss : 72.75630187988281\n",
      "Min generator_loss : 15.007721900939941\n",
      "generator loss decrease : 0.08056259155273438\n",
      "generator loss decrease ratio : (0.9975532293319702)\n",
      "Max generator loss decrease : -2.3832626342773438\n",
      "current lowest generator loss : 32.926021575927734\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.11311912536621\n",
      "train_f1_score : 0.6951505839824677\n",
      "train_f1_rounded_score : 0.6953240036964417\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.53148651123047\n",
      "valid_f1_score : 0.6488849520683289\n",
      "valid_f1_rounded_score : 0.6494110226631165\n",
      "discriminator_learning is False\n",
      "[Epoch 46/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 35.513462] time: 9:07:35.382847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 46/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 28.921494] time: 9:12:01.365906\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 32.66485595703125\n",
      "Max generator_loss : 69.68569946289062\n",
      "Min generator_loss : 15.21538257598877\n",
      "generator loss decrease : 0.18060302734375\n",
      "generator loss decrease ratio : (0.9945014119148254)\n",
      "Max generator loss decrease : 3.0706024169921875\n",
      "current lowest generator loss : 32.845458984375\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.30620002746582\n",
      "train_f1_score : 0.6931287944316864\n",
      "train_f1_rounded_score : 0.693569004535675\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.6622314453125\n",
      "valid_f1_score : 0.6370446979999542\n",
      "valid_f1_rounded_score : 0.6378505229949951\n",
      "discriminator_learning is False\n",
      "[Epoch 47/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.473797] time: 9:18:00.336635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 47/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 29.099623] time: 9:22:04.033462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 32.76402282714844\n",
      "Max generator_loss : 70.3727798461914\n",
      "Min generator_loss : 14.758162498474121\n",
      "generator loss decrease : -0.0991668701171875\n",
      "generator loss decrease ratio : (1.0030359029769897)\n",
      "Max generator loss decrease : -0.6870803833007812\n",
      "current lowest generator loss : 32.66485595703125\n",
      "current Learning_rate = 0.0008\n",
      "train_f1_loss : 28.85689926147461\n",
      "train_f1_score : 0.6978335082530975\n",
      "train_f1_rounded_score : 0.6980255246162415\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.05218505859375\n",
      "valid_f1_score : 0.6434326171875\n",
      "valid_f1_rounded_score : 0.643927812576294\n",
      "discriminator_learning is False\n",
      "[Epoch 48/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 30.029259] time: 9:27:50.456982\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 48/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 28.171001] time: 9:31:55.192968\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 32.59995651245117\n",
      "Max generator_loss : 74.78030395507812\n",
      "Min generator_loss : 14.397356986999512\n",
      "generator loss decrease : 0.06489944458007812\n",
      "generator loss decrease ratio : (0.9980131983757019)\n",
      "Max generator loss decrease : -4.407524108886719\n",
      "current lowest generator loss : 32.66485595703125\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 29.237253189086914\n",
      "train_f1_score : 0.6938507556915283\n",
      "train_f1_rounded_score : 0.6939958930015564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.061214447021484\n",
      "valid_f1_score : 0.6433380842208862\n",
      "valid_f1_rounded_score : 0.6437777876853943\n",
      "discriminator_learning is False\n",
      "[Epoch 49/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 23.101492] time: 9:37:39.963909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 49/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 28.360600] time: 9:41:43.490538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 32.481788635253906\n",
      "Max generator_loss : 69.66757202148438\n",
      "Min generator_loss : 14.566112518310547\n",
      "generator loss decrease : 0.11816787719726562\n",
      "generator loss decrease ratio : (0.9963752031326294)\n",
      "Max generator loss decrease : 5.11273193359375\n",
      "current lowest generator loss : 32.59995651245117\n",
      "current Learning_rate = 0.0008\n",
      "save weights\n",
      "train_f1_loss : 30.33956527709961\n",
      "train_f1_score : 0.6823082268238068\n",
      "train_f1_rounded_score : 0.6821497082710266\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.945194244384766\n",
      "valid_f1_score : 0.6445529460906982\n",
      "valid_f1_rounded_score : 0.64473956823349\n",
      "discriminator_learning is False\n",
      "[Epoch 50/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 37.816166] time: 9:47:29.536906\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 50/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 44.829792] time: 9:51:36.998860\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 34.3138313293457\n",
      "Max generator_loss : 76.15510559082031\n",
      "Min generator_loss : 16.327922821044922\n",
      "generator loss decrease : -1.8320426940917969\n",
      "generator loss decrease ratio : (1.0564022064208984)\n",
      "Max generator loss decrease : -6.4875335693359375\n",
      "current lowest generator loss : 32.481788635253906\n",
      "current Learning_rate = 0.0008\n",
      "train_f1_loss : 28.66836929321289\n",
      "train_f1_score : 0.6998076438903809\n",
      "train_f1_rounded_score : 0.7001137137413025\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.207786560058594\n",
      "valid_f1_score : 0.6418032944202423\n",
      "valid_f1_rounded_score : 0.6424064040184021\n",
      "discriminator_learning is False\n",
      "[Epoch 51/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 33.328426] time: 9:57:22.688492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 51/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 45.390820] time: 10:01:26.197950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.722564697265625\n",
      "Max generator_loss : 76.5640640258789\n",
      "Min generator_loss : 15.863944053649902\n",
      "generator loss decrease : -1.2407760620117188\n",
      "generator loss decrease ratio : (1.0381991863250732)\n",
      "Max generator loss decrease : -0.40895843505859375\n",
      "current lowest generator loss : 32.481788635253906\n",
      "current Learning_rate = 0.0008\n",
      "train_f1_loss : 28.556608200073242\n",
      "train_f1_score : 0.7009779214859009\n",
      "train_f1_rounded_score : 0.7010713815689087\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.3170166015625\n",
      "valid_f1_score : 0.651130735874176\n",
      "valid_f1_rounded_score : 0.65162193775177\n",
      "discriminator_learning is False\n",
      "[Epoch 52/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 35.386772] time: 10:07:11.126246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 52/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 46.268234] time: 10:11:15.136695\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.60818862915039\n",
      "Max generator_loss : 76.656982421875\n",
      "Min generator_loss : 15.377723693847656\n",
      "generator loss decrease : -1.1263999938964844\n",
      "generator loss decrease ratio : (1.0346778631210327)\n",
      "Max generator loss decrease : -0.09291839599609375\n",
      "current lowest generator loss : 32.481788635253906\n",
      "current Learning_rate = 0.0008\n",
      "train_f1_loss : 29.57950210571289\n",
      "train_f1_score : 0.6902669966220856\n",
      "train_f1_rounded_score : 0.690908670425415\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 35.86231994628906\n",
      "valid_f1_score : 0.6244783401489258\n",
      "valid_f1_rounded_score : 0.6253415942192078\n",
      "discriminator_learning is False\n",
      "[Epoch 53/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 32.083481] time: 10:17:00.604309\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 53/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 46.671947] time: 10:21:03.968558\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.443572998046875\n",
      "Max generator_loss : 76.2235336303711\n",
      "Min generator_loss : 15.700979232788086\n",
      "generator loss decrease : -0.9617843627929688\n",
      "generator loss decrease ratio : (1.0296099185943604)\n",
      "Max generator loss decrease : 0.43344879150390625\n",
      "current lowest generator loss : 32.481788635253906\n",
      "current Learning_rate = 0.0008\n",
      "train_f1_loss : 28.515708923339844\n",
      "train_f1_score : 0.7014061808586121\n",
      "train_f1_rounded_score : 0.7016951441764832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.58338928222656\n",
      "valid_f1_score : 0.6378702521324158\n",
      "valid_f1_rounded_score : 0.6385037302970886\n",
      "discriminator_learning is False\n",
      "[Epoch 54/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 30.363241] time: 10:26:59.949345\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% |###################################                                     |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 54/325] [Batch 3100/6200] [D loss: 0.692960, acc: 100%] [G loss: 44.417652] time: 10:31:09.069437\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 1.0\n",
      "Mean generator_loss : 33.248260498046875\n",
      "Max generator_loss : 75.69418334960938\n",
      "Min generator_loss : 15.29141902923584\n",
      "generator loss decrease : -0.7664718627929688\n",
      "generator loss decrease ratio : (1.023597002029419)\n",
      "Max generator loss decrease : 0.5293502807617188\n",
      "current lowest generator loss : 32.481788635253906\n",
      "current Learning_rate = 0.0008\n",
      "train_f1_loss : 28.34046745300293\n",
      "train_f1_score : 0.7032411694526672\n",
      "train_f1_rounded_score : 0.7034735083580017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 33.78583908081055\n",
      "valid_f1_score : 0.6462215781211853\n",
      "valid_f1_rounded_score : 0.646822452545166\n",
      "discriminator_learning is False\n",
      "[Epoch 55/325] [Batch 0/6200] [D loss: 0.692960, acc: 100%] [G loss: 34.928558] time: 10:37:04.484487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Internal Python error in the inspect module.                       |\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3418, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-3-a0ebaebf294c>\", line 4, in <module>\n",
      "    gan.train(epochs=325, batch_size=batch_size, sample_interval=3100, epoch_shuffle_term=50)\n",
      "  File \"<ipython-input-1-b93c4aeea8e7>\", line 296, in train\n",
      "    generator_loss = self.combined.train_on_batch(\n",
      "  File \"C:\\Users\\gr300\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\", line 1695, in train_on_batch\n",
      "    logs = train_function(iterator)\n",
      "  File \"C:\\Users\\gr300\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 780, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"C:\\Users\\gr300\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\def_function.py\", line 807, in _call\n",
      "    return self._stateless_fn(*args, **kwds)  # pylint: disable=not-callable\n",
      "  File \"C:\\Users\\gr300\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\", line 2829, in __call__\n",
      "    return graph_function._filtered_call(args, kwargs)  # pylint: disable=protected-access\n",
      "  File \"C:\\Users\\gr300\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1843, in _filtered_call\n",
      "    return self._call_flat(\n",
      "  File \"C:\\Users\\gr300\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\", line 1923, in _call_flat\n",
      "    return self._build_call_outputs(self._inference_function.call(\n",
      "  File \"C:\\Users\\gr300\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\", line 545, in call\n",
      "    outputs = execute.execute(\n",
      "  File \"C:\\Users\\gr300\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\execute.py\", line 59, in quick_execute\n",
      "    tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2045, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 1170, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 316, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 350, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\inspect.py\", line 1503, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\inspect.py\", line 1461, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\inspect.py\", line 708, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\inspect.py\", line 754, in getmodule\n",
      "    os.path.realpath(f)] = module.__name__\n",
      "  File \"C:\\Users\\gr300\\anaconda3\\lib\\ntpath.py\", line 664, in realpath\n",
      "    if _getfinalpathname(spath) == path:\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "object of type 'NoneType' has no len()",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "    \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "\u001b[1;32m<ipython-input-3-a0ebaebf294c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m#gan.load_study_info()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mgan\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m325\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_interval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3100\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepoch_shuffle_term\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-1-b93c4aeea8e7>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self, epochs, batch_size, sample_interval, epoch_shuffle_term)\u001b[0m\n\u001b[0;32m    295\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 296\u001b[1;33m                 generator_loss = self.combined.train_on_batch(\n\u001b[0m\u001b[0;32m    297\u001b[0m                     \u001b[1;33m[\u001b[0m\u001b[0moriginal_img\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmasked_img\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[1;34m(self, x, y, sample_weight, class_weight, reset_metrics, return_dict)\u001b[0m\n\u001b[0;32m   1694\u001b[0m       \u001b[0mtrain_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_train_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1695\u001b[1;33m       \u001b[0mlogs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1696\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    779\u001b[0m         \u001b[0mcompiler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"nonXla\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 780\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    781\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    806\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 807\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    808\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2828\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2829\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2830\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1842\u001b[0m     \"\"\"\n\u001b[1;32m-> 1843\u001b[1;33m     return self._call_flat(\n\u001b[0m\u001b[0;32m   1844\u001b[0m         [t for t in nest.flatten((args, kwargs), expand_composites=True)\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1922\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1923\u001b[1;33m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[0;32m   1924\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    544\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m           outputs = execute.execute(\n\u001b[0m\u001b[0;32m    546\u001b[0m               \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 59\u001b[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[0;32m     60\u001b[0m                                         inputs, attrs, num_outputs)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[1;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[0;32m   2044\u001b[0m                         \u001b[1;31m# in the engines. This should return a list of strings.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2045\u001b[1;33m                         \u001b[0mstb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_render_traceback_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2046\u001b[0m                     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'KeyboardInterrupt' object has no attribute '_render_traceback_'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "    \u001b[1;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[1;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[0;32m   2045\u001b[0m                         \u001b[0mstb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_render_traceback_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2046\u001b[0m                     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2047\u001b[1;33m                         stb = self.InteractiveTB.structured_traceback(etype,\n\u001b[0m\u001b[0;32m   2048\u001b[0m                                             value, tb, tb_offset=tb_offset)\n\u001b[0;32m   2049\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[1;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[0;32m   1434\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1435\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1436\u001b[1;33m         return FormattedTB.structured_traceback(\n\u001b[0m\u001b[0;32m   1437\u001b[0m             self, etype, value, tb, tb_offset, number_of_lines_of_context)\n\u001b[0;32m   1438\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[1;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[0;32m   1334\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmode\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mverbose_modes\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1335\u001b[0m             \u001b[1;31m# Verbose modes need a full traceback\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1336\u001b[1;33m             return VerboseTB.structured_traceback(\n\u001b[0m\u001b[0;32m   1337\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0metype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtb_offset\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnumber_of_lines_of_context\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1338\u001b[0m             )\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[1;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[0;32m   1191\u001b[0m         \u001b[1;34m\"\"\"Return a nice text document describing the traceback.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1192\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1193\u001b[1;33m         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,\n\u001b[0m\u001b[0;32m   1194\u001b[0m                                                                tb_offset)\n\u001b[0;32m   1195\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\u001b[0m in \u001b[0;36mformat_exception_as_a_whole\u001b[1;34m(self, etype, evalue, etb, number_of_lines_of_context, tb_offset)\u001b[0m\n\u001b[0;32m   1149\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1150\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1151\u001b[1;33m         \u001b[0mlast_unique\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrecursion_repeat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfind_recursion\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0morig_etype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mevalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrecords\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1152\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1153\u001b[0m         \u001b[0mframes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat_records\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrecords\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlast_unique\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrecursion_repeat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\IPython\\core\\ultratb.py\u001b[0m in \u001b[0;36mfind_recursion\u001b[1;34m(etype, value, records)\u001b[0m\n\u001b[0;32m    449\u001b[0m     \u001b[1;31m# first frame (from in to out) that looks different.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    450\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_recursion_error\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0metype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrecords\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 451\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrecords\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    452\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    453\u001b[0m     \u001b[1;31m# Select filename, lineno, func_name to track frames with\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: object of type 'NoneType' has no len()"
     ]
    }
   ],
   "source": [
    "#gan.find_error = True\n",
    "#gan.find_error_epoch = 5\n",
    "#gan.load_study_info()\n",
    "gan.train(epochs=325, batch_size=batch_size, sample_interval=3100, epoch_shuffle_term=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-35-f92eaba7459e>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-35-f92eaba7459e>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    gan.discriminator.optimizer.\u001b[0m\n\u001b[1;37m                                ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "gan.discriminator.optimizer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 8 calls to <function Model.make_train_function.<locals>.train_function at 0x00000288678493A0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 8 calls to <function Model.make_train_function.<locals>.train_function at 0x00000288678493A0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elapsed time : 4.586500883102417\n",
      "elapsed time : 0.29304075241088867\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "temp_source = gan.original_img\n",
    "temp_mask = gan.masked_img\n",
    "\n",
    "start_time = time.time()\n",
    "gan.generator.train_on_batch(temp_source, temp_mask)\n",
    "print(f\"elapsed time : {time.time() - start_time}\")\n",
    "\n",
    "temp_source = tf.convert_to_tensor(temp_source)\n",
    "temp_mask = tf.convert_to_tensor(temp_mask)\n",
    "\n",
    "start_time = time.time()\n",
    "gan.generator.train_on_batch(temp_source, temp_mask)\n",
    "print(f\"elapsed time : {time.time() - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-cba4fde71471>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m     batch_index = gan.train_loaded_data_index[batch_i: batch_i +\n\u001b[0;32m      6\u001b[0m                                                gan.batch_size]\n\u001b[1;32m----> 7\u001b[1;33m     original_img, masked_img = gan.data_loader.get_data(\n\u001b[0m\u001b[0;32m      8\u001b[0m         data_mode=\"train\", index=batch_index)\n",
      "\u001b[1;32m~\\Desktop\\Works\\jupyterlab\\의료데이터(pix2pix)_size_20_v0.2\\data_loader\\medical_segmentation_data_loader_v2.py\u001b[0m in \u001b[0;36mget_data\u001b[1;34m(self, data_mode, index)\u001b[0m\n\u001b[0;32m     86\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m                     data_tuple = (\n\u001b[1;32m---> 88\u001b[1;33m                         self.train_loaded_data[0][index], self.train_loaded_data[1][index])\n\u001b[0m\u001b[0;32m     89\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mdata_mode\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"valid\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     90\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "batch_i = 0\n",
    "\n",
    "while batch_i + gan.batch_size <= gan.data_loader.train_data_length:\n",
    "\n",
    "    batch_index = gan.train_loaded_data_index[batch_i: batch_i +\n",
    "                                               gan.batch_size]\n",
    "    original_img, masked_img = gan.data_loader.get_data(\n",
    "        data_mode=\"train\", index=batch_index)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gan.data_loader.train_data_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Iterator : 260초\n",
    "# Queue Iterator : 200초"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elapsed time : 204.91554951667786\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import threading\n",
    "from queue import Queue\n",
    "\n",
    "ITER_NUM = 620\n",
    "batch_size = 10\n",
    "\n",
    "gan.generator.compile(\n",
    "    loss=sm.losses.BinaryFocalLoss(),\n",
    "    optimizer=Nadam(gan.generator_learning_rate),\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "\n",
    "def batch_setter(queue):\n",
    "    batch_i = 0\n",
    "    count = 0\n",
    "    while batch_i + gan.batch_size <= gan.data_loader.train_data_length and count < ITER_NUM:\n",
    "        \n",
    "        batch_index = gan.train_loaded_data_index[batch_i: batch_i +\n",
    "                                                   gan.batch_size]        \n",
    "        \n",
    "        batch_tuple = gan.data_loader.get_data(\n",
    "        data_mode=\"train\", index=batch_index)\n",
    "\n",
    "        queue.put(batch_tuple)\n",
    "        queue.join()\n",
    "        count += 1\n",
    "    \n",
    "def batch_getter(queue):\n",
    "    \n",
    "    original_img, masked_img = queue.get()\n",
    "    tensor_original_img = tf.convert_to_tensor(original_img)\n",
    "    tensor_masked_img = tf.convert_to_tensor(masked_img)\n",
    "    queue.task_done()\n",
    "    \n",
    "    return tensor_original_img, tensor_masked_img\n",
    "    \n",
    "def batch_trainer(original_img, masked_img):\n",
    "    \n",
    "    gan.generator.train_on_batch(temp_source, temp_mask)\n",
    "\n",
    "q = Queue()\n",
    "\n",
    "setter = threading.Thread(target=batch_setter, args=(q,),daemon=True)\n",
    "setter.start()\n",
    "start_time = time.time()\n",
    "for i in range(ITER_NUM):\n",
    "    tensor_original_img, tensor_masked_img = batch_getter(q)\n",
    "    \n",
    "    gan.generator.train_on_batch(tensor_original_img, tensor_masked_img)\n",
    "print(f\"elapsed time : {time.time() - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elapsed time : 260.5594081878662\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "batch_i = 0\n",
    "count = 0\n",
    "while batch_i + gan.batch_size <= gan.data_loader.train_data_length and count < ITER_NUM:\n",
    "\n",
    "    batch_index = gan.train_loaded_data_index[batch_i: batch_i +\n",
    "                                               gan.batch_size]        \n",
    "    batch_tuple = gan.data_loader.get_data(\n",
    "    data_mode=\"train\", index=batch_index)\n",
    "    \n",
    "    gan.generator.train_on_batch(*batch_tuple)\n",
    "    \n",
    "    count += 1\n",
    "print(f\"elapsed time : {time.time() - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'tensorflow.python.framework.ops.EagerTensor'>\n",
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "\n",
    "temp = tensor_masked_img\n",
    "print(type(temp))\n",
    "print(type(temp.numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(temp.numpy(), tf.Tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gan_module.model.build_model import build_discriminator\n",
    "\n",
    "temp = build_discriminator(\n",
    "            input_img_shape=(512,512,3),\n",
    "            output_img_shape=(512,512,1),\n",
    "            discriminator_power=4,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 512, 512, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            [(None, 512, 512, 1) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 512, 512, 4)  0           input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 512, 512, 4)  148         concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 512, 512, 4)  16          conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu (LeakyReLU)         (None, 512, 512, 4)  0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 512, 512, 4)  148         leaky_re_lu[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 512, 512, 4)  16          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)       (None, 512, 512, 4)  0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 512, 512, 4)  148         leaky_re_lu_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 512, 512, 4)  16          conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)       (None, 512, 512, 4)  0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "add (Add)                       (None, 512, 512, 4)  0           leaky_re_lu_2[0][0]              \n",
      "                                                                 leaky_re_lu[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 512, 512, 4)  16          add[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)       (None, 512, 512, 4)  0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 512, 512, 8)  296         leaky_re_lu_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 512, 512, 8)  32          conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_5 (LeakyReLU)       (None, 512, 512, 8)  0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D)    (None, 256, 256, 8)  0           leaky_re_lu_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d (AveragePooli (None, 256, 256, 4)  0           leaky_re_lu_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 256, 256, 8)  584         max_pooling2d[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 256, 256, 8)  40          average_pooling2d[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 256, 256, 8)  32          conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 256, 256, 8)  32          conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_6 (LeakyReLU)       (None, 256, 256, 8)  0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)       (None, 256, 256, 8)  0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 256, 256, 8)  0           leaky_re_lu_6[0][0]              \n",
      "                                                                 leaky_re_lu_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 256, 256, 8)  32          add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_7 (LeakyReLU)       (None, 256, 256, 8)  0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 256, 256, 8)  584         leaky_re_lu_7[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 256, 256, 8)  32          conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_8 (LeakyReLU)       (None, 256, 256, 8)  0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 256, 256, 8)  584         leaky_re_lu_8[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 256, 256, 8)  32          conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_9 (LeakyReLU)       (None, 256, 256, 8)  0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 256, 256, 8)  0           leaky_re_lu_9[0][0]              \n",
      "                                                                 leaky_re_lu_7[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 256, 256, 8)  32          add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_10 (LeakyReLU)      (None, 256, 256, 8)  0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 256, 256, 8)  584         leaky_re_lu_10[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 256, 256, 8)  32          conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_12 (LeakyReLU)      (None, 256, 256, 8)  0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 128, 128, 8)  0           leaky_re_lu_12[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_1 (AveragePoo (None, 128, 128, 8)  0           leaky_re_lu_10[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 128, 128, 8)  584         max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 128, 128, 8)  72          average_pooling2d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 128, 128, 8)  32          conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 128, 128, 8)  32          conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_13 (LeakyReLU)      (None, 128, 128, 8)  0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_11 (LeakyReLU)      (None, 128, 128, 8)  0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 128, 128, 8)  0           leaky_re_lu_13[0][0]             \n",
      "                                                                 leaky_re_lu_11[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 128, 128, 8)  32          add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_14 (LeakyReLU)      (None, 128, 128, 8)  0           batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 128, 128, 8)  584         leaky_re_lu_14[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 128, 128, 8)  32          conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_15 (LeakyReLU)      (None, 128, 128, 8)  0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 128, 128, 8)  584         leaky_re_lu_15[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 128, 128, 8)  32          conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_16 (LeakyReLU)      (None, 128, 128, 8)  0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 128, 128, 8)  0           leaky_re_lu_16[0][0]             \n",
      "                                                                 leaky_re_lu_14[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 128, 128, 8)  32          add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_17 (LeakyReLU)      (None, 128, 128, 8)  0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 128, 128, 16) 1168        leaky_re_lu_17[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 128, 128, 16) 64          conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_19 (LeakyReLU)      (None, 128, 128, 16) 0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 64, 64, 16)   0           leaky_re_lu_19[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_2 (AveragePoo (None, 64, 64, 8)    0           leaky_re_lu_17[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 64, 64, 16)   2320        max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 64, 64, 16)   144         average_pooling2d_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 64, 64, 16)   64          conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 64, 64, 16)   64          conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_20 (LeakyReLU)      (None, 64, 64, 16)   0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_18 (LeakyReLU)      (None, 64, 64, 16)   0           batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 64, 64, 16)   0           leaky_re_lu_20[0][0]             \n",
      "                                                                 leaky_re_lu_18[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 64, 64, 16)   64          add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_21 (LeakyReLU)      (None, 64, 64, 16)   0           batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 64, 64, 16)   2320        leaky_re_lu_21[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 64, 64, 16)   64          conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_22 (LeakyReLU)      (None, 64, 64, 16)   0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 64, 64, 16)   2320        leaky_re_lu_22[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 64, 64, 16)   64          conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_23 (LeakyReLU)      (None, 64, 64, 16)   0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 64, 64, 16)   0           leaky_re_lu_23[0][0]             \n",
      "                                                                 leaky_re_lu_21[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 64, 64, 16)   64          add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_24 (LeakyReLU)      (None, 64, 64, 16)   0           batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 64, 64, 16)   2320        leaky_re_lu_24[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNo (None, 64, 64, 16)   64          conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_26 (LeakyReLU)      (None, 64, 64, 16)   0           batch_normalization_26[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, 32, 32, 16)   0           leaky_re_lu_26[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_3 (AveragePoo (None, 32, 32, 16)   0           leaky_re_lu_24[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 32, 32, 16)   2320        max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 32, 32, 16)   272         average_pooling2d_3[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNo (None, 32, 32, 16)   64          conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, 32, 32, 16)   64          conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_27 (LeakyReLU)      (None, 32, 32, 16)   0           batch_normalization_27[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_25 (LeakyReLU)      (None, 32, 32, 16)   0           batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 32, 32, 16)   0           leaky_re_lu_27[0][0]             \n",
      "                                                                 leaky_re_lu_25[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_28 (BatchNo (None, 32, 32, 16)   64          add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_28 (LeakyReLU)      (None, 32, 32, 16)   0           batch_normalization_28[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 32, 32, 16)   2320        leaky_re_lu_28[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_29 (BatchNo (None, 32, 32, 16)   64          conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_29 (LeakyReLU)      (None, 32, 32, 16)   0           batch_normalization_29[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 32, 32, 16)   2320        leaky_re_lu_29[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_30 (BatchNo (None, 32, 32, 16)   64          conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_30 (LeakyReLU)      (None, 32, 32, 16)   0           batch_normalization_30[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 32, 32, 16)   0           leaky_re_lu_30[0][0]             \n",
      "                                                                 leaky_re_lu_28[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNo (None, 32, 32, 16)   64          add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_31 (LeakyReLU)      (None, 32, 32, 16)   0           batch_normalization_31[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)              (None, 32, 32, 32)   4640        leaky_re_lu_31[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_33 (BatchNo (None, 32, 32, 32)   128         conv2d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_33 (LeakyReLU)      (None, 32, 32, 32)   0           batch_normalization_33[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)  (None, 16, 16, 32)   0           leaky_re_lu_33[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_4 (AveragePoo (None, 16, 16, 16)   0           leaky_re_lu_31[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)              (None, 16, 16, 32)   9248        max_pooling2d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)              (None, 16, 16, 32)   544         average_pooling2d_4[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_34 (BatchNo (None, 16, 16, 32)   128         conv2d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_32 (BatchNo (None, 16, 16, 32)   128         conv2d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_34 (LeakyReLU)      (None, 16, 16, 32)   0           batch_normalization_34[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_32 (LeakyReLU)      (None, 16, 16, 32)   0           batch_normalization_32[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 16, 16, 32)   0           leaky_re_lu_34[0][0]             \n",
      "                                                                 leaky_re_lu_32[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_35 (BatchNo (None, 16, 16, 32)   128         add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_35 (LeakyReLU)      (None, 16, 16, 32)   0           batch_normalization_35[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)              (None, 16, 16, 32)   9248        leaky_re_lu_35[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_36 (BatchNo (None, 16, 16, 32)   128         conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_36 (LeakyReLU)      (None, 16, 16, 32)   0           batch_normalization_36[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)              (None, 16, 16, 32)   9248        leaky_re_lu_36[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_37 (BatchNo (None, 16, 16, 32)   128         conv2d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_37 (LeakyReLU)      (None, 16, 16, 32)   0           batch_normalization_37[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 16, 16, 32)   0           leaky_re_lu_37[0][0]             \n",
      "                                                                 leaky_re_lu_35[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_38 (BatchNo (None, 16, 16, 32)   128         add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_38 (LeakyReLU)      (None, 16, 16, 32)   0           batch_normalization_38[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_29 (Conv2D)              (None, 16, 16, 32)   9248        leaky_re_lu_38[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_40 (BatchNo (None, 16, 16, 32)   128         conv2d_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_40 (LeakyReLU)      (None, 16, 16, 32)   0           batch_normalization_40[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2D)  (None, 8, 8, 32)     0           leaky_re_lu_40[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_5 (AveragePoo (None, 8, 8, 32)     0           leaky_re_lu_38[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_30 (Conv2D)              (None, 8, 8, 32)     9248        max_pooling2d_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)              (None, 8, 8, 32)     1056        average_pooling2d_5[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_41 (BatchNo (None, 8, 8, 32)     128         conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_39 (BatchNo (None, 8, 8, 32)     128         conv2d_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_41 (LeakyReLU)      (None, 8, 8, 32)     0           batch_normalization_41[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_39 (LeakyReLU)      (None, 8, 8, 32)     0           batch_normalization_39[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 8, 8, 32)     0           leaky_re_lu_41[0][0]             \n",
      "                                                                 leaky_re_lu_39[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_42 (BatchNo (None, 8, 8, 32)     128         add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_42 (LeakyReLU)      (None, 8, 8, 32)     0           batch_normalization_42[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_32 (Conv2D)              (None, 8, 8, 1)      289         leaky_re_lu_42[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_44 (BatchNo (None, 8, 8, 1)      4           conv2d_32[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_44 (LeakyReLU)      (None, 8, 8, 1)      0           batch_normalization_44[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_33 (Conv2D)              (None, 8, 8, 1)      10          leaky_re_lu_44[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_31 (Conv2D)              (None, 8, 8, 1)      33          leaky_re_lu_42[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_45 (BatchNo (None, 8, 8, 1)      4           conv2d_33[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_43 (BatchNo (None, 8, 8, 1)      4           conv2d_31[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_45 (LeakyReLU)      (None, 8, 8, 1)      0           batch_normalization_45[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_43 (LeakyReLU)      (None, 8, 8, 1)      0           batch_normalization_43[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 8, 8, 1)      0           leaky_re_lu_45[0][0]             \n",
      "                                                                 leaky_re_lu_43[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_46 (BatchNo (None, 8, 8, 1)      4           add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_46 (LeakyReLU)      (None, 8, 8, 1)      0           batch_normalization_46[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_Sigmoid (TensorFlow [(None, 8, 8, 1)]    0           leaky_re_lu_46[0][0]             \n",
      "==================================================================================================\n",
      "Total params: 78,408\n",
      "Trainable params: 76,992\n",
      "Non-trainable params: 1,416\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "temp.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}