pix2pix_mono_seg_ensemble_discriminator_simplify.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TBD 1 : logger 추가\n",
    "# TBD 2: flask github 참고, method, class, 파일의 맨 윗단 마다 pydoc 형식으로 달기\n",
    "# TBD 3: 축약어를 자제할것 (특히 변수)\n",
    "\n",
    "# -------------------------\n",
    "#   To-do\n",
    "# -------------------------\n",
    "\n",
    "# 0. add data-setter, receiver system use python queue.Queue() class\n",
    "# this will resolve i/o bottleneck\n",
    "# 1. add logger\n",
    "# 2. make image drawer overlay mask on image\n",
    "# 3. make iterable\n",
    "# 4. make verbose turn on and off\n",
    "# 5. write pydoc\n",
    "\n",
    "# python basic Module\n",
    "import os\n",
    "import sys\n",
    "import types\n",
    "import progressbar\n",
    "from datetime import datetime\n",
    "from shutil import copy\n",
    "from pickle import dump, load\n",
    "\n",
    "# math, image, plot Module\n",
    "import numpy as np\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt  # TBD\n",
    "\n",
    "# tensorflow Module\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import backend as keras_backend\n",
    "from tensorflow.keras.layers import GaussianNoise\n",
    "from tensorflow.keras.layers import Input, Concatenate\n",
    "from tensorflow.keras.layers import Conv2D\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.optimizers import Nadam\n",
    "from tensorflow.keras.initializers import RandomNormal\n",
    "from tensorflow.keras import losses\n",
    "\n",
    "# keras segmentaion third-party Moudle\n",
    "import segmentation_models as sm\n",
    "\n",
    "# custom Module\n",
    "from gan_module.data_loader.medical_segmentation_data_loader import DataLoader\n",
    "from gan_module.data_loader.manage_batch import BatchQueueManager\n",
    "\n",
    "from gan_module.model.build_model import build_generator, build_discriminator\n",
    "from gan_module.util.draw_images import ImageDrawer\n",
    "from gan_module import custom_loss\n",
    "from gan_module.custom_loss import f1_loss_for_training, f1_score, dice_loss_for_training\n",
    "from gan_module.util.manage_learning_rate import learning_rate_scheduler\n",
    "from gan_module.config import CONFIG\n",
    "\n",
    "custom_loss.AXIS = [1, 2, 3]\n",
    "\n",
    "USE_GPU = True\n",
    "\n",
    "if USE_GPU:\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "    gpu_devices = tf.config.experimental.list_physical_devices(\"GPU\")\n",
    "    for device in gpu_devices:\n",
    "        tf.config.experimental.set_memory_growth(device, True)\n",
    "else:\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = '-1'\n",
    "\n",
    "KERNEL_INITIALIZER = RandomNormal(mean=0.0, stddev=0.02)\n",
    "\n",
    "\n",
    "class Pix2PixSegmentation:\n",
    "    def __init__(\n",
    "        self,\n",
    "        generator_power=32,\n",
    "        discriminator_power=32,\n",
    "        generator_learning_rate=1e-4,\n",
    "        discriminator_learning_rate=1e-4,\n",
    "        temp_weights_path=\".\",\n",
    "        draw_images=True,\n",
    "        on_memory=True,\n",
    "        code_test=False\n",
    "    ):\n",
    "        # Input shape\n",
    "        img_shape = CONFIG[\"img_shape\"]\n",
    "        input_channels = CONFIG[\"input_channels\"]\n",
    "        output_channels = CONFIG[\"output_channels\"]\n",
    "\n",
    "        input_img_shape = (*img_shape, input_channels)\n",
    "        output_img_shape = (*img_shape, output_channels)\n",
    "        # set parameter\n",
    "        self.start_epoch = None\n",
    "        self.history = {\"generator_loss\": [],\n",
    "                        \"f1_loss_train\": [], \"f1_score_train\": [],\n",
    "                        \"f1_loss_valid\": [], \"f1_score_valid\": []}\n",
    "        self.temp_weights_path = temp_weights_path\n",
    "\n",
    "        # Configure data loader\n",
    "        self.dataset_name = \"tumor\"\n",
    "        self.data_loader = DataLoader(\n",
    "            dataset_name=self.dataset_name,\n",
    "            config_dict=CONFIG,\n",
    "            on_memory=on_memory, code_test=code_test\n",
    "        )\n",
    "\n",
    "        self.loaded_data_index = {\n",
    "            \"train\": np.arange(self.data_loader.data_length[\"train\"]),\n",
    "            \"valid\": np.arange(self.data_loader.data_length[\"valid\"])\n",
    "        }\n",
    "\n",
    "        # Configure Image Drawer\n",
    "        self.draw_images = draw_images\n",
    "        self.image_drawer = ImageDrawer(\n",
    "            dataset_name=self.dataset_name, data_loader=self.data_loader\n",
    "        )\n",
    "        self.discriminator_loss_ratio = keras_backend.variable(2.75)\n",
    "        self.f1_loss_ratio = keras_backend.variable(97.25)\n",
    "        self.discriminator_acc_previous = 0.5\n",
    "        self.discriminator_acces = np.array(\n",
    "            [0.5 for _ in range(self.data_loader.data_length[\"train\"])])\n",
    "        self.discriminator_acces_previous = self.discriminator_acces.copy()\n",
    "        self.generator_losses = np.array(\n",
    "            [1 for _ in range(self.data_loader.data_length[\"train\"])], dtype=np.float32)\n",
    "        self.generator_losses_previous = self.generator_losses.copy()\n",
    "        self.generator_loss_min = 500\n",
    "        self.generator_loss_previous = 100\n",
    "        self.generator_loss_max_previous = 1000\n",
    "        self.generator_loss_max_min = 1000\n",
    "        self.generator_loss_min_min = 1000\n",
    "        self.weight_save_stack = False\n",
    "        self.training_end_stack = 0\n",
    "        # Calculate output shape of D (PatchGAN)\n",
    "        patch = 2 ** 3\n",
    "        self.disc_patch = (patch, patch, 1)\n",
    "        # Number of filters in the first layer of G and D\n",
    "        self.generator_learning_rate = generator_learning_rate\n",
    "        self.discriminator_learning_rate = discriminator_learning_rate\n",
    "        generator_optimizer = Nadam(self.generator_learning_rate)\n",
    "        discriminator_optimizer = Nadam(self.discriminator_learning_rate)\n",
    "        \n",
    "        # Build the generator\n",
    "        self.generator = build_generator(\n",
    "            input_img_shape=input_img_shape,\n",
    "            output_channels=output_channels,\n",
    "            generator_power=generator_power,\n",
    "            kernel_initializer=KERNEL_INITIALIZER,\n",
    "        )\n",
    "        # Build and compile the discriminator\n",
    "        self.discriminator = build_discriminator(\n",
    "            input_img_shape=input_img_shape,\n",
    "            output_img_shape=output_img_shape,\n",
    "            discriminator_power=discriminator_power,\n",
    "            kernel_initializer=KERNEL_INITIALIZER,\n",
    "        )\n",
    "        # self.discriminator = self.build_discriminator()\n",
    "        # 'mse' or tf.keras.losses.Huber() tf.keras.losses.LogCosh()\n",
    "        self.discriminator.compile(\n",
    "            loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1),\n",
    "            optimizer=discriminator_optimizer,\n",
    "            metrics=[\"accuracy\"],\n",
    "        )\n",
    "\n",
    "        # -------------------------\n",
    "        # Construct Computational\n",
    "        #   Graph of Generator\n",
    "        # -------------------------\n",
    "\n",
    "        # Input images and their conditioning images\n",
    "        original_img = Input(shape=input_img_shape)\n",
    "        masked_img = Input(shape=output_img_shape)\n",
    "        # generate image from original_img for target masked_img\n",
    "        model_masked_img = self.generator(original_img)\n",
    "\n",
    "        # For the combined model we will only train the generator\n",
    "        self.discriminator.trainable = False\n",
    "        # Discriminators determines validity of translated images / condition pairs\n",
    "        model_validity = self.discriminator([original_img, model_masked_img])\n",
    "        # give score by\n",
    "        # 1. how generator trick discriminator\n",
    "        # 2. how generator's image same as real photo in pixel\n",
    "        # 3. if you want change loss, see doc https://keras.io/api/losses/\n",
    "        # 4. 'mse', 'mae', tf.keras.losses.LogCosh(),  tf.keras.losses.Huber()\n",
    "        self.combined = Model(\n",
    "            inputs=[original_img, masked_img],\n",
    "            outputs=[model_validity, model_masked_img],\n",
    "        )\n",
    "        \n",
    "        self.combined.compile(\n",
    "            loss=[\n",
    "                tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1),\n",
    "                dice_loss_for_training\n",
    "            ],\n",
    "            loss_weights=[2.75, 97.25],\n",
    "            optimizer=generator_optimizer,\n",
    "        )\n",
    "\n",
    "    def train(self, epochs, batch_size=1, epoch_shuffle_term=10):\n",
    "\n",
    "        start_time = datetime.now()\n",
    "\n",
    "        # Adversarial loss ground truths\n",
    "        self.training_end_stack = 0\n",
    "        self.batch_size = batch_size\n",
    "        valid_patch = np.ones(\n",
    "            (self.batch_size, *self.disc_patch), dtype=np.float32)\n",
    "        fake_patch = np.zeros(\n",
    "            (self.batch_size, *self.disc_patch), dtype=np.float32)\n",
    "        # TBD : move batch_queue_manager to __init__\n",
    "        self.batch_queue_manager = BatchQueueManager(self)\n",
    "\n",
    "        if self.start_epoch is None:\n",
    "            self.start_epoch = 0\n",
    "        for epoch in range(self.start_epoch, epochs):\n",
    "            batch_i = 0\n",
    "\n",
    "            discriminator_losses = []\n",
    "            generator_loss_max_in_epoch = 0\n",
    "            generator_loss_min_in_epoch = 1000\n",
    "\n",
    "            # shffle data maybe\n",
    "            if epoch % epoch_shuffle_term == 0:\n",
    "                np.random.shuffle(self.loaded_data_index[\"train\"])\n",
    "\n",
    "            if self.discriminator_acc_previous < 0.75:\n",
    "                discriminator_learning = True\n",
    "                print(\"discriminator_learning is True\")\n",
    "            else:\n",
    "                discriminator_learning = False\n",
    "                print(\"discriminator_learning is False\")\n",
    "                \n",
    "            generator_current_learning_rate = learning_rate_scheduler(\n",
    "                self.generator_learning_rate,\n",
    "                epoch,\n",
    "            )\n",
    "            discriminator_current_learning_rate = learning_rate_scheduler(\n",
    "                self.discriminator_learning_rate,\n",
    "                epoch,\n",
    "            ) * (1 - self.discriminator_acc_previous)\n",
    "            keras_backend.set_value(\n",
    "                self.discriminator.optimizer.learning_rate,\n",
    "                discriminator_current_learning_rate,\n",
    "            )\n",
    "            keras_backend.set_value(\n",
    "                self.discriminator.optimizer.learning_rate,\n",
    "                discriminator_current_learning_rate,\n",
    "            )\n",
    "            keras_backend.set_value(\n",
    "                self.discriminator_loss_ratio,\n",
    "                keras_backend.variable(0.5) + 4.5 * self.discriminator_acc_previous,\n",
    "            )\n",
    "            keras_backend.set_value(\n",
    "                self.f1_loss_ratio,\n",
    "                keras_backend.variable(99.5) - 4.5 * self.discriminator_acc_previous\n",
    "            )\n",
    "            bar = progressbar.ProgressBar(\n",
    "                maxval=self.data_loader.data_length[\"train\"]).start()\n",
    "            \n",
    "            while batch_i + self.batch_size <= self.data_loader.data_length[\"train\"]:\n",
    "                bar.update(batch_i)\n",
    "\n",
    "                batch_index = self.loaded_data_index[\"train\"][batch_i: batch_i +\n",
    "                                                              self.batch_size]\n",
    "\n",
    "                original_img, masked_img = self.batch_queue_manager.get_batch(\n",
    "                    data_mode=\"train\")\n",
    "                model_masked_img = self.generator.predict_on_batch(\n",
    "                    original_img)\n",
    "\n",
    "                # ---------------------\n",
    "                #  Train Discriminator\n",
    "                # ---------------------\n",
    "                # Train Discriminator for valid image if it failed to detect fake image\n",
    "\n",
    "                if discriminator_learning:\n",
    "                    self.discriminator.train_on_batch(\n",
    "                        [original_img, masked_img], valid_patch)\n",
    "\n",
    "                batch_discriminator_acc_previous = np.mean(\n",
    "                    self.discriminator_acces_previous[batch_index])\n",
    "\n",
    "                # -----------------\n",
    "                #  Train Generator\n",
    "                # -----------------\n",
    "\n",
    "                # Train the generators\n",
    "\n",
    "                generator_loss = self.combined.train_on_batch(\n",
    "                    [original_img, masked_img],\n",
    "                    [valid_patch, masked_img]\n",
    "                )\n",
    "                # train discriminator for fake image if it failed to detect fake image\n",
    "                if (batch_discriminator_acc_previous <= 0.5 or epoch == 0) and discriminator_learning:\n",
    "                    discriminator_loss = self.discriminator.train_on_batch(\n",
    "                        [original_img, model_masked_img], fake_patch)\n",
    "                else:\n",
    "                    discriminator_loss = self.discriminator.test_on_batch(\n",
    "                        [original_img, model_masked_img], fake_patch)\n",
    "\n",
    "                self.discriminator_acces[batch_index] = discriminator_loss[1]\n",
    "                self.generator_losses[batch_index] = generator_loss[0]\n",
    "                \n",
    "\n",
    "                # 한 배치 끝\n",
    "                batch_i += self.batch_size\n",
    "            # training batch 사이클 끝\n",
    "            print(f\"epoch: {epoch}/{epochs}\")\n",
    "            print(f\"discriminator_acces : {np.mean(self.discriminator_acces)}\")\n",
    "            print(\n",
    "                f\"Mean generator_loss : {np.mean(self.generator_losses)}\")\n",
    "            print(f\"Max generator_loss : {np.max(self.generator_losses)}\")\n",
    "            print(f\"Min generator_loss : {np.min(self.generator_losses)}\")\n",
    "            print(\n",
    "                f\"generator loss decrease : {self.generator_loss_min - np.mean(self.generator_losses)}\"\n",
    "            )\n",
    "            print(\n",
    "                f\"generator loss decrease ratio : ({np.mean(self.generator_losses) / self.generator_loss_min})\"\n",
    "            )\n",
    "            print(\n",
    "                f\"Max generator loss decrease : {self.generator_loss_max_previous - np.max(self.generator_losses)}\"\n",
    "            )\n",
    "            print(\n",
    "                f\"current lowest generator loss : {self.generator_loss_min}\")\n",
    "            print(\n",
    "                f\"current Learning_rate : {generator_current_learning_rate}\")\n",
    "            print(f\"elapsed_time : {datetime.now() - start_time}\")\n",
    "            self.image_drawer.sample_images(\n",
    "                self.generator, epoch)\n",
    "            \n",
    "            if np.mean(self.generator_losses) / self.generator_loss_min < 1.1:\n",
    "                if self.generator_loss_min > np.mean(self.generator_losses):\n",
    "                    self.generator_loss_min = np.mean(self.generator_losses)\n",
    "                    self.generator_loss_max_min = generator_loss_max_in_epoch\n",
    "                    self.generator_loss_min_min = generator_loss_min_in_epoch\n",
    "                    self.weight_save_stack = True\n",
    "                    self.save_study_info()\n",
    "                    print(\"save weights\")\n",
    "\n",
    "                train_f1_loss_list = []\n",
    "                train_f1_score_list = []\n",
    "                for index in range(0, self.data_loader.data_length[\"train\"], self.batch_size):\n",
    "\n",
    "                    train_source_img, train_masked_img = self.batch_queue_manager.get_batch(\n",
    "                        data_mode=\"train\")\n",
    "\n",
    "                    train_model_masked_img = self.generator.predict_on_batch(\n",
    "                        train_source_img)\n",
    "\n",
    "                    train_f1_loss = f1_loss_for_training(\n",
    "                        train_masked_img, train_model_masked_img)\n",
    "                    train_f1_score = f1_score(\n",
    "                        train_masked_img, train_model_masked_img)\n",
    "                    train_f1_loss_list.append(train_f1_loss)\n",
    "                    train_f1_score_list.append(train_f1_score)\n",
    "\n",
    "                print(\n",
    "                    f\"train_f1_loss : {np.mean(train_f1_loss_list) * self.f1_loss_ratio}\")\n",
    "                print(f\"train_f1_score : {1 - np.mean(train_f1_loss_list)}\")\n",
    "                print(\n",
    "                    f\"train_f1_rounded_score : {np.mean(train_f1_score_list)}\")\n",
    "\n",
    "                valid_f1_loss_list = []\n",
    "                valid_f1_score_list = []\n",
    "                for index in range(0, self.data_loader.data_length[\"valid\"], self.batch_size):\n",
    "\n",
    "                    valid_source_img, valid_masked_img = self.batch_queue_manager.get_batch(\n",
    "                        data_mode=\"valid\")\n",
    "\n",
    "                    valid_model_masked_img = self.generator.predict_on_batch(\n",
    "                        valid_source_img)\n",
    "\n",
    "                    valid_f1_loss = f1_loss_for_training(\n",
    "                        valid_masked_img, valid_model_masked_img)\n",
    "                    valid_f1_score = f1_score(\n",
    "                        valid_masked_img, valid_model_masked_img)\n",
    "                    valid_f1_loss_list.append(valid_f1_loss)\n",
    "                    valid_f1_score_list.append(valid_f1_score)\n",
    "\n",
    "                print(\n",
    "                    f\"valid_f1_loss : {np.mean(valid_f1_loss_list) * self.f1_loss_ratio}\")\n",
    "                print(f\"valid_f1_score : {1 - np.mean(valid_f1_loss_list)}\")\n",
    "                print(\n",
    "                    f\"valid_f1_rounded_score : {np.mean(valid_f1_score_list)}\")\n",
    "            else:\n",
    "                print(\"loss decrease.\")\n",
    "                self.load_best_weights()\n",
    "\n",
    "            # previous generator_loss 갱신\n",
    "            self.generator_loss_previous = np.mean(self.generator_losses)\n",
    "            self.generator_loss_max_previous = generator_loss_max_in_epoch\n",
    "\n",
    "            if epoch >= 10 and self.weight_save_stack:\n",
    "                copy(\n",
    "                    \"generator.h5\",\n",
    "                    \"./generator_weights/generator_\"\n",
    "                    + str(round(self.generator_loss_min, 5))\n",
    "                    + \"_\"\n",
    "                    + str(round(self.generator_loss_max_min, 5))\n",
    "                    + \".h5\",\n",
    "                )\n",
    "                self.weight_save_stack = False\n",
    "\n",
    "            self.discriminator_acc_previous = np.mean(self.discriminator_acces)\n",
    "            self.discriminator_acces_previous = self.discriminator_acces.copy()\n",
    "            self.generator_losses_previous = self.generator_losses.copy()\n",
    "            # TBD: add epoch bigger than history length\n",
    "            self.history[\"generator_loss\"].append(\n",
    "                np.mean(self.generator_losses))\n",
    "            self.history[\"f1_loss_train\"].append(\n",
    "                np.mean(train_f1_loss_list))\n",
    "            self.history[\"f1_score_train\"].append(\n",
    "                np.mean(train_f1_score_list))\n",
    "            self.history[\"f1_loss_valid\"].append(\n",
    "                np.mean(valid_f1_loss_list))\n",
    "            self.history[\"f1_score_valid\"].append(\n",
    "                np.mean(valid_f1_score_list))\n",
    "\n",
    "    def get_info_folderPath(self):\n",
    "        return (\n",
    "            str(round(self.generator_loss_min, 5))\n",
    "            + \"_\"\n",
    "            + str(round(self.generator_loss_max_min, 5))\n",
    "        )\n",
    "\n",
    "    def save_study_info(self, path=None):\n",
    "\n",
    "        if path is None:\n",
    "            path = self.temp_weights_path\n",
    "\n",
    "        generator_weigth_path = os.path.join(path, \"generator.h5\")\n",
    "        discriminator_weigth_path = os.path.join(path, \"discriminator.h5\")\n",
    "        combined_weigth_path = os.path.join(path, \"combined.h5\")\n",
    "\n",
    "        self.generator.save_weights(generator_weigth_path)\n",
    "        self.discriminator.save_weights(discriminator_weigth_path)\n",
    "        self.combined.save_weights(combined_weigth_path)\n",
    "\n",
    "        study_info = {}\n",
    "        study_info[\"start_epoch\"] = self.start_epoch\n",
    "        study_info[\"train_loaded_data_index\"] = self.loaded_data_index[\"train\"]\n",
    "        study_info[\"generator_loss_min\"] = self.generator_loss_min\n",
    "        study_info[\"generator_loss_max_min\"] = self.generator_loss_max_min\n",
    "        study_info[\"generator_loss_min_min\"] = self.generator_loss_min_min\n",
    "        study_info[\"generator_losses_previous\"] = self.generator_losses_previous\n",
    "        study_info[\"discriminator_acces\"] = self.discriminator_acces\n",
    "        study_info[\"history\"] = self.history\n",
    "        file = open(path + \"/study_info.pkl\", \"wb\")\n",
    "        dump(study_info, file)\n",
    "        file.close()\n",
    "\n",
    "    def load_study_info(self):\n",
    "\n",
    "        self.generator.load_weights(\"generator.h5\")\n",
    "        self.discriminator.load_weights(\"discriminator.h5\")\n",
    "        self.combined.load_weights(\"combined.h5\")\n",
    "\n",
    "        if os.path.isfile(\"study_info.pkl\"):\n",
    "            file = open(\"study_info.pkl\", \"rb\")\n",
    "            study_info = load(file)\n",
    "            file.close()\n",
    "            self.start_epoch = study_info[\"start_epoch\"]\n",
    "            self.loaded_data_index[\"train\"] = study_info[\"train_loaded_data_index\"]\n",
    "            self.generator_loss_min = study_info[\"generator_loss_min\"]\n",
    "            self.generator_loss_max_min = study_info[\"generator_loss_max_min\"]\n",
    "            self.generator_loss_min_min = study_info[\"generator_loss_min_min\"]\n",
    "            self.generator_losses_previous = study_info[\"generator_losses_previous\"]\n",
    "            self.discriminator_acces = study_info[\"discriminator_acces\"]\n",
    "            self.history = study_info[\"history\"]\n",
    "        else:\n",
    "            print(\"No info pkl file!\")\n",
    "\n",
    "    def load_best_weights(self):\n",
    "        self.generator.load_weights(self.temp_weights_path + \"/generator.h5\")\n",
    "        self.discriminator.load_weights(\n",
    "            self.temp_weights_path + \"/discriminator.h5\")\n",
    "        self.combined.load_weights(self.temp_weights_path + \"/combined.h5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "generator_lr = 1e-3\n",
    "discriminator_lr = 5e-4\n",
    "batch_size = 4\n",
    "g_lr = generator_lr * batch_size\n",
    "d_lr = discriminator_lr * batch_size\n",
    "gan = Pix2PixSegmentation(generator_power=4, discriminator_power=4, \n",
    "                          generator_learning_rate=g_lr, discriminator_learning_rate=d_lr,\n",
    "                          on_memory=True, code_test=False, draw_images=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 74.94190216064453\n",
      "Max generator_loss : 98.4444351196289\n",
      "Min generator_loss : 33.604515075683594\n",
      "generator loss decrease : 425.05809783935547\n",
      "generator loss decrease ratio : (0.14988380432128906)\n",
      "Max generator loss decrease : 901.5555648803711\n",
      "current lowest generator loss : 500\n",
      "current Learning_rate = 0.0004\n",
      "elapsed_time = 0:14:21.545612\n",
      "save weights\n",
      "train_f1_loss : 69.32411193847656\n",
      "train_f1_score : 0.2871556878089905\n",
      "train_f1_rounded_score : 0.45159637928009033\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 70.49111938476562\n",
      "valid_f1_score : 0.27515560388565063\n",
      "valid_f1_rounded_score : 0.4749564230442047\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.6967741935483871\n",
      "Mean generator_loss : 65.20829772949219\n",
      "Max generator_loss : 97.87315368652344\n",
      "Min generator_loss : 29.088171005249023\n",
      "generator loss decrease : 9.733604431152344\n",
      "generator loss decrease ratio : (0.8701180219650269)\n",
      "Max generator loss decrease : -97.87315368652344\n",
      "current lowest generator loss : 74.94190216064453\n",
      "current Learning_rate = 0.0008\n",
      "elapsed_time = 0:30:00.369420\n",
      "save weights\n",
      "train_f1_loss : 57.73124694824219\n",
      "train_f1_score : 0.4197864532470703\n",
      "train_f1_rounded_score : 0.5426164865493774\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 53.79901123046875\n",
      "valid_f1_score : 0.45930641889572144\n",
      "valid_f1_rounded_score : 0.5908359885215759\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.30709677419354836\n",
      "Mean generator_loss : 53.720401763916016\n",
      "Max generator_loss : 95.91558074951172\n",
      "Min generator_loss : 21.729328155517578\n",
      "generator loss decrease : 11.487895965576172\n",
      "generator loss decrease ratio : (0.8238276839256287)\n",
      "Max generator loss decrease : -95.91558074951172\n",
      "current lowest generator loss : 65.20829772949219\n",
      "current Learning_rate = 0.0012\n",
      "elapsed_time = 0:44:26.378938\n",
      "save weights\n",
      "train_f1_loss : 54.547996520996094\n",
      "train_f1_score : 0.4339410662651062\n",
      "train_f1_rounded_score : 0.5150406360626221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 50.20939254760742\n",
      "valid_f1_score : 0.4789639115333557\n",
      "valid_f1_rounded_score : 0.5612741708755493\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 46.4517707824707\n",
      "Max generator_loss : 91.82311248779297\n",
      "Min generator_loss : 18.01472282409668\n",
      "generator loss decrease : 7.2686309814453125\n",
      "generator loss decrease ratio : (0.864695131778717)\n",
      "Max generator loss decrease : -91.82311248779297\n",
      "current lowest generator loss : 53.720401763916016\n",
      "current Learning_rate = 0.0016\n",
      "elapsed_time = 0:59:30.068289\n",
      "save weights\n",
      "train_f1_loss : 46.740234375\n",
      "train_f1_score : 0.5236327350139618\n",
      "train_f1_rounded_score : 0.5337313413619995\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 49.28897476196289\n",
      "valid_f1_score : 0.4976564645767212\n",
      "valid_f1_rounded_score : 0.5075386166572571\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 43.04558563232422\n",
      "Max generator_loss : 88.73362731933594\n",
      "Min generator_loss : 16.87168312072754\n",
      "generator loss decrease : 3.4061851501464844\n",
      "generator loss decrease ratio : (0.926672637462616)\n",
      "Max generator loss decrease : -88.73362731933594\n",
      "current lowest generator loss : 46.4517707824707\n",
      "current Learning_rate = 0.002\n",
      "elapsed_time = 1:15:06.478829\n",
      "save weights\n",
      "train_f1_loss : 44.0322380065918\n",
      "train_f1_score : 0.5574649572372437\n",
      "train_f1_rounded_score : 0.5695021748542786\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 43.02777862548828\n",
      "valid_f1_score : 0.5675600171089172\n",
      "valid_f1_rounded_score : 0.5796874761581421\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 41.48855972290039\n",
      "Max generator_loss : 85.21238708496094\n",
      "Min generator_loss : 17.278059005737305\n",
      "generator loss decrease : 1.5570259094238281\n",
      "generator loss decrease ratio : (0.963828444480896)\n",
      "Max generator loss decrease : -85.21238708496094\n",
      "current lowest generator loss : 43.04558563232422\n",
      "current Learning_rate = 0.0024\n",
      "elapsed_time = 1:30:41.999366\n",
      "save weights\n",
      "train_f1_loss : 40.948177337646484\n",
      "train_f1_score : 0.5884605348110199\n",
      "train_f1_rounded_score : 0.6027809381484985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 38.86347961425781\n",
      "valid_f1_score : 0.609412282705307\n",
      "valid_f1_rounded_score : 0.6236271262168884\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 40.5056266784668\n",
      "Max generator_loss : 84.1563491821289\n",
      "Min generator_loss : 16.4763126373291\n",
      "generator loss decrease : 0.9829330444335938\n",
      "generator loss decrease ratio : (0.9763083457946777)\n",
      "Max generator loss decrease : -84.1563491821289\n",
      "current lowest generator loss : 41.48855972290039\n",
      "current Learning_rate = 0.0028\n",
      "elapsed_time = 1:46:12.435424\n",
      "save weights\n",
      "train_f1_loss : 37.40164566040039\n",
      "train_f1_score : 0.6241040527820587\n",
      "train_f1_rounded_score : 0.6315369009971619\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 37.47148513793945\n",
      "valid_f1_score : 0.6234021782875061\n",
      "valid_f1_rounded_score : 0.6314181685447693\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.46\n",
      "Mean generator_loss : 40.062278747558594\n",
      "Max generator_loss : 83.97616577148438\n",
      "Min generator_loss : 16.56574058532715\n",
      "generator loss decrease : 0.4433479309082031\n",
      "generator loss decrease ratio : (0.9890546798706055)\n",
      "Max generator loss decrease : -83.97616577148438\n",
      "current lowest generator loss : 40.5056266784668\n",
      "current Learning_rate = 0.0032\n",
      "elapsed_time = 2:01:48.861369\n",
      "save weights\n",
      "train_f1_loss : 38.89118957519531\n",
      "train_f1_score : 0.609133780002594\n",
      "train_f1_rounded_score : 0.6137192845344543\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 39.149295806884766\n",
      "valid_f1_score : 0.6065397560596466\n",
      "valid_f1_rounded_score : 0.6119461059570312\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.5432258064516129\n",
      "Mean generator_loss : 39.23699188232422\n",
      "Max generator_loss : 82.61573028564453\n",
      "Min generator_loss : 16.774850845336914\n",
      "generator loss decrease : 0.825286865234375\n",
      "generator loss decrease ratio : (0.9793999195098877)\n",
      "Max generator loss decrease : -82.61573028564453\n",
      "current lowest generator loss : 40.062278747558594\n",
      "current Learning_rate = 0.0036000000000000003\n",
      "elapsed_time = 2:16:36.847722\n",
      "save weights\n",
      "train_f1_loss : 37.114871978759766\n",
      "train_f1_score : 0.6190611720085144\n",
      "train_f1_rounded_score : 0.6222674250602722\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 36.333648681640625\n",
      "valid_f1_score : 0.6270794570446014\n",
      "valid_f1_rounded_score : 0.6306325197219849\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 38.10550308227539\n",
      "Max generator_loss : 83.02185821533203\n",
      "Min generator_loss : 15.221029281616211\n",
      "generator loss decrease : 1.1314888000488281\n",
      "generator loss decrease ratio : (0.9711626768112183)\n",
      "Max generator loss decrease : -83.02185821533203\n",
      "current lowest generator loss : 39.23699188232422\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 2:31:17.039243\n",
      "save weights\n",
      "train_f1_loss : 42.891300201416016\n",
      "train_f1_score : 0.558074414730072\n",
      "train_f1_rounded_score : 0.5607362389564514\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 43.48058319091797\n",
      "valid_f1_score : 0.5520028173923492\n",
      "valid_f1_rounded_score : 0.5550512671470642\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 38.84111404418945\n",
      "Max generator_loss : 88.78291320800781\n",
      "Min generator_loss : 14.766106605529785\n",
      "generator loss decrease : -0.7356109619140625\n",
      "generator loss decrease ratio : (1.019304633140564)\n",
      "Max generator loss decrease : -88.78291320800781\n",
      "current lowest generator loss : 38.10550308227539\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 2:46:56.403585\n",
      "train_f1_loss : 38.48203659057617\n",
      "train_f1_score : 0.6132458746433258\n",
      "train_f1_rounded_score : 0.6148419976234436\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 36.97917175292969\n",
      "valid_f1_score : 0.6283500492572784\n",
      "valid_f1_rounded_score : 0.6303502321243286\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 38.49127197265625\n",
      "Max generator_loss : 81.8115005493164\n",
      "Min generator_loss : 16.080625534057617\n",
      "generator loss decrease : -0.3857688903808594\n",
      "generator loss decrease ratio : (1.0101237297058105)\n",
      "Max generator loss decrease : -81.8115005493164\n",
      "current lowest generator loss : 38.10550308227539\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 3:02:33.339622\n",
      "train_f1_loss : 36.887577056884766\n",
      "train_f1_score : 0.6292705833911896\n",
      "train_f1_rounded_score : 0.6358424425125122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 36.565513610839844\n",
      "valid_f1_score : 0.6325074136257172\n",
      "valid_f1_rounded_score : 0.6394422650337219\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.5496774193548387\n",
      "Mean generator_loss : 38.46792221069336\n",
      "Max generator_loss : 81.98648071289062\n",
      "Min generator_loss : 14.758145332336426\n",
      "generator loss decrease : -0.36241912841796875\n",
      "generator loss decrease ratio : (1.0095109939575195)\n",
      "Max generator loss decrease : -81.98648071289062\n",
      "current lowest generator loss : 38.10550308227539\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 3:18:22.577530\n",
      "train_f1_loss : 36.89860916137695\n",
      "train_f1_score : 0.629159688949585\n",
      "train_f1_rounded_score : 0.6312313079833984\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 37.088924407958984\n",
      "valid_f1_score : 0.6272470057010651\n",
      "valid_f1_rounded_score : 0.629781186580658\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.4541935483870968\n",
      "Mean generator_loss : 37.730690002441406\n",
      "Max generator_loss : 82.279296875\n",
      "Min generator_loss : 13.89442253112793\n",
      "generator loss decrease : 0.3748130798339844\n",
      "generator loss decrease ratio : (0.9901638031005859)\n",
      "Max generator loss decrease : -82.279296875\n",
      "current lowest generator loss : 38.10550308227539\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 3:32:57.758637\n",
      "save weights\n",
      "train_f1_loss : 37.67360305786133\n",
      "train_f1_score : 0.6117182075977325\n",
      "train_f1_rounded_score : 0.6173651814460754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 38.531890869140625\n",
      "valid_f1_score : 0.6028723120689392\n",
      "valid_f1_rounded_score : 0.609767735004425\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 37.06138610839844\n",
      "Max generator_loss : 79.67748260498047\n",
      "Min generator_loss : 12.878314971923828\n",
      "generator loss decrease : 0.6693038940429688\n",
      "generator loss decrease ratio : (0.9822610020637512)\n",
      "Max generator loss decrease : -79.67748260498047\n",
      "current lowest generator loss : 37.730690002441406\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 3:47:45.474044\n",
      "save weights\n",
      "train_f1_loss : 37.17893600463867\n",
      "train_f1_score : 0.6185059249401093\n",
      "train_f1_rounded_score : 0.6218627691268921\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 35.476722717285156\n",
      "valid_f1_score : 0.6359723806381226\n",
      "valid_f1_rounded_score : 0.6396198868751526\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 37.77935028076172\n",
      "Max generator_loss : 82.53418731689453\n",
      "Min generator_loss : 13.623322486877441\n",
      "generator loss decrease : -0.7179641723632812\n",
      "generator loss decrease ratio : (1.0193723440170288)\n",
      "Max generator loss decrease : -82.53418731689453\n",
      "current lowest generator loss : 37.06138610839844\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 4:03:16.491581\n",
      "train_f1_loss : 38.15963363647461\n",
      "train_f1_score : 0.6164861023426056\n",
      "train_f1_rounded_score : 0.61678546667099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 34.76260757446289\n",
      "valid_f1_score : 0.6506270468235016\n",
      "valid_f1_rounded_score : 0.6513795852661133\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.0\n",
      "Mean generator_loss : 37.814212799072266\n",
      "Max generator_loss : 80.75830078125\n",
      "Min generator_loss : 14.256397247314453\n",
      "generator loss decrease : -0.7528266906738281\n",
      "generator loss decrease ratio : (1.020313024520874)\n",
      "Max generator loss decrease : -80.75830078125\n",
      "current lowest generator loss : 37.06138610839844\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 4:18:45.328652\n",
      "train_f1_loss : 35.217227935791016\n",
      "train_f1_score : 0.6460579931735992\n",
      "train_f1_rounded_score : 0.6477833390235901\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 35.145912170410156\n",
      "valid_f1_score : 0.6467747390270233\n",
      "valid_f1_rounded_score : 0.6487274169921875\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.6432258064516129\n",
      "Mean generator_loss : 37.7579231262207\n",
      "Max generator_loss : 80.37793731689453\n",
      "Min generator_loss : 14.923700332641602\n",
      "generator loss decrease : -0.6965370178222656\n",
      "generator loss decrease ratio : (1.0187941789627075)\n",
      "Max generator loss decrease : -80.37793731689453\n",
      "current lowest generator loss : 37.06138610839844\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 4:34:13.908293\n",
      "train_f1_loss : 37.05307388305664\n",
      "train_f1_score : 0.6276073157787323\n",
      "train_f1_rounded_score : 0.6301535964012146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 36.20487594604492\n",
      "valid_f1_score : 0.6361319124698639\n",
      "valid_f1_rounded_score : 0.6391090154647827\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% |####################################################################### |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "discriminator_acces : 0.36129032258064514\n",
      "Mean generator_loss : 37.18083190917969\n",
      "Max generator_loss : 81.75773620605469\n",
      "Min generator_loss : 13.317173957824707\n",
      "generator loss decrease : -0.11944580078125\n",
      "generator loss decrease ratio : (1.003222942352295)\n",
      "Max generator loss decrease : -81.75773620605469\n",
      "current lowest generator loss : 37.06138610839844\n",
      "current Learning_rate = 0.004\n",
      "elapsed_time = 4:48:39.588826\n",
      "train_f1_loss : 37.68098068237305\n",
      "train_f1_score : 0.6099498867988586\n",
      "train_f1_rounded_score : 0.611587405204773\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% |                                                                        |\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_f1_loss : 37.35661697387695\n",
      "valid_f1_score : 0.6133075058460236\n",
      "valid_f1_rounded_score : 0.6152362823486328\n",
      "discriminator_learning is True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54% |#######################################                                 |\r"
     ]
    }
   ],
   "source": [
    "#gan.load_study_info()\n",
    "gan.train(epochs=325, batch_size=batch_size, epoch_shuffle_term=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[60.188560485839844, 0.6891670823097229, 0.5914608836174011]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "original_img = np.ones((1,512,512,3))\n",
    "masked_img = np.ones((1,512,512,1))\n",
    "valid_patch = np.ones((1,8,8,1))\n",
    "\n",
    "gan.combined.test_on_batch(\n",
    "    [original_img, masked_img],\n",
    "    [valid_patch, masked_img],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=100.0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gan.f1_loss_ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=0.5>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gan.discriminator_loss_ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[60.188560485839844, 0.6891670823097229, 0.5914608836174011]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59.19494146108627"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0.6891670823097229 * 0.5 + 0.5914608836174011 * 99.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3.1\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-35-f92eaba7459e>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-35-f92eaba7459e>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    gan.discriminator.optimizer.\u001b[0m\n\u001b[1;37m                                ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "gan.discriminator.optimizer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 8 calls to <function Model.make_train_function.<locals>.train_function at 0x00000288678493A0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 8 calls to <function Model.make_train_function.<locals>.train_function at 0x00000288678493A0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elapsed time : 4.586500883102417\n",
      "elapsed time : 0.29304075241088867\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "temp_source = gan.original_img\n",
    "temp_mask = gan.masked_img\n",
    "\n",
    "start_time = time.time()\n",
    "gan.generator.train_on_batch(temp_source, temp_mask)\n",
    "print(f\"elapsed time : {time.time() - start_time}\")\n",
    "\n",
    "temp_source = tf.convert_to_tensor(temp_source)\n",
    "temp_mask = tf.convert_to_tensor(temp_mask)\n",
    "\n",
    "start_time = time.time()\n",
    "gan.generator.train_on_batch(temp_source, temp_mask)\n",
    "print(f\"elapsed time : {time.time() - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-cba4fde71471>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m     batch_index = gan.train_loaded_data_index[batch_i: batch_i +\n\u001b[0;32m      6\u001b[0m                                                gan.batch_size]\n\u001b[1;32m----> 7\u001b[1;33m     original_img, masked_img = gan.data_loader.get_data(\n\u001b[0m\u001b[0;32m      8\u001b[0m         data_mode=\"train\", index=batch_index)\n",
      "\u001b[1;32m~\\Desktop\\Works\\jupyterlab\\의료데이터(pix2pix)_size_20_v0.2\\data_loader\\medical_segmentation_data_loader_v2.py\u001b[0m in \u001b[0;36mget_data\u001b[1;34m(self, data_mode, index)\u001b[0m\n\u001b[0;32m     86\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m                     data_tuple = (\n\u001b[1;32m---> 88\u001b[1;33m                         self.train_loaded_data[0][index], self.train_loaded_data[1][index])\n\u001b[0m\u001b[0;32m     89\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mdata_mode\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"valid\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     90\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "batch_i = 0\n",
    "\n",
    "while batch_i + gan.batch_size <= gan.data_loader.train_data_length:\n",
    "\n",
    "    batch_index = gan.train_loaded_data_index[batch_i: batch_i +\n",
    "                                               gan.batch_size]\n",
    "    original_img, masked_img = gan.data_loader.get_data(\n",
    "        data_mode=\"train\", index=batch_index)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gan.data_loader.train_data_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Iterator : 260초\n",
    "# Queue Iterator : 200초"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elapsed time : 204.91554951667786\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import threading\n",
    "from queue import Queue\n",
    "\n",
    "ITER_NUM = 620\n",
    "batch_size = 10\n",
    "\n",
    "gan.generator.compile(\n",
    "    loss=sm.losses.BinaryFocalLoss(),\n",
    "    optimizer=Nadam(gan.generator_learning_rate),\n",
    "    metrics=[\"accuracy\"],\n",
    ")\n",
    "\n",
    "def batch_setter(queue):\n",
    "    batch_i = 0\n",
    "    count = 0\n",
    "    while batch_i + gan.batch_size <= gan.data_loader.train_data_length and count < ITER_NUM:\n",
    "        \n",
    "        batch_index = gan.train_loaded_data_index[batch_i: batch_i +\n",
    "                                                   gan.batch_size]        \n",
    "        \n",
    "        batch_tuple = gan.data_loader.get_data(\n",
    "        data_mode=\"train\", index=batch_index)\n",
    "\n",
    "        queue.put(batch_tuple)\n",
    "        queue.join()\n",
    "        count += 1\n",
    "    \n",
    "def batch_getter(queue):\n",
    "    \n",
    "    original_img, masked_img = queue.get()\n",
    "    tensor_original_img = tf.convert_to_tensor(original_img)\n",
    "    tensor_masked_img = tf.convert_to_tensor(masked_img)\n",
    "    queue.task_done()\n",
    "    \n",
    "    return tensor_original_img, tensor_masked_img\n",
    "    \n",
    "def batch_trainer(original_img, masked_img):\n",
    "    \n",
    "    gan.generator.train_on_batch(temp_source, temp_mask)\n",
    "\n",
    "q = Queue()\n",
    "\n",
    "setter = threading.Thread(target=batch_setter, args=(q,),daemon=True)\n",
    "setter.start()\n",
    "start_time = time.time()\n",
    "for i in range(ITER_NUM):\n",
    "    tensor_original_img, tensor_masked_img = batch_getter(q)\n",
    "    \n",
    "    gan.generator.train_on_batch(tensor_original_img, tensor_masked_img)\n",
    "print(f\"elapsed time : {time.time() - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "elapsed time : 260.5594081878662\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "batch_i = 0\n",
    "count = 0\n",
    "while batch_i + gan.batch_size <= gan.data_loader.train_data_length and count < ITER_NUM:\n",
    "\n",
    "    batch_index = gan.train_loaded_data_index[batch_i: batch_i +\n",
    "                                               gan.batch_size]        \n",
    "    batch_tuple = gan.data_loader.get_data(\n",
    "    data_mode=\"train\", index=batch_index)\n",
    "    \n",
    "    gan.generator.train_on_batch(*batch_tuple)\n",
    "    \n",
    "    count += 1\n",
    "print(f\"elapsed time : {time.time() - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'tensorflow.python.framework.ops.EagerTensor'>\n",
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "\n",
    "temp = tensor_masked_img\n",
    "print(type(temp))\n",
    "print(type(temp.numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(temp.numpy(), tf.Tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gan_module.model.build_model import build_dual_discriminator\n",
    "\n",
    "temp = build_dual_discriminator(\n",
    "            input_img_shape=(512,512,3),\n",
    "            output_img_shape=(512,512,1),\n",
    "            discriminator_power=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.optimizers import Nadam\n",
    "\n",
    "temp.compile(\n",
    "    loss=[\n",
    "        tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1),\n",
    "        tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1)\n",
    "    ],\n",
    "    optimizer=Nadam(),\n",
    "    metrics=[\"accuracy\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "UnknownError",
     "evalue": " Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.\n\t [[node functional_1/conv2d_34/Conv2D (defined at <ipython-input-3-d8b1761e412e>:7) ]] [Op:__inference_test_function_13463]\n\nFunction call stack:\ntest_function\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnknownError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-d8b1761e412e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mpatch_mockup\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mones\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m \u001b[0mtemp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtest_on_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mimage_mockup\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask_mockup\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mpatch_mockup\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpatch_mockup\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mtest_on_batch\u001b[1;34m(self, x, y, sample_weight, reset_metrics, return_dict)\u001b[0m\n\u001b[0;32m   1752\u001b[0m                                                     y, sample_weight)\n\u001b[0;32m   1753\u001b[0m       \u001b[0mtest_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_test_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1754\u001b[1;33m       \u001b[0mlogs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtest_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1755\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1756\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    778\u001b[0m       \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    779\u001b[0m         \u001b[0mcompiler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"nonXla\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 780\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    781\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    782\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    838\u001b[0m         \u001b[1;31m# Lifting succeeded, so variables are initialized and we can run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    839\u001b[0m         \u001b[1;31m# stateless function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 840\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    841\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    842\u001b[0m       \u001b[0mcanon_args\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcanon_kwds\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2827\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2828\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2829\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2830\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2831\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1841\u001b[0m       \u001b[0;31m`\u001b[0m\u001b[0margs\u001b[0m\u001b[0;31m`\u001b[0m \u001b[1;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1842\u001b[0m     \"\"\"\n\u001b[1;32m-> 1843\u001b[1;33m     return self._call_flat(\n\u001b[0m\u001b[0;32m   1844\u001b[0m         [t for t in nest.flatten((args, kwargs), expand_composites=True)\n\u001b[0;32m   1845\u001b[0m          if isinstance(t, (ops.Tensor,\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1921\u001b[0m         and executing_eagerly):\n\u001b[0;32m   1922\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1923\u001b[1;33m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[0;32m   1924\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0;32m   1925\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    543\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 545\u001b[1;33m           outputs = execute.execute(\n\u001b[0m\u001b[0;32m    546\u001b[0m               \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m               \u001b[0mnum_outputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     57\u001b[0m   \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 59\u001b[1;33m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[0;32m     60\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mUnknownError\u001b[0m:  Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.\n\t [[node functional_1/conv2d_34/Conv2D (defined at <ipython-input-3-d8b1761e412e>:7) ]] [Op:__inference_test_function_13463]\n\nFunction call stack:\ntest_function\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "image_mockup = np.ones((1,512,512,3))\n",
    "mask_mockup = np.ones((1,512,512,1))\n",
    "patch_mockup = np.ones((1,8,8,1))\n",
    "\n",
    "temp.test_on_batch([image_mockup, mask_mockup], [patch_mockup, patch_mockup])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}