learn04_finetune+activation+initializer+optimizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# fine-tune"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 技巧\n",
    "- 使用之前的模型进行初始化\n",
    "- 步骤\n",
    "    - save model\n",
    "        - third party\n",
    "    - restore models\n",
    "        - from third party：使用第三方 迁移学习\n",
    "        - from myself：恢复原状态 断点恢复\n",
    "    - keep some layers fixed\n",
    "        - trainable作为某一层参数可以指定为False从而fixed\n",
    "    - 类型：ckp默认\n",
    " - 也可以从graph文件中直接导入 图结构以及参数\n",
    " - 也可以从数据文件中直接读取参数数据 导入到已有模型中作为初始化\n",
    " - 本节课是基本的save sess， restore sess 需要有完整图结构\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data prepocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['batches.meta', 'data_batch_1', 'data_batch_2', 'data_batch_3', 'data_batch_4', 'data_batch_5', 'readme.html', 'test_batch']\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "CIFAR_DIR = './cifar-10-batches-py'\n",
    "print(os.listdir(CIFAR_DIR))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 3072)\n",
      "(50000,)\n",
      "(10000, 3072)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "def load_data(filename):\n",
    "    \"\"\"read data from data file.\"\"\"\n",
    "    with open(filename, 'rb') as f:\n",
    "        data = pickle.load(f, encoding='bytes')\n",
    "        return data[b'data'], data[b'labels']\n",
    "\n",
    "# tensorflow.Dataset.\n",
    "class CifarData:\n",
    "    def __init__(self, filenames, need_shuffle):\n",
    "        all_data = []\n",
    "        all_labels = []\n",
    "        for filename in filenames:\n",
    "            data, labels = load_data(filename)\n",
    "            all_data.append(data)\n",
    "            all_labels.append(labels)\n",
    "        # hstack vstack 用于合并二维数组变成矩阵形式\n",
    "        self._data = np.vstack(all_data)\n",
    "        self._data = self._data / 127.5 - 1\n",
    "        # 小心\n",
    "        self._labels = np.hstack(all_labels)\n",
    "        print(self._data.shape)\n",
    "        print(self._labels.shape)\n",
    "        \n",
    "        self._num_examples = self._data.shape[0]\n",
    "        self._need_shuffle = need_shuffle\n",
    "        self._indicator = 0\n",
    "        if self._need_shuffle:\n",
    "            self._shuffle_data()\n",
    "            \n",
    "    def _shuffle_data(self):\n",
    "        # [0,1,2,3,4,5] -> [5,3,2,4,0,1]\n",
    "        p = np.random.permutation(self._num_examples)\n",
    "        self._data = self._data[p]\n",
    "        self._labels = self._labels[p]\n",
    "    \n",
    "    def next_batch(self, batch_size):\n",
    "        \"\"\"return batch_size examples as a batch.\"\"\"\n",
    "        end_indicator = self._indicator + batch_size\n",
    "        if end_indicator > self._num_examples:\n",
    "            if self._need_shuffle:\n",
    "                self._shuffle_data()\n",
    "                self._indicator = 0\n",
    "                end_indicator = batch_size\n",
    "            else:\n",
    "                raise Exception(\"have no more examples\")\n",
    "        if end_indicator > self._num_examples:\n",
    "            raise Exception(\"batch size is larger than all examples\")\n",
    "        batch_data = self._data[self._indicator: end_indicator]\n",
    "        batch_labels = self._labels[self._indicator: end_indicator]\n",
    "        self._indicator = end_indicator\n",
    "        return batch_data, batch_labels\n",
    "\n",
    "train_filenames = [os.path.join(CIFAR_DIR, 'data_batch_%d' % i) for i in range(1, 6)]\n",
    "test_filenames = [os.path.join(CIFAR_DIR, 'test_batch')]\n",
    "\n",
    "train_data = CifarData(train_filenames, True)\n",
    "test_data = CifarData(test_filenames, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model\n",
    "- 可以达到70%\n",
    "- 使用指定的activation\n",
    "    - 提升显著\n",
    "- 使用指定的kernel_initializer (可以通过variable_scope指定统一的):\n",
    "    - **tf....initializer** tf自带的initializer\n",
    "        - 默认是tf.glorot_uniform_initializer (None)\n",
    "    - **tf.keras.initializers.he_uniform** 也有一部分keras中的initializer\n",
    "- 在trainop中指定optimizer\n",
    "\n",
    "> 以上三者相互制约"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph() # 利用这个可清空defualt graph以及nodes\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 3072])\n",
    "y = tf.placeholder(tf.int64, [None])\n",
    "# [None], eg: [0,5,6,3]\n",
    "x_image = tf.reshape(x, [-1, 3, 32, 32])\n",
    "# 32*32\n",
    "x_image = tf.transpose(x_image, perm=[0, 2, 3, 1])\n",
    "\n",
    "def vggnet(x_image,activation,kernel_initializer=None):\n",
    "    # conv1: 神经元图， feature_map, 输出图像\n",
    "    conv1_1 = tf.layers.conv2d(x_image,\n",
    "                             32, # output channel number\n",
    "                             (3,3), # kernel size\n",
    "                             padding = 'same',\n",
    "                             activation = activation,\n",
    "                             kernel_initializer=kernel_initializer\n",
    "                             name = 'conv1_1')\n",
    "\n",
    "    conv1_2 = tf.layers.conv2d(conv1_1,\n",
    "                             32, # output channel number\n",
    "                             (3,3), # kernel size\n",
    "                             padding = 'same',\n",
    "                             activation = activation,\n",
    "                             kernel_initializer=kernel_initializer\n",
    "                             name = 'conv1_2')\n",
    "    # 16 * 16\n",
    "    pooling1 = tf.layers.max_pooling2d(conv1_2,\n",
    "                                       (2, 2), # kernel size\n",
    "                                       (2, 2), # stride\n",
    "                                       name = 'pool1')\n",
    "\n",
    "\n",
    "    conv2_1 = tf.layers.conv2d(pooling1,\n",
    "                             64, # output channel number\n",
    "                             (3,3), # kernel size\n",
    "                             padding = 'same',\n",
    "                             activation = activation,\n",
    "                             kernel_initializer=kernel_initializer\n",
    "                             name = 'conv2_1')\n",
    "    conv2_2 = tf.layers.conv2d(conv2_1,\n",
    "                             64, # output channel number\n",
    "                             (3,3), # kernel size\n",
    "                             padding = 'same',\n",
    "                             activation = activation,\n",
    "                             kernel_initializer=kernel_initializer\n",
    "                             name = 'conv2_2')\n",
    "    # 8 * 8\n",
    "    pooling2 = tf.layers.max_pooling2d(conv2_2,\n",
    "                                       (2, 2), # kernel size\n",
    "                                       (2, 2), # stride\n",
    "                                       name = 'pool2')\n",
    "\n",
    "    conv3_1 = tf.layers.conv2d(pooling2,\n",
    "                             128, # output channel number\n",
    "                             (3,3), # kernel size\n",
    "                             padding = 'same',\n",
    "                             activation = activation,\n",
    "                             kernel_initializer=kernel_initializer\n",
    "                             name = 'conv3_1')\n",
    "    conv3_2 = tf.layers.conv2d(conv3_1,\n",
    "                             128, # output channel number\n",
    "                             (3,3), # kernel size\n",
    "                             padding = 'same',\n",
    "                             activation = activation,\n",
    "                             kernel_initializer=kernel_initializer\n",
    "                             name = 'conv3_2')\n",
    "\n",
    "    # 4 * 4 * 32\n",
    "    pooling3 = tf.layers.max_pooling2d(conv3_2,\n",
    "                                       (2, 2), # kernel size\n",
    "                                       (2, 2), # stride\n",
    "                                       name = 'pool3')\n",
    "    # [None, 4 * 4 * 32]\n",
    "    flatten = tf.layers.flatten(pooling3)\n",
    "    return flatten\n",
    "\n",
    "flatten = vggnet(x_image,tf.nn.relu,None)\n",
    "# flatten = vggnet(x_image,tf.nn.relu,tf.truncated_normal_initializer)\n",
    "# flatten = vggnet(x_image,tf.nn.relu,tf.keras.initializers.he_uniform)\n",
    "# flatten = vggnet(x_image,tf.nn.selu,None)\n",
    "\n",
    "\n",
    "y_ = tf.layers.dense(flatten, 10)\n",
    "\n",
    "loss = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=y_)\n",
    "# y_ -> sofmax\n",
    "# y -> one_hot\n",
    "# loss = ylogy_\n",
    "\n",
    "# [1,0,1,1,1,0,0,0]\n",
    "correct_prediction = tf.equal(tf.argmax(y_, 1), y)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float64))\n",
    "\n",
    "with tf.name_scope('train_op'):\n",
    "    # 自适应的小心初始rate容易飞 但是分自适应的初始学习率也要小一点\n",
    "    train_op = tf.train.AdamOptimizer(learning_rate = -3,\n",
    "                                      beta1 = 0.9,\n",
    "                                      beta2 = 0.99).minimize(loss)\n",
    "#     train_op = tf.train.MomentumOptimizer(learning_rate = 1e-4,\n",
    "#                                           momentum=0.9).minimize(loss)\n",
    "#     train_op = tf.train.RMSPropOptimizer(learning_rate=1e-3,\n",
    "#                                          decay=0.9).minimize(loss)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## saver路径指定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_dir = './run'\n",
    "model_dir = os.path.join(run_dir,'model_save')\n",
    "if not os.path.exists(model_dir):\n",
    "    os.mkdir(model_dir)\n",
    "'''\n",
    "saver 初始化时候需要已经有需要保存的变量\n",
    "在调用save的时候只需要传入sess作为当前状态即可\n",
    "相当于先声明saver需要保存的量 就是saver功能\n",
    "之后具体执行saver\n",
    "可以指定每隔多少分钟/多少次保存一次\n",
    "注意默认只会保存后五次 保存为ckp checkpoint文件\n",
    "index data meta\n",
    "saver 也会在graph中占据名字 所以需要小心命名问题\n",
    "'''\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "def get_model_path(model_name):\n",
    "    model_path = os.path.join(model_dir,model_name)\n",
    "    return model_path\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train+saver操作 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./run\\model_save\\ckp-1000\n",
      "model restore from ./run\\model_save\\ckp-1000 successfully\n",
      "[Train] Step: 50, loss: 1.30102, acc: 0.50000\n",
      "[Train] Step: 100, loss: 1.82802, acc: 0.30000\n",
      "model save to ckp-0100\n",
      "[Train] Step: 150, loss: 1.47921, acc: 0.45000\n",
      "[Train] Step: 200, loss: 1.21130, acc: 0.60000\n",
      "(10000, 3072)\n",
      "(10000,)\n",
      "[Test ] Step: 200, acc: 0.51850\n",
      "model save to ckp-0200\n",
      "[Train] Step: 250, loss: 0.95369, acc: 0.60000\n",
      "[Train] Step: 300, loss: 1.34743, acc: 0.40000\n",
      "model save to ckp-0300\n",
      "[Train] Step: 350, loss: 1.31383, acc: 0.50000\n",
      "[Train] Step: 400, loss: 1.18771, acc: 0.60000\n",
      "(10000, 3072)\n",
      "(10000,)\n",
      "[Test ] Step: 400, acc: 0.54200\n",
      "model save to ckp-0400\n",
      "[Train] Step: 450, loss: 0.96996, acc: 0.65000\n",
      "[Train] Step: 500, loss: 1.47084, acc: 0.45000\n",
      "model save to ckp-0500\n",
      "[Train] Step: 550, loss: 1.45809, acc: 0.45000\n",
      "[Train] Step: 600, loss: 1.13738, acc: 0.50000\n",
      "(10000, 3072)\n",
      "(10000,)\n",
      "[Test ] Step: 600, acc: 0.58700\n",
      "model save to ckp-0600\n",
      "[Train] Step: 650, loss: 0.83219, acc: 0.75000\n",
      "[Train] Step: 700, loss: 1.39104, acc: 0.50000\n",
      "model save to ckp-0700\n",
      "[Train] Step: 750, loss: 1.18280, acc: 0.60000\n",
      "[Train] Step: 800, loss: 1.13406, acc: 0.55000\n",
      "(10000, 3072)\n",
      "(10000,)\n",
      "[Test ] Step: 800, acc: 0.59500\n",
      "model save to ckp-0800\n",
      "[Train] Step: 850, loss: 0.92947, acc: 0.65000\n",
      "[Train] Step: 900, loss: 1.59518, acc: 0.45000\n",
      "model save to ckp-0900\n",
      "[Train] Step: 950, loss: 0.91006, acc: 0.65000\n",
      "[Train] Step: 1000, loss: 1.32163, acc: 0.60000\n",
      "(10000, 3072)\n",
      "(10000,)\n",
      "[Test ] Step: 1000, acc: 0.60200\n"
     ]
    }
   ],
   "source": [
    "# cfg = tf.ConfigProto(allow_soft_placement=True )\n",
    "# cfg.gpu_options.allow_growth = True\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "batch_size = 20\n",
    "train_steps = 1000\n",
    "test_steps = 100\n",
    "# save control\n",
    "output_model_every_steps = 100\n",
    "\n",
    "# train 10k: 71.35%\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    # reload\n",
    "    # zrestore需要的是 model而不是具体文件名\n",
    "    model_path = get_model_path('ckp-1000')\n",
    "    if os.path.exists(model_path + '.index'):\n",
    "        saver.restore(sess,model_path)\n",
    "        print('model restore from %s successfully' % model_path)\n",
    "    else:\n",
    "        print('model restore from %s failedly' % model_path)\n",
    "        \n",
    "        \n",
    "    for i in range(train_steps):\n",
    "        batch_data, batch_labels = train_data.next_batch(batch_size)\n",
    "        loss_val, acc_val, _ = sess.run(\n",
    "            [loss, accuracy, train_op],\n",
    "            feed_dict={\n",
    "                x: batch_data,\n",
    "                y: batch_labels})\n",
    "        if (i+1) % 50 == 0:\n",
    "            print('[Train] Step: %d, loss: %4.5f, acc: %4.5f' \n",
    "                  % (i+1, loss_val, acc_val))\n",
    "        if (i+1) % 200 == 0:\n",
    "            test_data = CifarData(test_filenames, False)\n",
    "            all_test_acc_val = []\n",
    "            for j in range(test_steps):\n",
    "                test_batch_data, test_batch_labels \\\n",
    "                    = test_data.next_batch(batch_size)\n",
    "                test_acc_val = sess.run(\n",
    "                    [accuracy],\n",
    "                    feed_dict = {\n",
    "                        x: test_batch_data, \n",
    "                        y: test_batch_labels\n",
    "                    })\n",
    "                all_test_acc_val.append(test_acc_val)\n",
    "            test_acc = np.mean(all_test_acc_val)\n",
    "            print('[Test ] Step: %d, acc: %4.5f'\n",
    "                  % (i+1, test_acc))\n",
    "        if (i+1) % output_model_every_steps == 0:\n",
    "            saver.save(sess,\n",
    "                      os.path.join(model_dir,'ckp-%04d' % (i+1)))\n",
    "            # %04d 指定4宽度 不足用0补充\n",
    "            print('model save to ckp-%04d' % (i+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}