autoencoder.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "autoencoder",
      "provenance": [],
      "authorship_tag": "ABX9TyMOIYDZesQBY/JrLFT+B26E",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cedro3/others2/blob/main/autoencoder.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u9tFQpan0D0B"
      },
      "source": [
        "# Keras AutoEncoder で異常検知をやってみる\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rB--3kVqzFuF"
      },
      "source": [
        "# tensolflowバージョン1.x　を選択\n",
        "%tensorflow_version 1.x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fP-s1A9tvU2x"
      },
      "source": [
        "from keras.layers import Input, Dense\n",
        "from keras.models import Model\n",
        "from keras.datasets import mnist\n",
        "from sklearn.model_selection import train_test_split\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        " \n",
        "# AutoEncoder ネットワーク構築\n",
        "encoding_dim = 32\n",
        "input_img = Input(shape=(784,))\n",
        "encoded = Dense(encoding_dim, activation='relu')(input_img)\n",
        "decoded = Dense(784, activation='sigmoid')(encoded)\n",
        "autoencoder = Model(input=input_img, output=decoded)\n",
        "autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy')\n",
        " \n",
        "# MNIST データ読み込み\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        " \n",
        "# データの前準備\n",
        "x_train, x_valid = train_test_split(x_train, test_size=0.175)\n",
        "x_train = x_train.astype('float32')/255.\n",
        "x_valid = x_valid.astype('float32')/255.\n",
        "x_test = x_test.astype('float32')/255.\n",
        "x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:])))\n",
        "x_valid = x_valid.reshape((len(x_valid), np.prod(x_valid.shape[1:])))\n",
        "x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:])))\n",
        " \n",
        "# 学習\n",
        "autoencoder.fit(x_train, x_train,\n",
        "                nb_epoch=50,\n",
        "                batch_size=256,\n",
        "                shuffle=True,\n",
        "                validation_data=(x_valid, x_valid))\n",
        " \n",
        "# 出力画像の取得\n",
        "decoded_imgs = autoencoder.predict(x_test)\n",
        " \n",
        "# サンプル画像表示\n",
        "n = 6\n",
        "plt.figure(figsize=(12, 4))\n",
        "for i in range(n):\n",
        "    # テスト画像を表示\n",
        "    ax = plt.subplot(2, n, i+1)\n",
        "    plt.imshow(x_test[i].reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(False)\n",
        "    ax.get_yaxis().set_visible(False)\n",
        " \n",
        "    # 出力画像を表示\n",
        "    ax = plt.subplot(2, n, i+1+n)\n",
        "    plt.imshow(decoded_imgs[i].reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(False)\n",
        "    ax.get_yaxis().set_visible(False)\n",
        "plt.savefig(\"result.png\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ffw3WSYxI1f"
      },
      "source": [
        "# 学習データを「１」のみにする\n",
        "x1 =[]\n",
        "for i in range(len(x_train)):\n",
        "    if y_train[i] == 1 :\n",
        "       x1.append(x_train[i])\n",
        "x_train = np.array(x1)\n",
        " \n",
        "# テストデータを「１」と「９」にする\n",
        "x2, y = [],[]\n",
        "for i in range(len(x_test)):\n",
        "    if y_test[i] == 1 or y_test[i] == 9 :\n",
        "       x2.append(x_test[i])\n",
        "       y.append(y_test[i])\n",
        "x_test = np.array(x2)\n",
        "y = np.array(y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MUywk8x_x72H"
      },
      "source": [
        "# 学習\n",
        "autoencoder.fit(x_train, x_train,\n",
        "                nb_epoch=300,\n",
        "                batch_size=256,\n",
        "                shuffle=True,\n",
        "                validation_data=(x_valid, x_valid))\n",
        " \n",
        "# 出力画像の取得\n",
        "decoded_imgs = autoencoder.predict(x_test)\n",
        " \n",
        "# サンプル画像表示\n",
        "n = 6\n",
        "plt.figure(figsize=(12, 4))\n",
        "for i in range(n):\n",
        "    # テスト画像を表示\n",
        "    ax = plt.subplot(2, n, i+1)\n",
        "    plt.imshow(x_test[i].reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(False)\n",
        "    ax.get_yaxis().set_visible(False)\n",
        " \n",
        "    # 出力画像を表示\n",
        "    ax = plt.subplot(2, n, i+1+n)\n",
        "    plt.imshow(decoded_imgs[i].reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(False)\n",
        "    ax.get_yaxis().set_visible(False)\n",
        "plt.savefig(\"result.png\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DNTiwMJWyqT4"
      },
      "source": [
        "# サンプル画像表示\n",
        "n = 6\n",
        "plt.figure(figsize=(12, 6))\n",
        "for i in range(n):\n",
        "    # テスト画像を表示\n",
        "    ax = plt.subplot(3, n, i+1)\n",
        "    plt.imshow(x_test[i].reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(False)\n",
        "    ax.get_yaxis().set_visible(False)\n",
        " \n",
        "    # 出力画像を表示\n",
        "    ax = plt.subplot(3, n, i+1+n)\n",
        "    plt.imshow(decoded_imgs[i].reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(False)\n",
        "    ax.get_yaxis().set_visible(False)\n",
        "    \n",
        "    # 入出力の差分画像を計算\n",
        "    diff_img = x_test[i] - decoded_imgs[i]\n",
        "    \n",
        "    # 入出力の差分数値を計算\n",
        "    diff = np.sum(np.abs(x_test[i]-decoded_imgs[i]))\n",
        "        \n",
        "    # 差分画像と差分数値の表示\n",
        "    ax = plt.subplot(3, n, i+1+n*2)\n",
        "    plt.imshow(diff_img.reshape(28, 28))\n",
        "    plt.gray()\n",
        "    ax.get_xaxis().set_visible(True)\n",
        "    ax.get_yaxis().set_visible(True) \n",
        "    ax.set_xlabel('score = '+str(diff))    \n",
        "        \n",
        "plt.savefig(\"result.png\")\n",
        "plt.show()\n",
        "plt.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3JiXcuqy3KA"
      },
      "source": [
        "# score を記録したファイルがあれば一端クリア\n",
        "import os\n",
        "if os.path.exists('scores_1.txt'):\n",
        "   os.remove('scores_1.txt')\n",
        "if os.path.exists('scores_9.txt'):\n",
        "   os.remove('scores_9.txt')\n",
        " \n",
        "# score の計算、結果のファイル保存\n",
        "for i in range(100):\n",
        "    score = np.sum(np.abs(x_test[i]-decoded_imgs[i]))\n",
        "    \n",
        "    if y[i] == 1:\n",
        "       with open('scores_1.txt','a') as f:\n",
        "            f.write(str(score)+'\\n')            \n",
        "    else:\n",
        "       with open('scores_9.txt','a') as f:\n",
        "            f.write(str(score)+'\\n')\n",
        " \n",
        "# ファイルを元にヒストグラムの表示\n",
        "import matplotlib.pyplot as plt\n",
        "import csv\n",
        "    \n",
        "x =[]\n",
        "with open('scores_1.txt', 'r') as f:\n",
        "     reader = csv.reader(f)\n",
        "     for row in reader:\n",
        "         row = int(float(row[0]))\n",
        "         x.append(row)\n",
        "y =[]\n",
        "with open('scores_9.txt', 'r') as f:\n",
        "     reader = csv.reader(f)\n",
        "     for row in reader:\n",
        "         row = int(float(row[0]))\n",
        "         y.append(row)\n",
        "             \n",
        "plt.title(\"Score Histgram\")\n",
        "plt.xlabel(\"Score\")\n",
        "plt.ylabel(\"freq\")\n",
        "plt.hist(x, bins=10, alpha=0.3, histtype='stepfilled', color='r', label=\"1\")\n",
        "plt.hist(y, bins=40, alpha=0.3, histtype='stepfilled', color='b', label='9')\n",
        "plt.legend(loc=1)\n",
        "plt.savefig(\"histgram.png\")\n",
        "plt.show()\n",
        "plt.close()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}