rudimentary_oop_neural_net.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wLYCuk5RBxfb"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "\n",
        "class Sigmoid:\n",
        "    def f(self, x):\n",
        "        return 1 / (1 + np.exp(-x))\n",
        "\n",
        "    def d_f(self, x):\n",
        "        f_x = self.f(x)\n",
        "        return f_x * (1 - f_x)\n",
        "\n",
        "def hyptan(x):\n",
        "    e_2w = np.exp(2 * x)\n",
        "    return (e_2w - 1) / (e_2w + 1)\n",
        "\n",
        "def silu(x):\n",
        "    return x / (1 + np.exp(-x))\n",
        "\n",
        "def relu(x):\n",
        "    return np.maximum(0, x)\n",
        "\n",
        "class MeanSquared:\n",
        "    def f(self, true_output, expected_output):\n",
        "        return (true_output - expected_output)**2\n",
        "\n",
        "    def d_f(self, true_output, expected_output):\n",
        "        return 2 * (true_output - expected_output)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Layer:\n",
        "    def __init__(self, nodes_in, nodes_out, activation = Sigmoid(), error = MeanSquared()):\n",
        "        self._nodes_in = nodes_in\n",
        "        self._nodes_out = nodes_out\n",
        "\n",
        "        self._inputs = np.zeros(shape = nodes_in)\n",
        "\n",
        "        self._activation = activation\n",
        "        self._error = error\n",
        "\n",
        "        self._weights =  np.random.uniform(-25., 25., size = (nodes_in, nodes_out))\n",
        "        self._biases = np.zeros(shape = nodes_out)\n",
        "\n",
        "        self._gradient_w = np.zeros(shape = (nodes_in, nodes_out))\n",
        "        self._gradient_b = np.zeros(shape = nodes_out)\n",
        "\n",
        "        self._z_vals = np.zeros(shape = nodes_out)\n",
        "        self._activation_vals = np.zeros(shape = nodes_out)\n",
        "\n",
        "    def __call__(self, inputs):\n",
        "        self._inputs = inputs\n",
        "\n",
        "        for out_node in range(self._nodes_out):\n",
        "            self._z_vals[out_node] = self._biases[out_node] + np.dot(inputs, self._weights[..., out_node])\n",
        "            self._activation_vals[out_node] = self._activation.f(self._z_vals[out_node])\n",
        "\n",
        "        return self._activation_vals\n",
        "\n",
        "    def copy(self, weights, biases):\n",
        "      self._weights = weights\n",
        "      self._biases = biases\n",
        "\n",
        "    def _calc_output_node_vals(self, expected_output):\n",
        "        node_values = np.zeros(shape = self._nodes_out)\n",
        "\n",
        "        for out_node in range(self._nodes_out):\n",
        "            cost_derivative = self._error.d_f(self._activation_vals[out_node], expected_output[out_node])\n",
        "            activation_val_derivative = self._activation.d_f(self._z_vals[out_node])\n",
        "            node_values[out_node] = cost_derivative * activation_val_derivative\n",
        "\n",
        "        return node_values\n",
        "\n",
        "    def _calc_hidden_node_vals(self, old_layer, old_node_vals):\n",
        "        new_node_vals = np.zeros(shape = self._nodes_out)\n",
        "\n",
        "        for new_node in range(new_node_vals.shape[0]):\n",
        "            new_node_val = 0\n",
        "\n",
        "            for old_node in range(old_node_vals.shape[0]):\n",
        "                weighted_derivative = old_layer._weights[new_node, old_node]\n",
        "                new_node_val += weighted_derivative * old_node_vals[old_node]\n",
        "\n",
        "            new_node_val *= self._activation.d_f(self._z_vals[new_node])\n",
        "            new_node_vals[new_node] = new_node_val\n",
        "        return new_node_vals\n",
        "\n",
        "    def _update_gradients(self, node_values):\n",
        "        for out_node in range(self._nodes_out):\n",
        "            for in_node in range(self._nodes_in):\n",
        "                cost_weight_derivative = self._inputs[in_node] * node_values[out_node]\n",
        "                self._gradient_w[in_node, out_node] += cost_weight_derivative\n",
        "\n",
        "            self._gradient_b[out_node] += node_values[out_node]\n",
        "\n",
        "    def _apply_gradients(self, learning_rate):\n",
        "        self._biases -= (learning_rate * self._gradient_b)\n",
        "        self._weights -= (learning_rate * self._gradient_w)\n",
        "\n",
        "    def _clear_gradients(self):\n",
        "        self._gradient_w = np.zeros(shape = (self._nodes_in, self._nodes_out))\n",
        "        self._gradient_b = np.zeros(shape = self._nodes_out)\n"
      ],
      "metadata": {
        "id": "EZN0uB64CEhf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class NeuralNet:\n",
        "    def __init__(self, layer_sizes, batch_size = 1000, epoch = 1):\n",
        "        self._layers = []\n",
        "\n",
        "        self._BATCH_SIZE = batch_size\n",
        "        self._EPOCH = epoch\n",
        "\n",
        "        for i in range(len(layer_sizes) - 1):\n",
        "            self._layers.append(LayerClass(layer_sizes[i], layer_sizes[i + 1]))\n",
        "\n",
        "        self._outputs = np.zeros(shape = layer_sizes[len(layer_sizes) - 1])\n",
        "\n",
        "    def _calc_outputs(self, inputs):\n",
        "        for layer in self._layers:\n",
        "            inputs = layer(inputs)\n",
        "\n",
        "        self._outputs = inputs\n",
        "\n",
        "    def __call__(self, inputs):\n",
        "        self._calc_outputs(inputs)\n",
        "        return self._outputs\n",
        "\n",
        "    def _cost(self, data_point):\n",
        "        self._calc_outputs(data_point['input'])\n",
        "        output_layer = self._layers[len(self._layers) - 1]\n",
        "        cost = 0.\n",
        "\n",
        "        for out_node in range(self._outputs.shape[0]):\n",
        "            cost += output_layer._error.f(self._outputs[out_node], data_point['expected_output'][out_node])\n",
        "\n",
        "        return cost\n",
        "\n",
        "    def _avg_cost(self, data_points):\n",
        "        total_cost = 0.\n",
        "\n",
        "        for data_point in data_points:\n",
        "            total_cost += self._cost(data_point)\n",
        "\n",
        "        return total_cost / len(data_points)\n",
        "\n",
        "    def _apply_gradients(self, learning_rate):\n",
        "        for layer in self._layers:\n",
        "            layer._apply_gradients(learning_rate)\n",
        "\n",
        "    def _clear_gradients(self) :\n",
        "        for layer in self._layers:\n",
        "            layer._clear_gradients()\n",
        "\n",
        "    def _back_prop(self, data_point):\n",
        "        self._calc_outputs(data_point['input'])\n",
        "\n",
        "        output_layer = self._layers[len(self._layers) - 1]\n",
        "        node_values = output_layer._calc_output_node_vals(data_point['expected_output'])\n",
        "        output_layer._update_gradients(node_values)\n",
        "\n",
        "        for layer in range(len(self._layers) - 2, -1, -1):\n",
        "            hidden_layer = self._layers[layer]\n",
        "            node_values = hidden_layer._calc_hidden_node_vals(self._layers[layer + 1], node_values)\n",
        "            hidden_layer._update_gradients(node_values)\n",
        "\n",
        "    def learn(self, batch, learning_rate):\n",
        "        avg_vals = []\n",
        "\n",
        "        for epoch in range(self._EPOCH):\n",
        "            for pos in range(0, len(batch), self._BATCH_SIZE):\n",
        "                mini_batch = batch[pos : np.minimum(pos + self._BATCH_SIZE, len(batch))]\n",
        "\n",
        "                for data_point in mini_batch:\n",
        "                    self._back_prop(data_point)\n",
        "\n",
        "                self._apply_gradients(learning_rate / np.minimum(self._BATCH_SIZE, len(mini_batch)))\n",
        "                self._clear_gradients()\n",
        "                # avg_vals.append(self._avg_cost(mini_batch))\n",
        "            \n",
        "            # self._apply_gradients(learning_rate / len(batch))\n",
        "            # self._clear_gradients()\n",
        "            avg_vals.append(self._avg_cost(batch))\n",
        "\n",
        "        return avg_vals\n",
        "\n",
        "        # h = .0001\n",
        "        #\n",
        "        # for mini_batch in range(0, len(traing_data), 1000):\n",
        "        #     original_cost = self._total_cost(training_data[mini_batch : np.minimum(mini_batch + 1000, len(traing_data))])\n",
        "        #\n",
        "        #     for layer in self._layers:\n",
        "        #         for in_node in range(layer._nodes_in):\n",
        "        #             for out_node in range(layer._nodes_out):\n",
        "        #                 layer._weights[in_node, out_node] += h\n",
        "        #                 delta_cost = self._total_cost(traing_data[mini_batch : np.minimum(mini_batch + 1000, len(traing_data))]) - original_cost\n",
        "        #                 layer._weights[in_node, out_node] -= h\n",
        "        #                 layer._gradient_w[in_node, out_node] = delta_cost / h\n",
        "        #\n",
        "        #         for out_node in range(layer._nodes_out):\n",
        "        #             layer._biases[out_node] += h\n",
        "        #             delta_cost = self._total_cost(traing_data[mini_batch : np.minimum(mini_batch + 1000, len(traing_data))]) - original_cost\n",
        "        #             layer._biases[out_node] -= hack\n",
        "        #             layer._gradient_b[out_node] delta_cost / h\n",
        "        #\n",
        "        #         layer._apply_gradients(learning_rate)\n"
      ],
      "metadata": {
        "id": "WKmjx6PZB_Mb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fruit_bot = NeuralNet((2, 3, 2))\n",
        "\n",
        "fruit_data = []\n",
        "for _ in range(10000000):\n",
        "    rand = np.random.uniform(5, 10, size = 2)\n",
        "\n",
        "    if rand[0] >= 5. or rand[1] >= 5.:\n",
        "        output = np.array([1, 0])\n",
        "    else:\n",
        "        output = np.array([0, 1])\n",
        "\n",
        "    fruit_data.append({\n",
        "        'input': rand,\n",
        "        'expected_output': output\n",
        "    })\n",
        "\n",
        "fruit_bot.learn(fruit_data, .01)\n",
        "\n",
        "print(fruit_bot(np.array([10., 10.]))) # poison [0, 1]\n",
        "print(fruit_bot(np.array([7., 9.]))) # poison [0, 1]\n",
        "print(fruit_bot(np.array([5., 6.]))) # poison [0, 1]\n",
        "print(fruit_bot(np.array([5., 1.]))) # poison [0, 1]\n",
        "print(fruit_bot(np.array([2., 4.]))) # good [1, 0]\n",
        "print(fruit_bot(np.array([4., 3.]))) # good [1, 0]\n",
        "print(fruit_bot(np.array([1., 1.]))) # good [1, 0]"
      ],
      "metadata": {
        "id": "sHqVw09eCHvM"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}