keras_to_loihi_ttbar_dense.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "03d312e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-24 23:30:45.139322: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
      "2024-04-24 23:30:48.116081: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import collections\n",
    "import os\n",
    "\n",
    "import nengo\n",
    "import nengo_dl\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import uproot\n",
    "\n",
    "try:\n",
    "    import requests\n",
    "\n",
    "    has_requests = True\n",
    "except ImportError:\n",
    "    has_requests = False\n",
    "\n",
    "import nengo_loihi\n",
    "\n",
    "scale = 10\n",
    "\n",
    "train_outdir = \"./train_examples/\"\n",
    "os.makedirs(train_outdir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0db0691b-b3bc-495c-afa5-53886287ec25",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`.\n",
    "def plot_confusion_matrix(y_true, y_pred, classes, normalize=False,  title=None, cmap=plt.cm.Blues, savename=\"./cm.pdf\"):\n",
    "    # Compute confusion matrix\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "        if not title: title = 'Normalized confusion matrix'\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "        if not title: title = 'Confusion matrix, without normalization'\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    ax.figure.colorbar(im, ax=ax)\n",
    "    ax.set(xticks=np.arange(cm.shape[1]), yticks=np.arange(cm.shape[0]), xticklabels=classes, yticklabels=classes,\n",
    "           title=title, ylabel='True label', xlabel='Predicted label')\n",
    "\n",
    "    # Rotate the tick labels and set their alignment.\n",
    "    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n",
    "\n",
    "    # Loop over data dimensions and create text annotations.\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i in range(cm.shape[0]):\n",
    "        for j in range(cm.shape[1]):\n",
    "            ax.text(j, i, format(cm[i, j], fmt), ha=\"center\", va=\"center\",\n",
    "                    color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "    fig.tight_layout()\n",
    "\n",
    "    plt.savefig(savename)\n",
    "    plt.gcf().clear()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5d65615a",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "File samples/data.h5 does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 26\u001b[0m\n\u001b[1;32m     23\u001b[0m class_names \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtthbb\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mttbb\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mttbj\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mttcc\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mttlf\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m     24\u001b[0m nClass, nVariables \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(class_names), \u001b[38;5;28mlen\u001b[39m(variables)\n\u001b[0;32m---> 26\u001b[0m pd_data \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_hdf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43msamples/data.h5\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mdf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28mprint\u001b[39m (pd_data)\n\u001b[1;32m     28\u001b[0m \u001b[38;5;28mprint\u001b[39m (pd_data\u001b[38;5;241m.\u001b[39mcolumns)\n",
      "File \u001b[0;32m~/miniconda3/envs/23snn/lib/python3.8/site-packages/pandas/io/pytables.py:418\u001b[0m, in \u001b[0;36mread_hdf\u001b[0;34m(path_or_buf, key, mode, errors, where, start, stop, columns, iterator, chunksize, **kwargs)\u001b[0m\n\u001b[1;32m    415\u001b[0m     exists \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m    417\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m exists:\n\u001b[0;32m--> 418\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFile \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpath_or_buf\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m does not exist\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    420\u001b[0m store \u001b[38;5;241m=\u001b[39m HDFStore(path_or_buf, mode\u001b[38;5;241m=\u001b[39mmode, errors\u001b[38;5;241m=\u001b[39merrors, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    421\u001b[0m \u001b[38;5;66;03m# can't auto open/close if we are using an iterator\u001b[39;00m\n\u001b[1;32m    422\u001b[0m \u001b[38;5;66;03m# so delegate to the iterator\u001b[39;00m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: File samples/data.h5 does not exist"
     ]
    }
   ],
   "source": [
    "# set up training data\n",
    "variables = [\n",
    "    'njets', 'nbjets', 'ncjets', 'nElectron', 'nMuon', 'MET_met', 'HT', # 'nLepton', 'MET_px', 'MET_py',\n",
    "    'Lepton_pt', 'Lepton_eta', 'Lepton_e',# 'Lepton_phi',\n",
    "    'Jet1_pt', 'Jet1_eta', 'Jet1_e', 'Jet1_btag', 'Jet2_pt', 'Jet2_eta', 'Jet2_e', 'Jet2_btag',# 'Jet_phi1', 'Jet_phi2',\n",
    "    'Jet3_pt', 'Jet3_eta', 'Jet3_e', 'Jet3_btag', 'Jet4_pt', 'Jet4_eta', 'Jet4_e', 'Jet4_btag',# 'Jet_phi3', 'Jet_phi4',\n",
    "    #'bjet1_pt', 'bjet1_eta', 'bjet1_e', 'bjet2_pt', 'bjet2_eta', 'bjet2_e',# 'bjet1_phi', 'bjet2_phi',\n",
    "    'selbjet1_pt', 'selbjet1_eta', 'selbjet1_e', 'selbjet2_pt', 'selbjet2_eta', 'selbjet2_e',# 'selbjet1_phi', 'selbjet2_phi',\n",
    "\n",
    "    'bbdR',   'bbdEta',   'bbdPhi',   'bbPt',   'bbEta',   'bbMass',   'bbHt',   'bbMt',  # 'bbPhi',\n",
    "    'nub1dR', 'nub1dEta', 'nub1dPhi', 'nub1Pt', 'nub1Eta', 'nub1Mass', 'nub1Ht', 'nub1Mt',# 'nub1Phi',\n",
    "    'nub2dR', 'nub2dEta', 'nub2dPhi', 'nub2Pt', 'nub2Eta', 'nub2Mass', 'nub2Ht', 'nub2Mt',# 'nub2Phi',\n",
    "    'nubbdR', 'nubbdEta', 'nubbdPhi', 'nubbPt', 'nubbEta', 'nubbMass', 'nubbHt', 'nubbMt',# 'nubbPhi',\n",
    "    'lb1dR',  'lb1dEta',  'lb1dPhi',  'lb1Pt',  'lb1Eta',  'lb1Mass',  'lb1Ht',  'lb1Mt', # 'lb1Phi',\n",
    "    'lb2dR',  'lb2dEta',  'lb2dPhi',  'lb2Pt',  'lb2Eta',  'lb2Mass',  'lb2Ht',  'lb2Mt', # 'lb2Phi',\n",
    "    'lbbdR',  'lbbdEta',  'lbbdPhi',  'lbbPt',  'lbbEta',  'lbbMass',  'lbbHt',  'lbbMt', # 'lbbPhi',\n",
    "    'Wjb1dR', 'Wjb1dEta', 'Wjb1dPhi', 'Wjb1Pt', 'Wjb1Eta', 'Wjb1Mass', 'Wjb1Ht', 'Wjb1Mt',# 'Wjb1Phi',\n",
    "    'Wjb2dR', 'Wjb2dEta', 'Wjb2dPhi', 'Wjb2Pt', 'Wjb2Eta', 'Wjb2Mass', 'Wjb2Ht', 'Wjb2Mt',# 'Wjb2Phi',\n",
    "    'Wlb1dR', 'Wlb1dEta', 'Wlb1dPhi', 'Wlb1Pt', 'Wlb1Eta', 'Wlb1Mass', 'Wlb1Ht', 'Wlb1Mt',# 'Wlb1Phi',\n",
    "    'Wlb2dR', 'Wlb2dEta', 'Wlb2dPhi', 'Wlb2Pt', 'Wlb2Eta', 'Wlb2Mass', 'Wlb2Ht', 'Wlb2Mt',# 'Wlb2Phi',\n",
    " \n",
    "]\n",
    "class_names = [\"tthbb\", \"ttbb\", \"ttbj\", \"ttcc\", \"ttlf\"]\n",
    "nClass, nVariables = len(class_names), len(variables)\n",
    "\n",
    "pd_data = pd.read_hdf('samples/data.h5', key='df', mode='r')\n",
    "print (pd_data)\n",
    "print (pd_data.columns)\n",
    "\n",
    "train_data = pd_data.filter(items = variables)\n",
    "train_data = (train_data - train_data.min())*scale/(train_data.max() - train_data.min())\n",
    "train_data = np.array(train_data).astype(float)\n",
    "train_out = np.array(pd_data.filter(items = [\"category\"])).reshape((len(pd_data),))\n",
    "\n",
    "print (train_out)\n",
    "\n",
    "trainlen = 267767\n",
    "train_images = train_data[:trainlen, 0::]\n",
    "train_labels = train_out[:trainlen]\n",
    "test_images = train_data[trainlen:, 0::]\n",
    "test_labels = train_out[trainlen:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e5ca0e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = tf.keras.Input(nVariables, name=\"input\")\n",
    "\n",
    "# transform input signal to spikes using trainable 1x1 convolutional layer\n",
    "to_spikes_layer = tf.keras.layers.Dense(units=nVariables, activation=tf.nn.relu, name=\"to-spikes\")\n",
    "to_spikes = to_spikes_layer(inp)\n",
    "\n",
    "# on-chip dense layers\n",
    "dense0_layer = tf.keras.layers.Dense(units=256, activation=tf.nn.relu, name=\"dense0\")\n",
    "dense0 = dense0_layer(to_spikes)\n",
    "dense1_layer = tf.keras.layers.Dense(units=256, activation=tf.nn.relu, name=\"dense1\")\n",
    "dense1 = dense1_layer(dense0)\n",
    "dense2_layer = tf.keras.layers.Dense(units=256, activation=tf.nn.relu, name=\"dense2\")\n",
    "dense2 = dense2_layer(dense1)\n",
    "# dense3_layer = tf.keras.layers.Dense(units=100, activation=tf.nn.relu, name=\"dense3\")\n",
    "# dense3 = dense3_layer(dense2)\n",
    "# dense4_layer = tf.keras.layers.Dense(units=100, activation=tf.nn.relu, name=\"dense4\")\n",
    "# dense4 = dense4_layer(dense3)\n",
    "# dense5_layer = tf.keras.layers.Dense(units=100, activation=tf.nn.relu, name=\"dense5\")\n",
    "# dense5 = dense5_layer(dense4)\n",
    "\n",
    "# since this final output layer has no activation function,\n",
    "# it will be converted to a `nengo.Node` and run off-chip\n",
    "out_layer = tf.keras.layers.Dense(units=nClass, name=\"out\")\n",
    "out = out_layer(dense2)\n",
    "\n",
    "model = tf.keras.Model(inputs=inp, outputs=out)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d172e28e",
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 100\n",
    "dt = 0.001  # simulation timestep\n",
    "presentation_time = 0.1  # input presentation time\n",
    "max_rate = 100  # neuron firing rates\n",
    "# neuron spike amplitude (scaled so that the overall output is ~1)\n",
    "amp = 1 / max_rate\n",
    "activation = nengo.SpikingRectifiedLinear()\n",
    "#activation = nengo_loihi.neurons.LoihiSpikingRectifiedLinear()\n",
    "params_file = train_outdir+\"./keras_to_loihi_dense\"\n",
    "converter = nengo_dl.Converter(model, swap_activations={tf.nn.relu: activation})\n",
    "\n",
    "with nengo_dl.Simulator(converter.net, seed=0, minibatch_size=200) as sim:\n",
    "    sim.compile(\n",
    "        optimizer=tf.optimizers.RMSprop(0.001),\n",
    "        loss={converter.outputs[out]: tf.losses.SparseCategoricalCrossentropy(from_logits=True)},\n",
    "        metrics={converter.outputs[out]: tf.metrics.sparse_categorical_accuracy},\n",
    "    )\n",
    "    sim.fit(\n",
    "        {converter.inputs[inp]: train_images},\n",
    "        {converter.outputs[out]: train_labels},\n",
    "        epochs=epochs,\n",
    "    )\n",
    "\n",
    "    # save the parameters to file\n",
    "    sim.save_params(params_file)\n",
    "    sim.freeze_params(converter.net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b24467f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_spiking_type(neuron_type):\n",
    "    return isinstance(neuron_type, (nengo.LIF, nengo.SpikingRectifiedLinear))\n",
    "\n",
    "n_test, n_steps, n_plots = 10000, 30, 30\n",
    "\n",
    "savename = train_outdir+\"SpikingNeurons_\"\n",
    "# convert the keras model to a nengo network\n",
    "nengo_converter = nengo_dl.Converter(\n",
    "    model,\n",
    "    scale_firing_rates=max_rate,\n",
    "    swap_activations={tf.nn.relu: activation},\n",
    "    synapse=0.005,\n",
    ")\n",
    "\n",
    "# get input/output objects\n",
    "nengo_input = nengo_converter.inputs[inp]\n",
    "nengo_output = nengo_converter.outputs[out]\n",
    "\n",
    "# add probes to layers to record activity\n",
    "with nengo_converter.net:\n",
    "    probes = collections.OrderedDict(\n",
    "        [\n",
    "            [to_spikes_layer, nengo.Probe(nengo_converter.layers[to_spikes])],\n",
    "            [dense0_layer, nengo.Probe(nengo_converter.layers[dense0])],\n",
    "            [dense1_layer, nengo.Probe(nengo_converter.layers[dense1])],\n",
    "            [dense2_layer, nengo.Probe(nengo_converter.layers[dense2])],\n",
    "            # [dense3_layer, nengo.Probe(nengo_converter.layers[dense3])],\n",
    "            # [dense4_layer, nengo.Probe(nengo_converter.layers[dense4])],\n",
    "            # [dense5_layer, nengo.Probe(nengo_converter.layers[dense5])],\n",
    "            #[out, nengo.Probe(nengo_converter.layers[out])],\n",
    "        ]\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5296f1cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# repeat inputs for some number of timesteps\n",
    "n_test = 200\n",
    "tiled_test_images = np.tile(test_images[:n_test], (1, n_steps, 1))\n",
    "#tiled_test_images = np.tile(test_images[:], (1, n_steps, 1))\n",
    "\n",
    "# # set some options to speed up simulation\n",
    "# with nengo_converter.net:\n",
    "#     nengo_dl.configure_settings(stateful=False)\n",
    "\n",
    "# build network, load in trained weights, run inference on test images\n",
    "with nengo_dl.Simulator(\n",
    "    nengo_converter.net, minibatch_size=200, # progress_bar=False\n",
    ") as sim:\n",
    "    data = sim.predict({nengo_input: tiled_test_images})\n",
    "    sim.compile(loss={nengo_converter.outputs[out]: tf.losses.SparseCategoricalCrossentropy(from_logits=True)})\n",
    "    print(\"accuracy w/ synaps: %.2f%%\"\n",
    "        % sim.evaluate(test_images, {nengo_converter.outputs[out]: test_labels}, verbose=0)[\"loss\"]\n",
    "    )\n",
    "\n",
    "# compute accuracy on test data, using output of network on\n",
    "# last timestep\n",
    "test_predictions = np.argmax(data[nengo_output][:, -1], axis=-1)\n",
    "print(\"Test accuracy: %.2f%%\" % (100 * np.mean(test_predictions == test_labels[:n_test, 0, 0])))\n",
    "# print(\"Test accuracy: %.2f%%\" % (100 * np.mean(test_predictions == test_labels[:, 0, 0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f6e86c7-c9e2-41d6-87d2-1b8e46755acf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the results\n",
    "mean_rates = []\n",
    "for i in range(n_plots):\n",
    "    plt.figure(figsize=(12, 6))\n",
    "\n",
    "    plt.subplot(1, 3, 1)\n",
    "    plt.title(test_labels[i, 0])\n",
    "    #plt.imshow(test_images[i, 0].reshape((7, 9)), cmap=\"gray\")\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "    n_layers = len(probes)\n",
    "    mean_rates_i = []\n",
    "    for j, layer in enumerate(probes.keys()):\n",
    "        #print (j, layer)\n",
    "        probe = probes[layer]\n",
    "        plt.subplot(n_layers, 3, (j * 3) + 2)\n",
    "        plt.suptitle(\"Neural activities\")\n",
    "\n",
    "        outputs = data[probe][i]\n",
    "\n",
    "        # look at only at non-zero outputs\n",
    "        nonzero = (outputs > 0).any(axis=0)\n",
    "        outputs = outputs[:, nonzero] if sum(nonzero) > 0 else outputs\n",
    "\n",
    "        # undo neuron amplitude to get real firing rates\n",
    "        outputs /= nengo_converter.layers[layer].ensemble.neuron_type.amplitude\n",
    "\n",
    "        rates = outputs.mean(axis=0)\n",
    "        mean_rate = rates.mean()\n",
    "        mean_rates_i.append(mean_rate)\n",
    "        #print('\"%s\" mean firing rate (example %d): %0.1f' % (layer.name, i, mean_rate))\n",
    "\n",
    "        if is_spiking_type(activation):\n",
    "            outputs *= 0.001\n",
    "            plt.ylabel(\"# of Spikes\")\n",
    "        else:\n",
    "            plt.ylabel(\"Firing rates (Hz)\")\n",
    "\n",
    "        # plot outputs of first 100 neurons\n",
    "        plt.plot(outputs[:, :100])\n",
    "\n",
    "    mean_rates.append(mean_rates_i)\n",
    "\n",
    "    plt.xlabel(\"Timestep\")\n",
    "\n",
    "    plt.subplot(1, 3, 3)\n",
    "    plt.title(\"Output predictions\")\n",
    "    plt.plot(tf.nn.softmax(data[nengo_output][i]))\n",
    "    plt.legend(class_names, loc=\"upper left\")\n",
    "    plt.xlabel(\"Timestep\")\n",
    "    plt.ylabel(\"Probability\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(savename+str(i)+\".png\")\n",
    "\n",
    "# take mean rates across all plotted examples\n",
    "mean_rates = np.array(mean_rates).mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50868dc8-1544-4c3c-b846-2358fcadc604",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_presentations = 100\n",
    "\n",
    "# if running on Loihi, increase the max input spikes per step\n",
    "hw_opts = dict(snip_max_spikes_per_step=max_rate)\n",
    "with nengo_loihi.Simulator(\n",
    "    nengo_converter.net,\n",
    "    dt=dt,\n",
    "    precompute=False,\n",
    "    hardware_options=hw_opts,\n",
    ") as sim:\n",
    "    # run the simulation on Loihi\n",
    "    sim.run(n_presentations * presentation_time)\n",
    "\n",
    "    # check classification accuracy\n",
    "    step = int(presentation_time / dt)\n",
    "    output = sim.data[nengo_converter.outputs[out]][step - 1 :: step]\n",
    "\n",
    "    acc = 100 * np.mean(\n",
    "        np.argmax(output, axis=-1) == test_labels[:n_presentations, -1, 0]\n",
    "    )\n",
    "    print(\"loihi accuracy: %.2f%%\" % acc)\n",
    "\n",
    "    predicted = np.argmax(output, axis=-1)\n",
    "    correct = test_labels[:n_presentations, -1, 0]\n",
    "\n",
    "    predicted = np.array(predicted, dtype=int)\n",
    "    correct = np.array(correct, dtype=int)\n",
    "\n",
    "    print(\"Predicted labels:\\t\", predicted)\n",
    "    print(\"Correct labels: \\t\", correct)\n",
    "    print(\"loihi acc: %.2f%%\" % acc)\n",
    "\n",
    "    np.set_printoptions(precision=2)\n",
    "\n",
    "\n",
    "    plot_confusion_matrix(correct, predicted, classes=class_names, \n",
    "                          savename=train_outdir+\"/confusion_matrix_val.pdf\")\n",
    "    plot_confusion_matrix(correct, predicted, classes=class_names, normalize=True, \n",
    "                          savename=train_outdir+\"/norm_confusion_matrix_val.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41012fe6-41ed-4da9-b527-6d4a63c2372f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}