transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "pip install transformers-interpret"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Requirement already satisfied: transformers-interpret in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (0.5.2)\n",
      "Requirement already satisfied: captum>=0.3.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers-interpret) (0.4.1)\n",
      "Requirement already satisfied: transformers>=3.0.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers-interpret) (4.16.2)\n",
      "Requirement already satisfied: torch>=1.2 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from captum>=0.3.1->transformers-interpret) (1.9.0)\n",
      "Requirement already satisfied: numpy in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from captum>=0.3.1->transformers-interpret) (1.20.3)\n",
      "Requirement already satisfied: matplotlib in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from captum>=0.3.1->transformers-interpret) (3.4.2)\n",
      "Requirement already satisfied: tqdm>=4.27 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (4.62.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (2021.8.3)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,>=0.10.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (0.11.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (21.0)\n",
      "Requirement already satisfied: sacremoses in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (0.0.47)\n",
      "Requirement already satisfied: filelock in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (3.4.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (6.0)\n",
      "Requirement already satisfied: requests in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (2.26.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from transformers>=3.0.0->transformers-interpret) (0.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers>=3.0.0->transformers-interpret) (3.10.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from packaging>=20.0->transformers>=3.0.0->transformers-interpret) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib->captum>=0.3.1->transformers-interpret) (2.8.2)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib->captum>=0.3.1->transformers-interpret) (9.0.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib->captum>=0.3.1->transformers-interpret) (1.3.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib->captum>=0.3.1->transformers-interpret) (0.10.0)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from requests->transformers>=3.0.0->transformers-interpret) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from requests->transformers>=3.0.0->transformers-interpret) (3.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from requests->transformers>=3.0.0->transformers-interpret) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from requests->transformers>=3.0.0->transformers-interpret) (1.26.6)\n",
      "Requirement already satisfied: six in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sacremoses->transformers>=3.0.0->transformers-interpret) (1.16.0)\n",
      "Requirement already satisfied: click in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sacremoses->transformers>=3.0.0->transformers-interpret) (8.0.1)\n",
      "Requirement already satisfied: joblib in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sacremoses->transformers>=3.0.0->transformers-interpret) (1.0.1)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "!pip install torch\n",
    "!pip install pytorch-lightning --quiet\n",
    "!pip install transformers --quiet\n",
    "!pip install torchmetrics\n",
    "!pip install scikit-multilearn\n",
    "!pip install pandas\n",
    "!pip install sklearn\n",
    "!pip install seaborn\n",
    "#!pip uninstall sagemaker -y\n",
    "!pip install sagemaker botocore==1.23.23\n",
    "#!pip uninstall boto3 botocore -y\n",
    "#!pip install --no-cache-dir sagemaker==1.72.1 botocore==1.23.23\n",
    "#!pip install boto3 botocore"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Requirement already satisfied: torch in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (1.9.0)\n",
      "Requirement already satisfied: typing_extensions in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from torch) (3.10.0.0)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: torchmetrics in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (0.7.2)\n",
      "Requirement already satisfied: packaging in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from torchmetrics) (21.0)\n",
      "Requirement already satisfied: pyDeprecate==0.3.* in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from torchmetrics) (0.3.1)\n",
      "Requirement already satisfied: numpy>=1.17.2 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from torchmetrics) (1.20.3)\n",
      "Requirement already satisfied: torch>=1.3.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from torchmetrics) (1.9.0)\n",
      "Requirement already satisfied: typing_extensions in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from torch>=1.3.1->torchmetrics) (3.10.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from packaging->torchmetrics) (2.4.7)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: scikit-multilearn in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (0.2.0)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: pandas in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (1.3.2)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pandas) (1.20.3)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pandas) (2021.1)\n",
      "Requirement already satisfied: six>=1.5 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from python-dateutil>=2.7.3->pandas) (1.16.0)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: sklearn in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (0.0)\n",
      "Requirement already satisfied: scikit-learn in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sklearn) (0.24.2)\n",
      "Requirement already satisfied: joblib>=0.11 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from scikit-learn->sklearn) (1.0.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from scikit-learn->sklearn) (2.2.0)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from scikit-learn->sklearn) (1.20.3)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from scikit-learn->sklearn) (1.6.2)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: seaborn in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (0.11.2)\n",
      "Requirement already satisfied: pandas>=0.23 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from seaborn) (1.3.2)\n",
      "Requirement already satisfied: scipy>=1.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from seaborn) (1.6.2)\n",
      "Requirement already satisfied: matplotlib>=2.2 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from seaborn) (3.4.2)\n",
      "Requirement already satisfied: numpy>=1.15 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from seaborn) (1.20.3)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib>=2.2->seaborn) (9.0.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib>=2.2->seaborn) (0.10.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib>=2.2->seaborn) (2.4.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib>=2.2->seaborn) (1.3.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from matplotlib>=2.2->seaborn) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pandas>=0.23->seaborn) (2021.1)\n",
      "Requirement already satisfied: six in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from cycler>=0.10->matplotlib>=2.2->seaborn) (1.16.0)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: sagemaker in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (2.77.1)\n",
      "Requirement already satisfied: botocore==1.23.23 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (1.23.23)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from botocore==1.23.23) (1.26.6)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from botocore==1.23.23) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from botocore==1.23.23) (2.8.2)\n",
      "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: importlib-metadata>=1.4.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (4.11.1)\n",
      "Requirement already satisfied: pathos in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (0.2.8)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (21.0)\n",
      "Requirement already satisfied: pandas in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (1.3.2)\n",
      "Requirement already satisfied: numpy>=1.9.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (1.20.3)\n",
      "Requirement already satisfied: attrs==20.3.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (20.3.0)\n",
      "Requirement already satisfied: google-pasta in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (0.2.0)\n",
      "Requirement already satisfied: protobuf>=3.1 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (3.19.4)\n",
      "Requirement already satisfied: protobuf3-to-dict>=0.1.5 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (0.1.5)\n",
      "Requirement already satisfied: boto3>=1.20.21 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from sagemaker) (1.20.23)\n",
      "Requirement already satisfied: s3transfer<0.6.0,>=0.5.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from boto3>=1.20.21->sagemaker) (0.5.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from importlib-metadata>=1.4.0->sagemaker) (3.5.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from packaging>=20.0->sagemaker) (2.4.7)\n",
      "Requirement already satisfied: six in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from protobuf3-to-dict>=0.1.5->sagemaker) (1.16.0)\n",
      "Requirement already satisfied: pytz>=2017.3 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pandas->sagemaker) (2021.1)\n",
      "Requirement already satisfied: dill>=0.3.4 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pathos->sagemaker) (0.3.4)\n",
      "Requirement already satisfied: multiprocess>=0.70.12 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pathos->sagemaker) (0.70.12.2)\n",
      "Requirement already satisfied: ppft>=1.6.6.4 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pathos->sagemaker) (1.6.6.4)\n",
      "Requirement already satisfied: pox>=0.3.0 in /Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages (from pathos->sagemaker) (0.3.0)\n",
      "\u001b[33mWARNING: You are using pip version 22.0.3; however, version 22.0.4 is available.\n",
      "You should consider upgrading via the '/Users/dolungwe/opt/anaconda3/envs/ubd/bin/python -m pip install --upgrade pip' command.\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#from transformers import AutoModelForSequenceClassification, AutoTokenizer"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "!nvidia-smi"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "zsh:1: command not found: nvidia-smi\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from transformers import BertTokenizerFast as BertTokenizer, BertModel, AdamW, get_linear_schedule_with_warmup\n",
    "\n",
    "import pytorch_lightning as pl\n",
    "from torchmetrics.functional import accuracy, f1, auroc, precision_recall_curve, average_precision\n",
    "#from pytorch_lightning.metrics.functional import accuracy, f1, auroc\n",
    "from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, multilabel_confusion_matrix\n",
    "\n",
    "import seaborn as sns\n",
    "from pylab import rcParams\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rc\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "\n",
    "RANDOM_SEED = 42\n",
    "\n",
    "sns.set(style='whitegrid', palette='muted', font_scale=1.2)\n",
    "HAPPY_COLORS_PALETTE = [\"#01BEFE\", \"#FFDD00\", \"#FF7D00\", \"#FF006D\", \"#ADFF02\", \"#8F00FF\"]\n",
    "sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))\n",
    "rcParams['figure.figsize'] = 12, 8\n",
    "\n",
    "pl.seed_everything(RANDOM_SEED)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# Read in the cleaned data\n",
    "data_prefix = 'https://raw.githubusercontent.com/nasa-petal/search-engine/main/data/'\n",
    "df = pd.read_csv(data_prefix + 'cleaned_leaves.csv')\n",
    "\n",
    "# Drop all non-feature columns\n",
    "non_feat = ['y', 'text']\n",
    "df.drop(non_feat, axis=1, inplace=True)\n",
    "\n",
    "# Drop all labels with < 20 papers\n",
    "LABEL_COLUMNS = df.columns.tolist()[:-1]\n",
    "df.drop([col for col, val in df[LABEL_COLUMNS].sum().iteritems() if val < 25], axis=1, inplace=True)\n",
    "dropcols = ['protect_from_animals', 'coordinate_by_self-organization', 'maintain_biodiversity', 'compete_within/between_species', 'cooperate_within/between_species']\n",
    "df.drop(dropcols, axis=1, inplace=True)\n",
    "\n",
    "#df = df[df.columns[df[LABEL_COLUMNS].sum()>3]]\n",
    "print(df.shape)\n",
    "df.head()"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(11012, 30)\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   distribute_liquids  sense_light_in_the_visible_spectrum  \\\n",
       "0                   0                                    0   \n",
       "1                   0                                    0   \n",
       "2                   0                                    1   \n",
       "3                   0                                    0   \n",
       "4                   0                                    0   \n",
       "\n",
       "   optimize_shape/materials  sense_chemicals  manage_stress/strain  \\\n",
       "0                         0                0                     0   \n",
       "1                         0                0                     0   \n",
       "2                         0                0                     0   \n",
       "3                         0                0                     0   \n",
       "4                         0                0                     1   \n",
       "\n",
       "   actively_move_through/on_liquids  manage_shear  \\\n",
       "0                                 0             0   \n",
       "1                                 0             0   \n",
       "2                                 0             0   \n",
       "3                                 0             0   \n",
       "4                                 0             1   \n",
       "\n",
       "   chemically_assemble_organic_compounds  change_size/shape  \\\n",
       "0                                      0                  0   \n",
       "1                                      0                  0   \n",
       "2                                      0                  0   \n",
       "3                                      0                  0   \n",
       "4                                      0                  0   \n",
       "\n",
       "   attach_temporarily  ...  manage_wear  respond_to_signals  \\\n",
       "0                   0  ...            0                   0   \n",
       "1                   0  ...            0                   0   \n",
       "2                   0  ...            0                   0   \n",
       "3                   0  ...            0                   0   \n",
       "4                   0  ...            0                   0   \n",
       "\n",
       "   protect_from_temperature  physically_assemble_structure  \\\n",
       "0                         0                              1   \n",
       "1                         0                              0   \n",
       "2                         0                              0   \n",
       "3                         1                              0   \n",
       "4                         0                              0   \n",
       "\n",
       "   prevent_fracture/rupture  protect_from_microbes  manage_impact  \\\n",
       "0                         0                      1              0   \n",
       "1                         0                      0              0   \n",
       "2                         0                      0              0   \n",
       "3                         0                      0              0   \n",
       "4                         0                      0              0   \n",
       "\n",
       "   protect_from_excess_liquids  actively_move_through/on_solids  \\\n",
       "0                            0                                0   \n",
       "1                            0                                0   \n",
       "2                            0                                0   \n",
       "3                            0                                0   \n",
       "4                            0                                0   \n",
       "\n",
       "                                            text_raw  \n",
       "0  Building a home from foam—túngara frog foam ne...  \n",
       "1  A nocturnal mammal, the greater mouse-eared ba...  \n",
       "2  Polarization sensitivity in two species of cut...  \n",
       "3  Identification and characterization of a multi...  \n",
       "4  DIFFERENCES IN POLYSACCHARIDE STRUCTURE BETWEE...  \n",
       "\n",
       "[5 rows x 30 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distribute_liquids</th>\n",
       "      <th>sense_light_in_the_visible_spectrum</th>\n",
       "      <th>optimize_shape/materials</th>\n",
       "      <th>sense_chemicals</th>\n",
       "      <th>manage_stress/strain</th>\n",
       "      <th>actively_move_through/on_liquids</th>\n",
       "      <th>manage_shear</th>\n",
       "      <th>chemically_assemble_organic_compounds</th>\n",
       "      <th>change_size/shape</th>\n",
       "      <th>attach_temporarily</th>\n",
       "      <th>...</th>\n",
       "      <th>manage_wear</th>\n",
       "      <th>respond_to_signals</th>\n",
       "      <th>protect_from_temperature</th>\n",
       "      <th>physically_assemble_structure</th>\n",
       "      <th>prevent_fracture/rupture</th>\n",
       "      <th>protect_from_microbes</th>\n",
       "      <th>manage_impact</th>\n",
       "      <th>protect_from_excess_liquids</th>\n",
       "      <th>actively_move_through/on_solids</th>\n",
       "      <th>text_raw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Building a home from foam—túngara frog foam ne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A nocturnal mammal, the greater mouse-eared ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Polarization sensitivity in two species of cut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Identification and characterization of a multi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>DIFFERENCES IN POLYSACCHARIDE STRUCTURE BETWEE...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "LABEL_COLUMNS = df.columns.tolist()[:-1]\n",
    "\n",
    "biom = df[df[LABEL_COLUMNS].sum(axis=1) > 0]\n",
    "nonbiom = df[df[LABEL_COLUMNS].sum(axis=1) == 0]\n",
    "\n",
    "# remove all non-biomimicry papers from dataset.\n",
    "df = biom"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "from skmultilearn.model_selection import iterative_train_test_split"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "def iterative_train_test_split_dataframe(X, y, test_size):\n",
    "    df_index = np.expand_dims(X.index.to_numpy(), axis=1)\n",
    "    df_index_y = np.expand_dims(y.index.to_numpy(), axis=1)\n",
    "    X_train, y_train, X_test, y_test = iterative_train_test_split(df_index, df_index_y, test_size = test_size)\n",
    "    X_train = X.loc[X_train[:,0]]\n",
    "    X_test = X.loc[X_test[:,0]]\n",
    "    y_train = y.loc[y_train[:,0]]\n",
    "    y_test = y.loc[y_test[:,0]]\n",
    "    return X_train, y_train, X_test, y_test"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "X_train, y_train, X_test, y_test = iterative_train_test_split_dataframe(X=df[['text_raw']], y=df[LABEL_COLUMNS], test_size = 0.15)\n",
    "train_df = pd.concat([X_train, y_train], axis=1)\n",
    "val_df = pd.concat([X_test, y_test], axis=1)\n",
    "print(train_df.shape, val_df.shape)\n",
    "\n",
    "'''\n",
    "X_train_val, y_train_val, X_test, y_test = iterative_train_test_split_dataframe(X=df[['text_raw']], y=df[LABEL_COLUMNS], test_size = 0.1)\n",
    "test_df = pd.concat([X_test, y_test], axis=1)\n",
    "X_train, y_train, X_val, y_val = iterative_train_test_split_dataframe(X=X_train_val, y=y_train_val, test_size = 0.13)\n",
    "train_df = pd.concat([X_train, y_train], axis=1)\n",
    "val_df = pd.concat([X_val, y_val], axis=1)\n",
    "#train_df, val_df = train_test_split(df, test_size=0.1)\n",
    "train_df.shape, val_df.shape, test_df.shape\n",
    "'''"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "(643, 30) (114, 30)\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "\"\\nX_train_val, y_train_val, X_test, y_test = iterative_train_test_split_dataframe(X=df[['text_raw']], y=df[LABEL_COLUMNS], test_size = 0.1)\\ntest_df = pd.concat([X_test, y_test], axis=1)\\nX_train, y_train, X_val, y_val = iterative_train_test_split_dataframe(X=X_train_val, y=y_train_val, test_size = 0.13)\\ntrain_df = pd.concat([X_train, y_train], axis=1)\\nval_df = pd.concat([X_val, y_val], axis=1)\\n#train_df, val_df = train_test_split(df, test_size=0.1)\\ntrain_df.shape, val_df.shape, test_df.shape\\n\""
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "BERT_MODEL_NAME = 'allenai/scibert_scivocab_uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_NAME)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "MAX_TOKEN_COUNT = 512"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "class BiomimicryDataset(Dataset):\n",
    "\n",
    "  def __init__(\n",
    "    self, \n",
    "    data: pd.DataFrame, \n",
    "    tokenizer: BertTokenizer, \n",
    "    max_token_len: int = 128\n",
    "  ):\n",
    "    self.tokenizer = tokenizer\n",
    "    self.data = data\n",
    "    self.max_token_len = max_token_len\n",
    "    \n",
    "  def __len__(self):\n",
    "    return len(self.data)\n",
    "\n",
    "  def __getitem__(self, index: int):\n",
    "    data_row = self.data.iloc[index]\n",
    "\n",
    "    text = data_row.text_raw\n",
    "    labels = data_row[LABEL_COLUMNS]\n",
    "\n",
    "    encoding = self.tokenizer.encode_plus(\n",
    "      text,\n",
    "      add_special_tokens=True,\n",
    "      max_length=self.max_token_len,\n",
    "      return_token_type_ids=False,\n",
    "      padding=\"max_length\",\n",
    "      truncation=True,\n",
    "      return_attention_mask=True,\n",
    "      return_tensors='pt',\n",
    "    )\n",
    "\n",
    "    return dict(\n",
    "      text=text,\n",
    "      input_ids=encoding[\"input_ids\"].flatten(),\n",
    "      attention_mask=encoding[\"attention_mask\"].flatten(),\n",
    "      labels=torch.FloatTensor(labels)\n",
    "    )"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "class BiomimicryDataModule(pl.LightningDataModule):\n",
    "\n",
    "  def __init__(self, train_df, test_df, tokenizer, batch_size=8, max_token_len=128):\n",
    "    super().__init__()\n",
    "    self.batch_size = batch_size\n",
    "    self.train_df = train_df\n",
    "    self.test_df = test_df\n",
    "    self.tokenizer = tokenizer\n",
    "    self.max_token_len = max_token_len\n",
    "\n",
    "  def setup(self, stage=None):\n",
    "    self.train_dataset = BiomimicryDataset(\n",
    "      self.train_df,\n",
    "      self.tokenizer,\n",
    "      self.max_token_len\n",
    "    )\n",
    "\n",
    "    self.test_dataset = BiomimicryDataset(\n",
    "      self.test_df,\n",
    "      self.tokenizer,\n",
    "      self.max_token_len\n",
    "    )\n",
    "\n",
    "  def train_dataloader(self):\n",
    "    return DataLoader(\n",
    "      self.train_dataset,\n",
    "      batch_size=self.batch_size,\n",
    "      shuffle=True,\n",
    "      num_workers=2\n",
    "    )\n",
    "\n",
    "  def val_dataloader(self):\n",
    "    return DataLoader(\n",
    "      self.test_dataset,\n",
    "      batch_size=self.batch_size,\n",
    "      num_workers=2\n",
    "    )\n",
    "\n",
    "  def test_dataloader(self):\n",
    "    return DataLoader(\n",
    "      self.test_dataset,\n",
    "      batch_size=self.batch_size,\n",
    "      num_workers=2\n",
    "    )"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "N_EPOCHS = 100\n",
    "BATCH_SIZE = 12 # smaller is better.\n",
    "\n",
    "data_module = BiomimicryDataModule(\n",
    "  train_df,\n",
    "  val_df,\n",
    "  tokenizer,\n",
    "  batch_size=BATCH_SIZE,\n",
    "  max_token_len=MAX_TOKEN_COUNT\n",
    ")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "class BiomimicryTagger(pl.LightningModule):\n",
    "\n",
    "  def __init__(self, n_classes: int, n_training_steps=None, n_warmup_steps=None):\n",
    "    super().__init__()\n",
    "    self.bert = BertModel.from_pretrained(BERT_MODEL_NAME, return_dict=True)\n",
    "    self.classifier = nn.Linear(self.bert.config.hidden_size, n_classes)\n",
    "    self.n_training_steps = n_training_steps\n",
    "    self.n_warmup_steps = n_warmup_steps\n",
    "    self.criterion = nn.BCELoss()\n",
    "\n",
    "  def forward(self, input_ids, attention_mask, labels=None):\n",
    "    output = self.bert(input_ids, attention_mask=attention_mask)\n",
    "    output = self.classifier(output.pooler_output)\n",
    "    output = torch.sigmoid(output)    \n",
    "    loss = 0\n",
    "    if labels is not None:\n",
    "        loss = self.criterion(output, labels)\n",
    "    return loss, output\n",
    "\n",
    "  def training_step(self, batch, batch_idx):\n",
    "    input_ids = batch[\"input_ids\"]\n",
    "    attention_mask = batch[\"attention_mask\"]\n",
    "    labels = batch[\"labels\"]\n",
    "    loss, outputs = self(input_ids, attention_mask, labels)\n",
    "    self.log(\"train_loss\", loss, prog_bar=True, logger=True)\n",
    "    return {\"loss\": loss, \"predictions\": outputs, \"labels\": labels}\n",
    "\n",
    "  def validation_step(self, batch, batch_idx):\n",
    "    input_ids = batch[\"input_ids\"]\n",
    "    attention_mask = batch[\"attention_mask\"]\n",
    "    labels = batch[\"labels\"]\n",
    "    loss, outputs = self(input_ids, attention_mask, labels)\n",
    "    self.log(\"val_loss\", loss, prog_bar=True, logger=True)\n",
    "    return loss\n",
    "\n",
    "  def test_step(self, batch, batch_idx):\n",
    "    input_ids = batch[\"input_ids\"]\n",
    "    attention_mask = batch[\"attention_mask\"]\n",
    "    labels = batch[\"labels\"]\n",
    "    loss, outputs = self(input_ids, attention_mask, labels)\n",
    "    self.log(\"test_loss\", loss, prog_bar=True, logger=True)\n",
    "    return loss\n",
    "\n",
    "  def training_epoch_end(self, outputs):\n",
    "    \n",
    "    labels = []\n",
    "    predictions = []\n",
    "    for output in outputs:\n",
    "      for out_labels in output[\"labels\"].detach().cpu():\n",
    "        labels.append(out_labels)\n",
    "      for out_predictions in output[\"predictions\"].detach().cpu():\n",
    "        predictions.append(out_predictions)\n",
    "\n",
    "    labels = torch.stack(labels).int()\n",
    "    predictions = torch.stack(predictions)\n",
    "\n",
    "    for i, name in enumerate(LABEL_COLUMNS):\n",
    "      class_roc_auc = auroc(predictions[:, i],  labels[:, i])\n",
    "      #class_ap = average_precision(predictions[:, i], labels[:, i])\n",
    "      self.logger.experiment.add_scalar(f\"{name}_roc_auc/Train\", class_roc_auc, self.current_epoch)\n",
    "      #self.logger.experiment.add_scalar(f\"{name}_ap/Train\", class_ap, self.current_epoch)\n",
    "\n",
    "\n",
    "  def configure_optimizers(self):\n",
    "\n",
    "    optimizer = AdamW(self.parameters(), lr=2e-5)\n",
    "\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "      optimizer,\n",
    "      num_warmup_steps=self.n_warmup_steps,\n",
    "      num_training_steps=self.n_training_steps\n",
    "    )\n",
    "\n",
    "    return dict(\n",
    "      optimizer=optimizer,\n",
    "      lr_scheduler=dict(\n",
    "        scheduler=scheduler,\n",
    "        interval='step'\n",
    "      )\n",
    "    )"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "dummy_model = nn.Linear(2, 1)\n",
    "\n",
    "optimizer = AdamW(params=dummy_model.parameters(), lr=0.001)\n",
    "\n",
    "warmup_steps = 20\n",
    "total_training_steps = 100\n",
    "\n",
    "scheduler = get_linear_schedule_with_warmup(\n",
    "  optimizer, \n",
    "  num_warmup_steps=warmup_steps,\n",
    "  num_training_steps=total_training_steps\n",
    ")\n",
    "\n",
    "learning_rate_history = []\n",
    "\n",
    "for step in range(total_training_steps):\n",
    "  optimizer.step()\n",
    "  scheduler.step()\n",
    "  learning_rate_history.append(optimizer.param_groups[0]['lr'])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use thePyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "steps_per_epoch=len(train_df) // BATCH_SIZE\n",
    "total_training_steps = steps_per_epoch * N_EPOCHS"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "warmup_steps = total_training_steps // 5\n",
    "warmup_steps, total_training_steps"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(1060, 5300)"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "model = BiomimicryTagger(\n",
    "  n_classes=len(LABEL_COLUMNS),\n",
    "  n_warmup_steps=warmup_steps,\n",
    "  n_training_steps=total_training_steps \n",
    ")"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "!rm -rf lightning_logs/\n",
    "!rm -rf checkpoints/"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "checkpoint_callback = ModelCheckpoint(\n",
    "  dirpath=\"checkpoints\",\n",
    "  filename=\"best-checkpoint\",\n",
    "  save_top_k=1, # save top 5 or 10.\n",
    "  verbose=True,\n",
    "  monitor=\"val_loss\",\n",
    "  mode=\"min\"\n",
    ")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "early_stopping_callback = EarlyStopping(monitor='val_loss', patience=10)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "trainer = pl.Trainer(\n",
    "  checkpoint_callback=checkpoint_callback,\n",
    "  callbacks=[early_stopping_callback],\n",
    "  max_epochs=N_EPOCHS,\n",
    "  gpus=0, #changed from 1 to 0\n",
    "  progress_bar_refresh_rate=30\n",
    ")"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:147: LightningDeprecationWarning: Setting `Trainer(checkpoint_callback=<pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint object at 0x7ffe65471340>)` is deprecated in v1.5 and will be removed in v1.7. Please consider using `Trainer(enable_checkpointing=<pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint object at 0x7ffe65471340>)`.\n",
      "  rank_zero_deprecation(\n",
      "/Users/dolungwe/opt/anaconda3/envs/ubd/lib/python3.9/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:90: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=30)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.\n",
      "  rank_zero_deprecation(\n",
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "\n",
    "#model_name = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
    "model = BiomimicryTagger.load_from_checkpoint(\n",
    "    '/Users/dolungwe/Documents/PeTaL/scibert-top29-aws-epoch46-122221.ckpt',\n",
    "    n_classes=len(LABEL_COLUMNS)\n",
    ")\n",
    "#tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "#dont use path\n",
    "#path: /Users/dolungwe/Documents/PeTaL/scibert-top29-aws-epoch46-122221.ckpt"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "from transformers_interpret import SequenceClassificationExplainer\n",
    "cls_explainer = SequenceClassificationExplainer(\n",
    "    model,\n",
    "    tokenizer)"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "AttributeError",
     "evalue": "'BiomimicryTagger' object has no attribute 'config'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/wf/cz4rdd_x6vn9rclbjndy6gv80000gp/T/ipykernel_25110/2367042836.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers_interpret\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSequenceClassificationExplainer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m cls_explainer = SequenceClassificationExplainer(\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     tokenizer)\n",
      "\u001b[0;32m~/opt/anaconda3/envs/ubd/lib/python3.9/site-packages/transformers_interpret/explainers/sequence_classification.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model, tokenizer, attribution_type, custom_labels)\u001b[0m\n\u001b[1;32m     55\u001b[0m             \u001b[0mAttributionTypeNotSupportedError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m         \"\"\"\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mattribution_type\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mSUPPORTED_ATTRIBUTION_TYPES\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m             raise AttributionTypeNotSupportedError(\n",
      "\u001b[0;32m~/opt/anaconda3/envs/ubd/lib/python3.9/site-packages/transformers_interpret/explainer.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model, tokenizer)\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"gpt2\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mref_token_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meos_token_id\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/ubd/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1128\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmodules\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1129\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mmodules\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1130\u001b[0;31m         raise AttributeError(\"'{}' object has no attribute '{}'\".format(\n\u001b[0m\u001b[1;32m   1131\u001b[0m             type(self).__name__, name))\n\u001b[1;32m   1132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'BiomimicryTagger' object has no attribute 'config'"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#word_attributions = cls_explainer(\"water movements\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#word_attributions"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[('[CLS]', 0.0),\n",
       " ('water', 0.04657656915824678),\n",
       " ('movements', 0.998914722689303),\n",
       " ('[SEP]', 0.0)]"
      ]
     },
     "metadata": {},
     "execution_count": 28
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.6 64-bit ('ubd': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "interpreter": {
   "hash": "839ad94591bce73c9d5f76d94edde9e62613c6949ed9183ac349367aa2169041"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}