Flair_Classification_BERT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.metrics import mean_squared_error,roc_auc_score,precision_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('final_data2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "flair_dict = {   \n",
    " 0: 'News',\n",
    " 1: 'FUNDAMENTALS',\n",
    " 2: 'NEWS',\n",
    " 3: 'DISCUSSION',\n",
    " 4: 'TECHNICALS',\n",
    " 5: 'Comedy',\n",
    " 6: 'COMEDY',\n",
    " 7: 'Exchange',\n",
    " 8: 'SENTIMENT',\n",
    " 9: 'STRATEGY',\n",
    " 10: 'Strategy',\n",
    " 11: 'Technical Analysis',\n",
    " 12: 'Educational'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>flair</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>text</th>\n",
       "      <th>score</th>\n",
       "      <th>num_comments</th>\n",
       "      <th>original</th>\n",
       "      <th>comments</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Ethereum Fees Now at July 2021 Lows of $2.54 -...</td>\n",
       "      <td>https://cryptonewsland.com/ethereum-fees-now-a...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>14</td>\n",
       "      <td>False</td>\n",
       "      <td>Ethereum [pros](/r/CryptoMarkets/comments/ux7...</td>\n",
       "      <td>0.9497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Coinbase CEO’s Response to Employee Rant Backf...</td>\n",
       "      <td>https://cryptonewsland.com/coinbase-ceos-respo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>USDC [pros](/r/CryptoMarkets/comments/v9izys/...</td>\n",
       "      <td>-0.4404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>Mars4 Metaverse is Selling Fast in Japan – Spo...</td>\n",
       "      <td>https://news.bitcoin.com/mars4-metaverse-is-se...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>Bitcoin [pros](/r/CryptoMarkets/comments/v9vc...</td>\n",
       "      <td>-0.2732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Terra Crisis: G-7 Nations Call for Urgent Cryp...</td>\n",
       "      <td>https://crypto.news/terra-g-7-nations-urgent-c...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14</td>\n",
       "      <td>13</td>\n",
       "      <td>False</td>\n",
       "      <td>Do Kwan screwed us. They were just looking fo...</td>\n",
       "      <td>0.6705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>White House says the Administration is Closely...</td>\n",
       "      <td>https://www.youtube.com/watch?v=4oSL1gCHFdQ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>Inflation [pros](/r/CryptoMarkets/comments/vd...</td>\n",
       "      <td>0.8979</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   flair                                              title  \\\n",
       "0      0  Ethereum Fees Now at July 2021 Lows of $2.54 -...   \n",
       "1      0  Coinbase CEO’s Response to Employee Rant Backf...   \n",
       "2      0  Mars4 Metaverse is Selling Fast in Japan – Spo...   \n",
       "3      0  Terra Crisis: G-7 Nations Call for Urgent Cryp...   \n",
       "4      0  White House says the Administration is Closely...   \n",
       "\n",
       "                                                 url text  score  \\\n",
       "0  https://cryptonewsland.com/ethereum-fees-now-a...  NaN     21   \n",
       "1  https://cryptonewsland.com/coinbase-ceos-respo...  NaN      4   \n",
       "2  https://news.bitcoin.com/mars4-metaverse-is-se...  NaN      2   \n",
       "3  https://crypto.news/terra-g-7-nations-urgent-c...  NaN     14   \n",
       "4        https://www.youtube.com/watch?v=4oSL1gCHFdQ  NaN      0   \n",
       "\n",
       "   num_comments  original                                           comments  \\\n",
       "0            14     False   Ethereum [pros](/r/CryptoMarkets/comments/ux7...   \n",
       "1             7     False   USDC [pros](/r/CryptoMarkets/comments/v9izys/...   \n",
       "2             9     False   Bitcoin [pros](/r/CryptoMarkets/comments/v9vc...   \n",
       "3            13     False   Do Kwan screwed us. They were just looking fo...   \n",
       "4             4     False   Inflation [pros](/r/CryptoMarkets/comments/vd...   \n",
       "\n",
       "   polarity  \n",
       "0    0.9497  \n",
       "1   -0.4404  \n",
       "2   -0.2732  \n",
       "3    0.6705  \n",
       "4    0.8979  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_rand_example(df, col_name, col_value, chars=180):    \n",
    "    title = df[df[col_name]==col_value].sample()['title']\n",
    "    url = df[df[col_name]==col_value].sample()['url']\n",
    "    text = df[df[col_name]==col_value].sample()['text']\n",
    "    comments = df[df[col_name]==col_value].sample()['comments']\n",
    "    print(title[0:chars] + \"...\")\n",
    "    print(url[0:chars] + \"...\")\n",
    "    print(text[0:chars] + \"...\")\n",
    "    print(comments[0:chars] + \"...\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Flair: News\n",
      "86    Ukrainian President Volodymyr Zelenskyy legali...\n",
      "Name: title, dtype: object\n",
      "92    https://youtu.be/ycJW54ItBHM...\n",
      "Name: url, dtype: object\n",
      "74    The price of HBAR could reach $0.81 in 2022, a...\n",
      "Name: text, dtype: object\n",
      "26     NFT [pros](/r/CryptoMarkets/comments/udlknm/n...\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: FUNDAMENTALS\n",
      "175    The majority of top crypto projects in the mar...\n",
      "Name: title, dtype: object\n",
      "184    https://youtu.be/03xxDEsmWv8...\n",
      "Name: url, dtype: object\n",
      "147    **Preface and Disclosures:**\\n\\nNone of the in...\n",
      "Name: text, dtype: object\n",
      "116     Monero [pros](/r/CryptoMarkets/comments/ui2qj...\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: NEWS\n",
      "279    Proof-of-Work Limit is Back in Discussion at t...\n",
      "Name: title, dtype: object\n",
      "203    https://crypto.news/terra-g-7-nations-urgent-c...\n",
      "Name: url, dtype: object\n",
      "279    NaN\n",
      "Name: text, dtype: object\n",
      "248    NaN\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: DISCUSSION\n",
      "383    Weekly Discussion Megathread - May 16, 2021 (G...\n",
      "Name: title, dtype: object\n",
      "311    https://www.reddit.com/r/CryptoMarkets/comment...\n",
      "Name: url, dtype: object\n",
      "378    NaN\n",
      "Name: text, dtype: object\n",
      "358     Alt season incoming, brace yourselves Any ide...\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: TECHNICALS\n",
      "435    MANA Technical Analysis: Next Bullish Stop for...\n",
      "Name: title, dtype: object\n",
      "492    https://www.reddit.com/r/CryptoMarkets/comment...\n",
      "Name: url, dtype: object\n",
      "495    NaN\n",
      "Name: text, dtype: object\n",
      "458    NaN\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: Comedy\n",
      "504    *Comedy* When the market is crashing and you a...\n",
      "Name: title, dtype: object\n",
      "506    https://www.youtube.com/watch?v=j1XtVYw7p5E&t...\n",
      "Name: url, dtype: object\n",
      "502    NaN\n",
      "Name: text, dtype: object\n",
      "504     Too soon :( First time I've seen this joke in...\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: COMEDY\n",
      "519    Weekly Discussion Megathread - 09/Nov/2020...\n",
      "Name: title, dtype: object\n",
      "516    https://www.youtube.com/watch?v=j1XtVYw7p5E&t...\n",
      "Name: url, dtype: object\n",
      "516    NaN\n",
      "Name: text, dtype: object\n",
      "516     Ughhh.  Awful...\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: Exchange\n",
      "570    Bitcoin's Network Has Just Seen the Largest Ne...\n",
      "Name: title, dtype: object\n",
      "597    https://thecryptobasic.com/2022/03/31/ukraines...\n",
      "Name: url, dtype: object\n",
      "527    NaN\n",
      "Name: text, dtype: object\n",
      "619    NaN\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: SENTIMENT\n",
      "646    The amount of Ethereum (ETH) staked on the Bea...\n",
      "Name: title, dtype: object\n",
      "685    http://blindsignals.com/crashing-sentiment-sur...\n",
      "Name: url, dtype: object\n",
      "685    NaN\n",
      "Name: text, dtype: object\n",
      "652    NaN\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: STRATEGY\n",
      "728    MicroStrategy CEO: 129,000 BTC purchase progra...\n",
      "Name: title, dtype: object\n",
      "760    https://medium.com/coinmonks/strategy-analysis...\n",
      "Name: url, dtype: object\n",
      "782    NaN\n",
      "Name: text, dtype: object\n",
      "729     Bitcoin [pros](/r/CryptoMarkets/comments/ubha...\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: Strategy\n",
      "897    Taking advantage of a bear market smartly does...\n",
      "Name: title, dtype: object\n",
      "901    https://kollider.medium.com/when-bets-become-t...\n",
      "Name: url, dtype: object\n",
      "860    NaN\n",
      "Name: text, dtype: object\n",
      "917     Marketing, Network, and Team are what matters...\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: Technical Analysis\n",
      "1014    Full Immersion in Technical Analysis: Oscillat...\n",
      "Name: title, dtype: object\n",
      "938    https://coinjoy.io/news/259117240/mana-technic...\n",
      "Name: url, dtype: object\n",
      "935    NaN\n",
      "Name: text, dtype: object\n",
      "961     Nope enjoy every second of it I’m staked in e...\n",
      "Name: comments, dtype: object\n",
      "\n",
      "Flair: Educational\n",
      "1098    What Is Graph Token...\n",
      "Name: title, dtype: object\n",
      "1091    https://www.reddit.com/r/CryptoMarkets/comment...\n",
      "Name: url, dtype: object\n",
      "1075    Dfx finance released a sweet little educationa...\n",
      "Name: text, dtype: object\n",
      "1109     So buy BIT? just tell me what to buy...\n",
      "Name: comments, dtype: object\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for flair_num in range(13):\n",
    "    print(f\"Flair: {flair_dict[flair_num]}\")\n",
    "    print_rand_example(df, 'flair', flair_num)\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "num_classes = 13\n",
    "\n",
    "y = tf.keras.utils.to_categorical(df[\"flair\"].values, num_classes=num_classes)\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(df['title'], y, test_size=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading pretrained BERT  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 768), dtype=float32, numpy=\n",
       "array([[ 1.08269252e-01, -9.16300267e-02, -3.19653004e-01,\n",
       "        -1.70694273e-02,  3.34133953e-01,  2.00266182e-01,\n",
       "        -2.46936023e-01, -5.43311000e-01, -3.56956989e-01,\n",
       "        -2.97796547e-01, -2.08980605e-01,  1.42025888e-01,\n",
       "         2.72499211e-02, -4.33980763e-01, -5.73600113e-01,\n",
       "        -8.82729530e-01,  2.37143829e-01, -1.87127650e-01,\n",
       "        -2.56524444e-01, -2.46629804e-01,  6.32454097e-01,\n",
       "         1.21354170e-01, -3.95606160e-01,  1.58425853e-01,\n",
       "        -6.60168827e-02, -1.00859690e+00,  3.03424567e-01,\n",
       "         2.40532998e-02, -9.84722450e-02, -6.36505425e-01,\n",
       "         1.15401782e-01, -7.62528837e-01, -5.43886542e-01,\n",
       "        -4.67391938e-01, -6.24043405e-01,  2.63592869e-01,\n",
       "        -4.41221267e-01,  6.62042052e-02, -3.88788015e-01,\n",
       "         1.61611661e-02,  1.15522750e-01, -4.35086757e-01,\n",
       "        -5.12637496e-01,  2.74716794e-01, -3.44001085e-01,\n",
       "        -1.88089877e-01,  1.52756153e-02, -3.23565960e-01,\n",
       "         3.03293645e-01,  3.83590311e-01,  3.73140424e-01,\n",
       "         1.05662203e+00,  2.65405297e-01, -2.77808249e-01,\n",
       "         6.00603074e-02,  9.51387361e-02,  1.53988777e-02,\n",
       "        -3.05229753e-01,  2.57459253e-01, -2.20641091e-01,\n",
       "        -3.19688857e-01, -4.33157533e-01, -1.83581308e-01,\n",
       "        -5.61815619e-01, -3.60706896e-01, -1.27188470e-02,\n",
       "        -2.26824537e-01, -4.15555865e-01, -1.52612373e-01,\n",
       "        -4.75152075e-01, -8.22974443e-01, -7.81015217e-01,\n",
       "         1.13984369e-01, -3.97321373e-01, -1.61369294e-01,\n",
       "        -1.50214992e-02, -3.85155290e-01, -1.41645387e-01,\n",
       "        -5.67433655e-01, -1.96933612e-01,  9.26956311e-02,\n",
       "         3.41263890e+00, -4.40866888e-01, -3.74857277e-01,\n",
       "         3.05707514e-01,  9.27157179e-02, -1.75334185e-01,\n",
       "        -1.90602824e-01,  2.08361074e-01,  1.57442793e-01,\n",
       "        -2.96183079e-01,  5.33570588e-01, -4.92963731e-01,\n",
       "        -6.09962881e-01,  7.23478720e-02, -2.35901520e-01,\n",
       "        -4.35635418e-01,  2.84979731e-01, -1.12737827e-01,\n",
       "        -8.74462053e-02,  1.06023408e-01, -2.77983785e-01,\n",
       "         4.13428769e-02, -8.70151445e-02,  6.58435941e-01,\n",
       "        -2.40861848e-01, -3.30954254e-01, -2.05701515e-01,\n",
       "         7.51759335e-02,  1.29357791e-02, -9.85725969e-02,\n",
       "        -1.16355136e-01,  8.02322999e-02,  2.37922862e-01,\n",
       "        -2.28398576e-01,  4.86106753e-01, -2.93942630e-01,\n",
       "         2.02164173e-01, -2.21727312e-01,  3.97714347e-01,\n",
       "        -1.21958494e-01, -1.49668783e-01,  1.49842098e-01,\n",
       "        -2.35190725e+00, -2.80008733e-01,  4.76798952e-01,\n",
       "         2.38757417e-01, -1.85722366e-01,  1.89203948e-01,\n",
       "        -1.34296343e-01,  5.39198965e-02,  8.31941366e-02,\n",
       "        -3.04814905e-01, -3.19170117e-01, -3.75501961e-01,\n",
       "        -1.86103687e-01, -1.66042864e-01, -3.35772306e-01,\n",
       "         1.98825389e-01,  1.06203899e-01,  2.22116962e-01,\n",
       "         1.69316590e-01, -8.01408291e-02,  5.85859299e-01,\n",
       "        -4.21976984e-01,  1.92272842e-01, -1.17814019e-01,\n",
       "         5.07521406e-02, -6.33531623e-03, -2.58769870e-01,\n",
       "        -1.55983031e-01, -9.29405242e-02, -1.29259378e-01,\n",
       "        -1.86782032e-01, -4.52822059e-01, -3.72048095e-02,\n",
       "        -1.79112524e-01,  7.37308264e-01, -2.70773888e-01,\n",
       "         6.77012280e-02, -4.02352333e-01,  3.56926292e-01,\n",
       "        -1.24754570e-01, -1.35014117e-01, -4.02243912e-01,\n",
       "        -2.08425894e-01, -1.80958584e-01, -1.68645173e-01,\n",
       "        -4.42371875e-01, -2.91358590e-01, -3.87244523e-01,\n",
       "        -2.18316764e-01, -1.20590210e-01,  1.26221985e-01,\n",
       "         2.22633228e-01,  4.13879663e-01, -3.56124371e-01,\n",
       "        -5.81821680e-01, -4.61167276e-01,  3.83982778e-01,\n",
       "        -2.19285041e-01,  4.97522801e-01, -2.60475203e-02,\n",
       "        -3.52640450e-01, -1.38494745e-01, -2.48727649e-01,\n",
       "        -6.27775565e-02,  2.00329229e-01, -7.88270354e-01,\n",
       "         1.19192190e-01,  1.42032400e-01, -6.05769992e-01,\n",
       "         6.37907535e-02, -2.26173699e-01, -6.35088503e-01,\n",
       "         3.88163716e-01, -3.55442435e-01, -8.75470400e-01,\n",
       "         1.42647192e-01, -6.87847510e-02, -5.20077109e-01,\n",
       "         1.39270961e-01,  3.53218526e-01, -4.94886011e-01,\n",
       "        -2.22826228e-01, -3.46113324e-01,  2.55099013e-02,\n",
       "        -3.59313369e-01, -3.39053243e-01, -5.22465371e-02,\n",
       "        -5.30673921e-01, -5.25371842e-02,  5.55584967e-01,\n",
       "        -5.38592517e-01,  6.42332211e-02, -6.47422194e-01,\n",
       "        -4.31252420e-01, -6.47720098e-02, -3.78567725e-01,\n",
       "         3.19087178e-01,  1.58831522e-01, -4.33927067e-02,\n",
       "        -1.97739318e-01,  2.61362642e-01, -6.11117601e-01,\n",
       "        -7.43353665e-01, -8.40410352e-01, -2.52397686e-01,\n",
       "         6.84557334e-02,  5.17156005e-01, -2.68190324e-01,\n",
       "         1.09282777e-01, -1.45635232e-01, -3.55043590e-01,\n",
       "        -1.29664198e-01,  1.52991164e+00,  1.95044622e-01,\n",
       "        -4.62972194e-01, -5.01527667e-01, -1.48183256e-01,\n",
       "         9.83450636e-02, -3.73081088e-01, -1.57468736e-01,\n",
       "        -2.43043363e-01, -1.59453019e-01,  9.99961197e-02,\n",
       "        -3.64762358e-02,  3.51281822e-01,  1.44407198e-01,\n",
       "         2.42307618e-01, -7.36036301e-01, -5.60946226e-01,\n",
       "        -1.87428370e-01,  3.36208165e-01, -4.80296880e-01,\n",
       "        -3.68308753e-01, -5.68377376e-01, -4.56657797e-01,\n",
       "        -7.02095687e-01, -1.54304933e-02,  3.06934774e-01,\n",
       "         2.63164937e-01,  3.38008016e-01,  9.23087522e-02,\n",
       "        -4.15672511e-01, -3.40315491e-01, -1.73356783e+00,\n",
       "         5.65594919e-02, -4.55609143e-01,  1.12783812e-01,\n",
       "        -2.25903898e-01,  2.04137489e-01,  3.44312698e-01,\n",
       "         2.50567086e-02,  9.63601246e-02, -7.12236613e-02,\n",
       "         1.45250082e-01, -4.04904634e-01, -2.34389901e-02,\n",
       "        -2.50297695e-01, -9.20385644e-02,  2.23543033e-01,\n",
       "        -1.35792625e+00, -3.98427963e-01, -6.01878762e-01,\n",
       "        -7.61914253e-02, -2.64523327e-01, -4.41331446e-01,\n",
       "         4.81946506e-02,  3.58299881e-01, -1.33150589e+00,\n",
       "         2.17106026e-02,  1.53144225e-01, -4.04418260e-01,\n",
       "        -5.83852768e-01, -7.22645700e-01,  3.77289727e-02,\n",
       "        -5.03761351e-01,  1.83343068e-01, -5.16875554e-03,\n",
       "         2.33440876e-01, -4.99748409e-01,  6.85964167e-01,\n",
       "        -7.93868527e-02,  3.08637947e-01, -1.59952328e-01,\n",
       "        -5.87115049e-01, -2.52798408e-01, -2.85169035e-01,\n",
       "         1.18621252e-01,  2.65130311e-01,  6.61568716e-02,\n",
       "         3.45352814e-02, -6.27703667e-01, -4.70491767e-01,\n",
       "        -3.76360327e-01, -1.35674372e-01, -3.00186932e-01,\n",
       "        -4.05924976e-01,  4.01468985e-02, -4.80753839e-01,\n",
       "        -6.21602237e-01, -7.45006576e-02, -8.79777730e-01,\n",
       "         1.89425126e-01,  5.04526019e-01, -2.83185780e-01,\n",
       "        -5.44335246e-01, -1.37117431e-01,  2.96508789e-01,\n",
       "        -2.39186972e-01,  1.59068197e-01,  3.14902931e-01,\n",
       "        -4.26553041e-01, -4.28237826e-01,  1.53175756e-01,\n",
       "        -4.79634196e-01, -1.42727286e-01, -2.89246142e-01,\n",
       "         3.99240941e-01, -3.32198650e-01, -2.85587218e-02,\n",
       "        -6.20941162e-01, -1.69072226e-01, -2.96477854e-01,\n",
       "        -1.12306750e+00, -5.29507816e-01,  1.20835865e+00,\n",
       "        -4.32094634e-01, -8.64042565e-02, -3.50320339e-03,\n",
       "        -1.35401070e-01, -3.38872135e-01, -2.52410740e-01,\n",
       "         9.61904377e-02,  4.19668972e-01, -2.71689534e-01,\n",
       "        -1.45881668e-01,  1.23759203e-01,  2.13959023e-01,\n",
       "         1.59111038e-01,  3.57742131e-01, -1.14355600e+00,\n",
       "        -8.31822380e-02,  6.55236959e-01, -2.34257564e-01,\n",
       "         6.68020010e-01, -3.47833216e-01, -6.14469126e-02,\n",
       "         1.16835445e-01, -3.47783864e-01, -1.11869529e-01,\n",
       "        -5.78172624e-01, -1.45530611e-01,  2.53924131e-01,\n",
       "         2.52304554e-01, -3.85980338e-01,  2.52888829e-01,\n",
       "        -1.17506318e-01, -4.99595672e-01,  6.26745597e-02,\n",
       "        -2.16234126e-03, -3.49671751e-01,  2.32552290e-01,\n",
       "        -9.08970013e-02, -6.51567057e-02, -2.55361140e-01,\n",
       "         6.48920313e-02, -3.53196412e-01, -6.00122690e-01,\n",
       "        -7.58729696e-01,  1.76420614e-01, -7.12873638e-02,\n",
       "        -2.55982369e-01, -2.59140015e-01, -7.13040978e-02,\n",
       "        -5.95095381e-02, -3.06402087e-01, -2.96071351e-01,\n",
       "        -1.00795776e-01,  1.87205553e-01, -1.47629138e-02,\n",
       "        -6.01508379e-01, -3.39255303e-01,  3.23090255e-01,\n",
       "         7.57319272e-01, -2.00505510e-01, -5.29878855e-01,\n",
       "         6.48645535e-02,  4.61983271e-02, -4.58673209e-01,\n",
       "        -2.86636919e-01,  2.50961751e-01, -6.18168414e-01,\n",
       "         4.05810237e-01, -6.87813312e-02,  4.24551457e-01,\n",
       "        -6.21736906e-02, -3.39808732e-01, -9.69122112e-01,\n",
       "        -9.13546234e-02,  1.42469034e-01, -3.27545643e-01,\n",
       "        -4.83615309e-01, -7.27520227e-01, -5.70859492e-01,\n",
       "         1.00367412e-01, -8.45976651e-01, -2.43919462e-01,\n",
       "         5.78141324e-02, -3.02724183e-01,  1.06635001e-02,\n",
       "         8.99900943e-02, -2.33167678e-01, -3.11468029e+00,\n",
       "         3.29167366e-01, -1.39614671e-01,  1.02732949e-01,\n",
       "        -9.09850672e-02,  8.20486397e-02,  3.56283933e-01,\n",
       "         1.19334078e+00,  8.39916766e-02,  5.25472313e-03,\n",
       "         9.40003246e-02, -4.75271851e-01, -3.02917473e-02,\n",
       "         4.20691133e-01, -2.70276088e-02, -1.62039652e-01,\n",
       "         3.78889479e-02, -2.65558988e-01, -1.00597069e-01,\n",
       "         1.69131707e-03, -4.98750597e-01, -7.69661069e-01,\n",
       "         7.53794163e-02, -2.00716943e-01,  3.21471155e-01,\n",
       "         6.44898415e-03, -2.47086808e-01, -2.70209432e-01,\n",
       "        -7.71890402e-01, -4.39381182e-01, -3.15874428e-01,\n",
       "        -2.83888459e-01, -1.42223746e-01, -2.45840862e-01,\n",
       "         1.71641819e-02, -3.88524830e-01, -1.42029703e-01,\n",
       "        -2.24418148e-01,  4.37545419e-01, -1.23886624e-02,\n",
       "         3.48796844e-01, -4.78708036e-02, -3.33002478e-01,\n",
       "        -5.60467958e-01,  1.76142991e-01,  2.88320988e-01,\n",
       "        -6.50513411e-01, -1.89969555e-01,  3.77421416e-02,\n",
       "        -4.02462691e-01,  2.74134248e-01, -3.48836720e-01,\n",
       "        -6.56167150e-01, -5.83127201e-01, -9.67237912e-03,\n",
       "         2.55938470e-01, -3.37733388e-01,  4.18113470e-02,\n",
       "        -4.22189832e-01, -1.48968473e-01,  6.61205640e-03,\n",
       "        -1.50332510e-01, -1.95643887e-01, -4.71435487e-01,\n",
       "         3.83720934e-01,  1.89864829e-01,  7.33698547e-01,\n",
       "        -1.09784633e-01, -4.95695621e-01,  5.03359437e-02,\n",
       "        -3.68161239e-02, -3.58069211e-01, -1.54379047e-02,\n",
       "        -2.72298735e-02,  8.80401492e-01, -1.29185840e-01,\n",
       "         3.61403018e-01, -2.78489649e-01,  1.58044040e-01,\n",
       "        -3.34318757e-01,  2.64281243e-01, -1.67408898e-01,\n",
       "         4.37207427e-03,  3.04006577e-01,  8.28974769e-02,\n",
       "        -4.43531722e-01, -7.67219841e-01,  1.27058297e-01,\n",
       "        -1.27811134e-01, -5.65582871e-01, -2.30786413e-01,\n",
       "        -6.54357135e-01,  6.04460418e-01, -1.91539571e-01,\n",
       "        -1.27589151e-01,  9.39528406e-01, -5.85638881e-01,\n",
       "         1.69813335e-01, -2.98428982e-01,  3.85908812e-01,\n",
       "        -1.69386700e-01, -3.00402671e-01,  1.76812693e-01,\n",
       "        -6.39458597e-01,  6.05480112e-02,  3.62400979e-01,\n",
       "        -6.38770759e-01, -1.89983839e-04,  1.84139594e-01,\n",
       "        -3.36119473e-01, -1.06192760e-01, -1.04941301e-01,\n",
       "        -3.88572544e-01, -1.81992054e-01,  2.77762234e-01,\n",
       "        -8.13040882e-02, -5.41395545e-01,  4.51610237e-01,\n",
       "        -2.73833722e-01, -2.82287061e-01,  5.22313356e-01,\n",
       "        -1.11360565e-01, -7.59104908e-01, -2.18092948e-01,\n",
       "         5.26549578e-01, -3.63157019e-02,  2.44014591e-01,\n",
       "        -1.01800568e-01,  4.35275622e-02, -3.86084691e-02,\n",
       "        -1.61702067e-01, -6.13519013e-01, -4.59829539e-01,\n",
       "         1.51405871e-01, -1.71733990e-01,  2.16186251e-02,\n",
       "        -3.03413689e-01, -5.67613482e-01, -3.47698340e-03,\n",
       "        -2.95787632e-01, -4.13330495e-01, -1.43417612e-01,\n",
       "        -3.43641818e-01, -3.13947722e-02, -7.09285617e-01,\n",
       "        -4.48446780e-01, -1.17010243e-01,  7.53774121e-02,\n",
       "         7.95772523e-02, -6.49365112e-02,  2.47721970e-01,\n",
       "        -9.75487977e-02, -1.74207255e-01, -2.81795770e-01,\n",
       "        -1.74393207e-01, -2.36101121e-01, -3.46551776e-01,\n",
       "         2.31090814e-01, -2.10902914e-01, -4.01737988e-01,\n",
       "         3.49367075e-02,  3.29537839e-01,  1.82514027e-01,\n",
       "        -9.21233714e-01, -1.78981751e-01,  9.38111320e-02,\n",
       "        -6.31086975e-02, -3.83345991e-01,  3.16464812e-01,\n",
       "         4.24019307e-01, -1.99327439e-01, -4.19511162e-02,\n",
       "         1.26034338e-02,  3.82632822e-01,  7.88045898e-02,\n",
       "        -2.43515298e-01,  1.11870784e-02,  7.11549699e-01,\n",
       "        -1.06267706e-01, -3.13524067e-01, -3.89124788e-02,\n",
       "        -5.21773517e-01, -7.98664168e-02,  7.38609284e-02,\n",
       "        -4.83713932e-02,  3.27068180e-01,  2.73987770e-01,\n",
       "        -1.64234102e-01,  1.95488939e-03, -1.73914358e-01,\n",
       "         1.41127352e-02, -5.05725890e-02, -2.91822225e-01,\n",
       "        -6.98175430e-01,  5.74841164e-02, -7.65891075e-01,\n",
       "         2.86359519e-01, -2.18708694e-01, -6.94723707e-03,\n",
       "        -5.80258071e-01,  5.33908568e-02, -9.47833210e-02,\n",
       "        -8.24903920e-02,  4.77858514e-01, -4.94444937e-01,\n",
       "        -5.06340683e-01, -8.48833382e-01, -1.35629809e+00,\n",
       "        -3.92206341e-01,  2.66225278e-01,  4.72581536e-01,\n",
       "         4.74547520e-02, -8.40480626e-02, -6.11177325e-01,\n",
       "        -2.77258128e-01, -4.33366865e-01,  8.09905753e-02,\n",
       "        -2.94399589e-01,  8.31183568e-02,  1.21778235e-01,\n",
       "        -4.86485690e-01,  2.07453161e-01, -3.73955250e-01,\n",
       "        -3.13676775e-01, -3.68659437e-01,  1.30866393e-01,\n",
       "        -1.02569740e-02,  2.28801996e-01,  1.24097347e-01,\n",
       "        -1.70713514e-01, -4.31590468e-01,  5.89256883e-01,\n",
       "        -1.13135383e-01, -3.66987705e-01,  2.04518795e+00,\n",
       "         3.01996708e-01, -3.01489174e-01, -3.45577896e-01,\n",
       "        -4.48928267e-01, -9.99411754e-03, -1.28506586e-01,\n",
       "        -1.52878538e-01,  2.88957179e-01, -9.74642560e-02,\n",
       "        -2.06355914e-01, -1.80277810e-01,  1.40995517e-01,\n",
       "         1.74816370e-01,  8.93046409e-02, -1.18417531e-01,\n",
       "        -1.87856808e-01, -1.78235397e-02, -7.73711503e-01,\n",
       "        -2.25853443e-01,  1.09384544e-01, -2.71458048e-02,\n",
       "        -4.08823818e-01,  4.82968576e-02, -5.72387159e-01,\n",
       "        -2.01550424e-01, -7.58018434e-01, -8.02606866e-02,\n",
       "         1.43199056e-01, -2.54355609e-01, -7.71069080e-02,\n",
       "        -7.10115582e-02, -8.19679648e-02, -3.81209552e-01,\n",
       "        -7.45249838e-02,  2.51101762e-01, -1.20165519e-01,\n",
       "         1.25487987e-02, -6.74843431e-01,  6.15665436e-01,\n",
       "        -2.00628415e-02, -5.02994776e-01,  2.55705088e-01,\n",
       "        -7.34702963e-03, -6.62068129e-01,  2.28922307e-01,\n",
       "        -7.45225012e-01, -1.20327860e-01, -3.93408298e-01,\n",
       "         1.37418017e-01,  2.78632522e-01, -1.75016269e-01,\n",
       "         1.22510949e-02, -2.72779148e-02, -6.05924189e-01,\n",
       "        -9.28303376e-02,  1.95985392e-01, -2.35154275e-02,\n",
       "        -4.10495132e-01,  9.41901356e-02, -2.48037353e-01,\n",
       "        -2.63281137e-01, -3.56169403e-01, -6.83451816e-02,\n",
       "        -6.24911010e-01, -3.70503485e-01,  4.48977835e-02,\n",
       "        -8.14883560e-02, -6.33918583e-01,  9.85705629e-02,\n",
       "         2.71174490e-01, -3.89516018e-02, -1.05085798e-01,\n",
       "        -4.01222348e-01, -3.58504832e-01, -6.68667495e-01,\n",
       "        -4.02835965e-01, -7.14626491e-01, -5.28744340e-01,\n",
       "        -6.12608492e-01,  1.63218394e-01,  2.29248464e-01,\n",
       "         8.38347301e-02, -4.62544966e-04, -1.57136664e-01,\n",
       "         8.11288580e-02, -1.69787228e-01, -4.18063372e-01,\n",
       "         9.97973740e-01,  6.68053329e-01, -3.24505985e-01,\n",
       "        -7.89365359e-03, -7.25979060e-02, -2.89816201e-01,\n",
       "         3.91992509e-01, -3.11666787e-01,  2.58146435e-01,\n",
       "         9.12462026e-02, -3.82214278e-01, -1.45118624e-01]], dtype=float32)>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow_hub as hub\n",
    "import tensorflow_text as text\n",
    "\n",
    "preprocessor = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder-cmlm/multilingual-preprocess/2\")\n",
    "encoder = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder-cmlm/multilingual-base/1\")\n",
    "\n",
    "\n",
    "def get_embeddings(sentences):\n",
    "    preprocessed_text = preprocessor(sentences)\n",
    "    return encoder(preprocessed_text)['pooled_output']\n",
    "\n",
    "\n",
    "get_embeddings([\n",
    "    \"Questa collezione di Haiku è una porta aperta sulla cultura giapponese.\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Computing cosine similarity\n",
    "import seaborn as sns\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_similarity(features, labels):\n",
    "    cos_sim = cosine_similarity(features)\n",
    "    fig = plt.figure(figsize=(10,8))\n",
    "    sns.set(font_scale=1.2)\n",
    "    cbar_kws=dict(use_gridspec=False, location=\"left\")\n",
    "    g = sns.heatmap(cos_sim, xticklabels=labels, yticklabels=labels,\n",
    "      vmin=0, vmax=1, annot=True, cmap=\"Blues\", cbar_kws=cbar_kws)\n",
    "    g.tick_params(labelright=True, labelleft=False)\n",
    "    g.set_yticklabels(labels, rotation=0)\n",
    "    g.set_title(\"Semantic Textual Similarity\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Precision, Recall, F1 Score\n",
    "from keras import backend as K\n",
    "\n",
    "def balanced_recall(y_true, y_pred):\n",
    "    \"\"\"This function calculates the balanced recall metric\n",
    "    recall = TP / (TP + FN)\n",
    "    \"\"\"\n",
    "    recall_by_class = 0\n",
    "    # iterate over each predicted class to get class-specific metric\n",
    "    for i in range(y_pred.shape[1]):\n",
    "        y_pred_class = y_pred[:, i]\n",
    "        y_true_class = y_true[:, i]\n",
    "        true_positives = K.sum(K.round(K.clip(y_true_class * y_pred_class, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true_class, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        recall_by_class = recall_by_class + recall\n",
    "    return recall_by_class / y_pred.shape[1]\n",
    "\n",
    "def balanced_precision(y_true, y_pred):\n",
    "    \"\"\"This function calculates the balanced precision metric\n",
    "    precision = TP / (TP + FP)\n",
    "    \"\"\"\n",
    "    precision_by_class = 0\n",
    "    # iterate over each predicted class to get class-specific metric\n",
    "    for i in range(y_pred.shape[1]):\n",
    "        y_pred_class = y_pred[:, i]\n",
    "        y_true_class = y_true[:, i]\n",
    "        true_positives = K.sum(K.round(K.clip(y_true_class * y_pred_class, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred_class, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        precision_by_class = precision_by_class + precision\n",
    "    # return average balanced metric for each class\n",
    "    return precision_by_class / y_pred.shape[1]\n",
    "\n",
    "def balanced_f1_score(y_true, y_pred):\n",
    "    \"\"\"This function calculates the F1 score metric\"\"\"\n",
    "    precision = balanced_precision(y_true, y_pred)\n",
    "    recall = balanced_recall(y_true, y_pred)\n",
    "    return 2 * ((precision * recall) / (precision + recall + K.epsilon()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Defining the model\n",
    "import tensorflow as tf\n",
    "i = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
    "x = preprocessor(i)\n",
    "x = encoder(x)\n",
    "x = tf.keras.layers.Dropout(0.2, name=\"dropout\")(x['pooled_output'])\n",
    "x = tf.keras.layers.Dense(13, activation='softmax', name=\"output\")(x)\n",
    "\n",
    "model = tf.keras.Model(i, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_acc = tf.keras.metrics.CategoricalAccuracy(name=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "WARNING:tensorflow:AutoGraph could not transform <function Model.make_train_function.<locals>.train_function at 0x0000019C5BB7DE58> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <function Model.make_train_function.<locals>.train_function at 0x0000019C5BB7DE58> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: AutoGraph could not transform <function Model.make_train_function.<locals>.train_function at 0x0000019C5BB7DE58> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "27/27 [==============================] - ETA: 0s - loss: 2.2927 - accuracy: 0.2155 - balanced_recall: 0.0055 - balanced_precision: 0.0107 - balanced_f1_score: 0.0071WARNING:tensorflow:AutoGraph could not transform <function Model.make_test_function.<locals>.test_function at 0x0000019C62419B88> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <function Model.make_test_function.<locals>.test_function at 0x0000019C62419B88> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: AutoGraph could not transform <function Model.make_test_function.<locals>.test_function at 0x0000019C62419B88> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "27/27 [==============================] - 267s 10s/step - loss: 2.2927 - accuracy: 0.2155 - balanced_recall: 0.0055 - balanced_precision: 0.0107 - balanced_f1_score: 0.0071 - val_loss: 2.1149 - val_accuracy: 0.3000 - val_balanced_recall: 0.0164 - val_balanced_precision: 0.0342 - val_balanced_f1_score: 0.0205\n",
      "Epoch 2/20\n",
      "27/27 [==============================] - 224s 8s/step - loss: 1.9825 - accuracy: 0.3702 - balanced_recall: 0.0209 - balanced_precision: 0.0313 - balanced_f1_score: 0.0237 - val_loss: 1.9429 - val_accuracy: 0.3964 - val_balanced_recall: 0.0564 - val_balanced_precision: 0.0627 - val_balanced_f1_score: 0.0549\n",
      "Epoch 3/20\n",
      "27/27 [==============================] - 226s 8s/step - loss: 1.7747 - accuracy: 0.4488 - balanced_recall: 0.0521 - balanced_precision: 0.0883 - balanced_f1_score: 0.0623 - val_loss: 1.8086 - val_accuracy: 0.4179 - val_balanced_recall: 0.0681 - val_balanced_precision: 0.0926 - val_balanced_f1_score: 0.0750\n",
      "Epoch 4/20\n",
      "27/27 [==============================] - 233s 9s/step - loss: 1.6521 - accuracy: 0.5048 - balanced_recall: 0.0656 - balanced_precision: 0.1113 - balanced_f1_score: 0.0791 - val_loss: 1.7182 - val_accuracy: 0.4857 - val_balanced_recall: 0.0845 - val_balanced_precision: 0.1268 - val_balanced_f1_score: 0.0966\n",
      "Epoch 5/20\n",
      "27/27 [==============================] - 245s 9s/step - loss: 1.5185 - accuracy: 0.5143 - balanced_recall: 0.0935 - balanced_precision: 0.1876 - balanced_f1_score: 0.1224 - val_loss: 1.6417 - val_accuracy: 0.4607 - val_balanced_recall: 0.0845 - val_balanced_precision: 0.1353 - val_balanced_f1_score: 0.1000\n",
      "Epoch 6/20\n",
      "27/27 [==============================] - 232s 9s/step - loss: 1.3933 - accuracy: 0.5583 - balanced_recall: 0.1396 - balanced_precision: 0.2622 - balanced_f1_score: 0.1786 - val_loss: 1.5844 - val_accuracy: 0.4679 - val_balanced_recall: 0.0954 - val_balanced_precision: 0.1510 - val_balanced_f1_score: 0.1142\n",
      "Epoch 7/20\n",
      "27/27 [==============================] - 234s 9s/step - loss: 1.3211 - accuracy: 0.5786 - balanced_recall: 0.1615 - balanced_precision: 0.3097 - balanced_f1_score: 0.2105 - val_loss: 1.5538 - val_accuracy: 0.4857 - val_balanced_recall: 0.1287 - val_balanced_precision: 0.2229 - val_balanced_f1_score: 0.1605\n",
      "Epoch 8/20\n",
      "27/27 [==============================] - 243s 9s/step - loss: 1.2610 - accuracy: 0.5988 - balanced_recall: 0.1870 - balanced_precision: 0.3160 - balanced_f1_score: 0.2309 - val_loss: 1.5065 - val_accuracy: 0.4929 - val_balanced_recall: 0.1463 - val_balanced_precision: 0.2392 - val_balanced_f1_score: 0.1781\n",
      "Epoch 9/20\n",
      "27/27 [==============================] - 224s 8s/step - loss: 1.1928 - accuracy: 0.6238 - balanced_recall: 0.2121 - balanced_precision: 0.3587 - balanced_f1_score: 0.2649 - val_loss: 1.4858 - val_accuracy: 0.4821 - val_balanced_recall: 0.1703 - val_balanced_precision: 0.2785 - val_balanced_f1_score: 0.2064\n",
      "Epoch 10/20\n",
      "27/27 [==============================] - 226s 8s/step - loss: 1.1509 - accuracy: 0.6286 - balanced_recall: 0.2438 - balanced_precision: 0.4019 - balanced_f1_score: 0.3010 - val_loss: 1.4632 - val_accuracy: 0.4893 - val_balanced_recall: 0.1810 - val_balanced_precision: 0.2849 - val_balanced_f1_score: 0.2175\n",
      "Epoch 11/20\n",
      "27/27 [==============================] - 224s 8s/step - loss: 1.1057 - accuracy: 0.6393 - balanced_recall: 0.2730 - balanced_precision: 0.4330 - balanced_f1_score: 0.3329 - val_loss: 1.4483 - val_accuracy: 0.4821 - val_balanced_recall: 0.1895 - val_balanced_precision: 0.2870 - val_balanced_f1_score: 0.2229\n",
      "Epoch 12/20\n",
      "27/27 [==============================] - 230s 9s/step - loss: 1.0651 - accuracy: 0.6488 - balanced_recall: 0.2655 - balanced_precision: 0.4291 - balanced_f1_score: 0.3256 - val_loss: 1.4361 - val_accuracy: 0.5000 - val_balanced_recall: 0.2036 - val_balanced_precision: 0.3019 - val_balanced_f1_score: 0.2391\n",
      "Epoch 13/20\n",
      "27/27 [==============================] - 231s 9s/step - loss: 1.0174 - accuracy: 0.6726 - balanced_recall: 0.3036 - balanced_precision: 0.4919 - balanced_f1_score: 0.3720 - val_loss: 1.4186 - val_accuracy: 0.4786 - val_balanced_recall: 0.2153 - val_balanced_precision: 0.3105 - val_balanced_f1_score: 0.2502\n",
      "Epoch 14/20\n",
      "27/27 [==============================] - 217s 8s/step - loss: 0.9993 - accuracy: 0.6774 - balanced_recall: 0.3125 - balanced_precision: 0.4804 - balanced_f1_score: 0.3758 - val_loss: 1.4202 - val_accuracy: 0.4929 - val_balanced_recall: 0.2167 - val_balanced_precision: 0.3170 - val_balanced_f1_score: 0.2538\n",
      "Epoch 15/20\n",
      "27/27 [==============================] - 239s 9s/step - loss: 0.9701 - accuracy: 0.6714 - balanced_recall: 0.3357 - balanced_precision: 0.5050 - balanced_f1_score: 0.4007 - val_loss: 1.4048 - val_accuracy: 0.4964 - val_balanced_recall: 0.2267 - val_balanced_precision: 0.3312 - val_balanced_f1_score: 0.2651\n",
      "Epoch 16/20\n",
      "27/27 [==============================] - 245s 9s/step - loss: 0.9348 - accuracy: 0.6821 - balanced_recall: 0.3560 - balanced_precision: 0.5218 - balanced_f1_score: 0.4211 - val_loss: 1.4036 - val_accuracy: 0.4786 - val_balanced_recall: 0.2346 - val_balanced_precision: 0.3298 - val_balanced_f1_score: 0.2710\n",
      "Epoch 17/20\n",
      "27/27 [==============================] - 235s 9s/step - loss: 0.9413 - accuracy: 0.6738 - balanced_recall: 0.3374 - balanced_precision: 0.4849 - balanced_f1_score: 0.3961 - val_loss: 1.4041 - val_accuracy: 0.4679 - val_balanced_recall: 0.2495 - val_balanced_precision: 0.3383 - val_balanced_f1_score: 0.2833\n",
      "Epoch 18/20\n",
      "27/27 [==============================] - 216s 8s/step - loss: 0.9005 - accuracy: 0.6952 - balanced_recall: 0.3672 - balanced_precision: 0.5174 - balanced_f1_score: 0.4262 - val_loss: 1.4040 - val_accuracy: 0.4679 - val_balanced_recall: 0.2427 - val_balanced_precision: 0.3319 - val_balanced_f1_score: 0.2764\n",
      "Epoch 19/20\n",
      "27/27 [==============================] - 216s 8s/step - loss: 0.8911 - accuracy: 0.6893 - balanced_recall: 0.3989 - balanced_precision: 0.5476 - balanced_f1_score: 0.4595 - val_loss: 1.3969 - val_accuracy: 0.4821 - val_balanced_recall: 0.2448 - val_balanced_precision: 0.3350 - val_balanced_f1_score: 0.2793\n",
      "Epoch 20/20\n",
      "27/27 [==============================] - 217s 8s/step - loss: 0.8681 - accuracy: 0.7095 - balanced_recall: 0.3826 - balanced_precision: 0.5319 - balanced_f1_score: 0.4428 - val_loss: 1.3952 - val_accuracy: 0.4821 - val_balanced_recall: 0.2495 - val_balanced_precision: 0.3536 - val_balanced_f1_score: 0.2899\n"
     ]
    }
   ],
   "source": [
    "# Training the model\n",
    "\n",
    "n_epochs = 20\n",
    "\n",
    "METRICS = [\n",
    "      tf.keras.metrics.CategoricalAccuracy(name=\"accuracy\"),\n",
    "      balanced_recall,\n",
    "      balanced_precision,\n",
    "      balanced_f1_score\n",
    "]\n",
    "\n",
    "earlystop_callback = tf.keras.callbacks.EarlyStopping(monitor = \"val_loss\", \n",
    "                                                      patience = 3,\n",
    "                                                      restore_best_weights = True)\n",
    "\n",
    "model.compile(optimizer = \"adam\",\n",
    "              loss = \"categorical_crossentropy\",\n",
    "              metrics = METRICS)\n",
    "\n",
    "model_fit = model.fit(x_train, \n",
    "                      y_train, \n",
    "                      epochs = n_epochs,\n",
    "                      validation_data = (x_test, y_test),\n",
    "                      callbacks = [earlystop_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.save('/saved_model/my_model')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed\n",
    "import joblib\n",
    "\n",
    "joblib.dump(model, 'savedModel1.pkl')\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the saved model\n",
    "from tensorflow import keras\n",
    "\n",
    "new_model = keras.models.load_model(\"/saved_model/my_model\", compile=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using these titles from cryptomarkets subreddit to predit the flair\n",
    "titles = ['Yahoo is Launching Metaverse Events in Hong Kong',\n",
    "         \"My prediction. BTC around 17K by Sunday. ETH around 700\",\n",
    "          \"The markets will rebound, they always rebound for everything else, crypto shouldn't be the exception\",\n",
    "          \"Through our actions we have turned crypto into just another financial asset and it had the potential for much more\",\n",
    "          \"Federal Reserve announces Highest Interest Rate-hike since 1994\"\n",
    "         ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 1s 1s/step\n",
      "12   Educational\n",
      "0   News\n",
      "1   FUNDAMENTALS\n",
      "12   Educational\n",
      "2   NEWS\n"
     ]
    }
   ],
   "source": [
    "res = [np.argmax(pred) for pred in model.predict(titles)]\n",
    "\n",
    "for i in res:\n",
    "    print(i, ' ', flair_dict[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}