names-EDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Link to Github repo](https://github.com/anushalihala/community-names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.probability import FreqDist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "RANDOM_STATE=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('15000_demoscrape.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Surname</th>\n",
       "      <th>Community</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yogita</td>\n",
       "      <td>singh</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dibyendu</td>\n",
       "      <td>ghosh</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>veeramony</td>\n",
       "      <td>ramachandran</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>amlan</td>\n",
       "      <td>datta</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>susmita</td>\n",
       "      <td>poddar</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Name       Surname Community\n",
       "0     yogita         singh   punjabi\n",
       "1   dibyendu         ghosh   bengali\n",
       "2  veeramony  ramachandran     tamil\n",
       "3      amlan         datta   bengali\n",
       "4    susmita        poddar   bengali"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 14999 entries, 0 to 14998\n",
      "Data columns (total 3 columns):\n",
      "Name         14999 non-null object\n",
      "Surname      14999 non-null object\n",
      "Community    14999 non-null object\n",
      "dtypes: object(3)\n",
      "memory usage: 468.7+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualisations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking at first names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "rajesh         121\n",
       "ashok           84\n",
       "vijay           78\n",
       "amit            77\n",
       "abhishek        76\n",
       "sandeep         74\n",
       "sanjay          69\n",
       "santosh         64\n",
       "suresh          63\n",
       "priyanka        61\n",
       "mohammed        60\n",
       "mukesh          60\n",
       "vinod           58\n",
       "pradeep         58\n",
       "rajeev          57\n",
       "nitin           57\n",
       "ajay            56\n",
       "mohd            55\n",
       "vikas           55\n",
       "ravi            54\n",
       "prakash         52\n",
       "vishal          51\n",
       "manoj           50\n",
       "suman           48\n",
       "mahesh          48\n",
       "rohit           47\n",
       "rajiv           44\n",
       "ashish          42\n",
       "anil            39\n",
       "rekha           39\n",
       "              ... \n",
       "baidyanath       1\n",
       "subhresh         1\n",
       "deepen           1\n",
       "kanya            1\n",
       "ramavtar         1\n",
       "janki            1\n",
       "mit              1\n",
       "christopher      1\n",
       "apollo           1\n",
       "sivaraman        1\n",
       "maulik           1\n",
       "sibith           1\n",
       "teji             1\n",
       "stayam           1\n",
       "r.k.             1\n",
       "saagar           1\n",
       "mandira          1\n",
       "raka             1\n",
       "swastik          1\n",
       "gaggan           1\n",
       "raghwendra       1\n",
       "safeeza          1\n",
       "udayanath        1\n",
       "avimanyu         1\n",
       "kiron            1\n",
       "mitul            1\n",
       "bore             1\n",
       "tandrima         1\n",
       "avinanda         1\n",
       "sachikant        1\n",
       "Name: Name, Length: 5222, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Name'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1304affca90>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAADuCAYAAAA3IMxxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAEx9JREFUeJzt3XuQXGWZx/Hv0z2TTC6EhGSIuQABDHcR3AFB11WJlNyWsFXiQlmacrFSVuGq69YuuLtVWGtZpbuuq255qSywxpVFWcQi66KYQilkSy4TbnI1EUgyBJKJJEDuzOTdP6YnDKFPd+ieTs858/1Upbr79Ok+T//zmzfPec97IqWEJKm4Su0uQJLUWga9JBWcQS9JBWfQS1LBGfSSVHAGvSQVnEEvSQVn0EtSwRn0klRwHe0uAGDWrFlpwYIF7S5DknJl1apVm1NK3fX2GxNBv2DBAnp7e9tdhiTlSkSsPZD9bN1IUsEZ9JJUcAa9JBWcQS9JBWfQS1LBGfSSVHB1gz4iro+ITRHx6Iht/xwRT0bEIxHxk4iYPuK9z0fEmoh4KiI+2KrCAVJK7N2b8C5ZkpTtQEb03wPO22/bSuCUlNKpwO+AzwNExEnAZcDJlc98OyLKo1btfm777Qsc83e3sXrTtlYdQpJyr27Qp5TuAl7cb9svUkoDlZf3APMrzxcDP0wp7U4pPQOsAc4cxXozamz1ESQpv0ajR/8XwM8qz+cB60e811fZ9gYRsTQieiOit7+/v6EDRzT0MUkaV5oK+oj4e2AAuGF4U5Xdqo63U0rLUko9KaWe7u66SzXUlKofQpJEE2vdRMQS4CJgUXrtbGgfcMSI3eYDGxovr04NlUdbN5KUraERfUScB1wFXJxS2jHirRXAZRExMSKOBhYC9zVfZlYdrfpmSSqOuiP6iLgReB8wKyL6gGsYmmUzEVgZQ2l7T0rpkymlxyLiJuBxhlo6V6aUBltV/DBH9JKUrW7Qp5Qur7L5uhr7fwn4UjNFHTiH9JJUTyGujPVkrCRly3XQD/fobd1IUrZ8B327C5CkHMh10EuS6st10IfzKyWprlwH/TB79JKULddBv+/KWGfdSFKmfAe9nRtJqivXQT/M1o0kZct10O+bR9/eMiRpTMt30DuTXpLqynXQD/OesZKULd9B74BekurKd9BXOJ6XpGy5DnrvMCVJ9eU76J1IL0l15TroX+OQXpKy5DroHc9LUn25Dvph9uglKVuug94rYyWpvnwHvc0bSaor10E/zNaNJGXLddC/dnNwk16SsuQ76NtdgCTlQN2gj4jrI2JTRDw6YtthEbEyIlZXHmdUtkdEfDMi1kTEIxHxjlYWP8zxvCRlO5AR/feA8/bbdjVwR0ppIXBH5TXA+cDCyr+lwHdGp8wMDuklqa66QZ9Sugt4cb/Ni4HllefLgUtGbP9+GnIPMD0i5oxWsdk1tvoIkpRfjfboZ6eUngeoPB5e2T4PWD9iv77KtjeIiKUR0RsRvf39/Q0VMTy90puDS1K20T4ZW62ZUjWFU0rLUko9KaWe7u7uxg5m60aS6mo06DcOt2Qqj5sq2/uAI0bsNx/Y0Hh5B8gBvSRlajToVwBLKs+XALeO2P6xyuybs4CXhls8reCAXpLq66i3Q0TcCLwPmBURfcA1wJeBmyLiCmAdcGll99uAC4A1wA7g4y2o+Q0c0EtStrpBn1K6POOtRVX2TcCVzRZ1oIZvPOKsG0nKlu8rY+3dSFJduQ76YU6vlKRsuQ56bw4uSfXlO+ht3UhSXbkO+mEO6CUpW86D3iG9JNWT86Af4o1HJClbroPem4NLUn35Dvp2FyBJOZDroN/HIb0kZcp10IfzKyWprlwH/TCvjJWkbLkOeq+MlaT68h30dm4kqa5cB/0wR/SSlC3XQf/azcElSVnyHfS2biSprlwH/TCXQJCkbIUIeklStkIEveN5ScqW66Dft6iZSS9JmfId9C5rJkl15TroX+OQXpKy5DronV4pSfU1FfQR8VcR8VhEPBoRN0ZEV0QcHRH3RsTqiPhRREwYrWKz2KOXpGwNB31EzAM+DfSklE4BysBlwFeAf00pLQS2AFeMRqHVaxh6NOclKVuzrZsOYFJEdACTgeeBc4CbK+8vBy5p8hiZPBkrSfU1HPQppeeArwLrGAr4l4BVwNaU0kBltz5gXrXPR8TSiOiNiN7+/v5Gy6jU0tTHJanQmmndzAAWA0cDc4EpwPlVdq0awymlZSmlnpRST3d3d4M1DB/ApJekLM20bj4APJNS6k8pvQrcArwLmF5p5QDMBzY0WWMmGzeSVF8zQb8OOCsiJsfQzVsXAY8DvwI+VNlnCXBrcyXWZ+tGkrI106O/l6GTrg8Av6181zLgKuBzEbEGmAlcNwp1VuU8ekmqr6P+LtlSStcA1+y3+WngzGa+90BFJen3OqSXpEy5vjJ2Qnmo/FcHDXpJypLroO/cF/R721yJJI1dOQ/6odaNQS9J2fId9B1D5e8ZMOglKUu+g75kj16S6sl10Fdy3lk3klRDroO+PDy9cq9BL0lZch30pUrQDzqil6RM+Q76kiN6Saon10EPUC6FI3pJqiH/QR+BA3pJypb7oI+wdSNJteQ+6MulYNCgl6RM+Q/6sEcvSbXkPuhLpfDGI5JUQ/6DPrB1I0k15D7onV4pSbXlPuhLEc66kaQach/05VK4qJkk1ZD7oC9F4H1HJClb/oO+5DLFklRL7oO+HF4wJUm15D7oS/boJamm/Ad9GPSSVEtTQR8R0yPi5oh4MiKeiIizI+KwiFgZEasrjzNGq9hqbN1IUm3Njui/Afw8pXQC8HbgCeBq4I6U0kLgjsrrlnFRM0mqreGgj4hpwJ8A1wGklPaklLYCi4Hlld2WA5c0W2QtXZ0ldr462MpDSFKuNTOiPwboB/4jIh6MiGsjYgowO6X0PEDl8fBqH46IpRHRGxG9/f39DRcxZWIHO/YY9JKUpZmg7wDeAXwnpXQ6sJ030aZJKS1LKfWklHq6u7sbLmJSZ5mdBr0kZWom6PuAvpTSvZXXNzMU/BsjYg5A5XFTcyXWNnlCme17Blp5CEnKtYaDPqX0ArA+Io6vbFoEPA6sAJZUti0Bbm2qwjomT+xwRC9JNXQ0+fm/BG6IiAnA08DHGfrjcVNEXAGsAy5t8hg1Te4ss3nbnlYeQpJyramgTyk9BPRUeWtRM9/7ZuwaGBrNb9s9wNSJzf7dkqTiyf2VscfPPgTA9o0kZch90E/sLAOwy7n0klRV7oO+qxL0uwcMekmqJv9B3zH0E3a96t1HJKma/Ae9rRtJqqlAQe+IXpKqKUDQD/0EFzaTpOoKEPS2biSplvwHfYdBL0m15D/oK62bXQP26CWpmtwH/fAFU7sd0UtSVbkP+n0jeoNekqrKfdBPKJeIcHqlJGXJfdBHBF0dZUf0kpQh90EPMGlCed9yxZKk1ytE0Hd1lGzdSFKGQgR9R7nEwKBBL0nVFCTog4G9qd1lSNKYVIygLwUDgwa9JFVTkKAvOaKXpAzFCPpyMLDXHr0kVVOMoLd1I0mZChL0JUf0kpShGEFfdkQvSVmaDvqIKEfEgxHx08rroyPi3ohYHRE/iogJzZdZW7nk9EpJyjIaI/rPAE+MeP0V4F9TSguBLcAVo3CMmjrLtm4kKUtTQR8R84ELgWsrrwM4B7i5ssty4JJmjnEgPBkrSdmaHdF/HfhbYHg4PRPYmlIaqLzuA+ZV+2BELI2I3ojo7e/vb6oIr4yVpGwNB31EXARsSimtGrm5yq5VEziltCyl1JNS6unu7m60DADKpRJrNm1r6jskqag6mvjsu4GLI+ICoAuYxtAIf3pEdFRG9fOBDc2XWVu58udlcG+iXKr2t0aSxq+GR/Qppc+nlOanlBYAlwG/TCl9BPgV8KHKbkuAW5uuso6T5x4KwE5vPiJJb9CKefRXAZ+LiDUM9eyva8ExXmfKxKH/mGzfPVBnT0kaf5pp3eyTUroTuLPy/GngzNH43gM1ZWIZMOglqZpCXBnb1TkU9A+u29rmSiRp7ClE0J80ZxoA2/c4opek/RUi6GdNnQjA9t2ejJWk/RUi6Ls6S0TADkf0kvQGhQj6iGByZ5ltnoyVpDcoRNDD0AnZ/1uzud1lSNKYU5igP6Srg5d3OqKXpP0VJujnTp/EK7tebXcZkjTmFCboTz9yOtv3DDIw6Lr0kjRSYYK+HEOLmfVv293mSiRpbClM0C+cfQgA23bZp5ekkQoT9FMrC5s9s3l7myuRpLGlMEF/1MzJADy03vVuJGmkwgT9Md1T6eossXvAk7GSNFJhgh5gWlcnfVt2tLsMSRpTChX0HaXg9sc2OsVSkkYoVNC//4TDAVexlKSRChX0p84funfsNlexlKR9ChX0w/eO3bB1Z5srkaSxo1BBP62rE4D1L3pCVpKGFSroF86eCsB37vx9myuRpLGjUEE/59BJHDd7Kqs3bWt3KZI0ZhQq6OG1G4VvdnEzSQIKGPTnnDgbgF88trHNlUjS2NBw0EfEERHxq4h4IiIei4jPVLYfFhErI2J15XHG6JVb33sXdgPwsGveSBLQ3Ih+APjrlNKJwFnAlRFxEnA1cEdKaSFwR+X1QXPo5E46SsHDfQa9JEETQZ9Sej6l9EDl+SvAE8A8YDGwvLLbcuCSZot8s3oWzODJF15h7950sA8tSWPOqPToI2IBcDpwLzA7pfQ8DP0xAA7P+MzSiOiNiN7+/v7RKGOfow6bAsBa59NLUvNBHxFTgR8Dn00pvXygn0spLUsp9aSUerq7u5st43XOe9tbAPjayt+N6vdKUh41FfQR0clQyN+QUrqlsnljRMypvD8H2NRciW/e2cfMBOD2R19gj+vTSxrnmpl1E8B1wBMppa+NeGsFsKTyfAlwa+PlNaars8y5J81mz+Befvmk0ywljW/NjOjfDXwUOCciHqr8uwD4MnBuRKwGzq28Pui+cPHJAPzkwefacXhJGjM6Gv1gSuluIDLeXtTo946WOdO6AFj7hx3s3ZsolbJKlaRiK9yVscNKpWDxaXN58oVX+Nj197W7HElqm8IGPcCnFy2koxTcvWYzv+93oTNJ41Ohg/7Y7qlcff4JAHxhxWNtrkaS2qPQQQ9wxR8fzWlHTOfXqzdz0/3r212OJB10hQ/6iOAT7zkagBvvX8eW7XvaXJEkHVyFD3qAi06dy4WnzuHBdVs5/YsruXv15naXJEkHzbgIeoCrPngC/3DhiQDccO9aftv3UpsrkqSDY9wE/ZEzJ7PkXQuYPW0iP3v0Bb7408fbXZIkHRTjJugBOssl7r7qHD548mwe7tvKhd/8NT+6f127y5KklhpXQQ9DYX/ZmUfynoWz6Nuykx+veo41m17hpR2vtrs0SWqJcRf0AO8//nCuXXIGZyyYwX3PvsgHvnYX53/jrnaXJUktMS6Dftg/Lj6Ff7v8dC46dQ4bXtrFV29/imt//bR3ppJUKA0valYEc6dPYu70SUyZWOYXj2/kW3euISV491tnceKcae0uT5JGRaTU/tFrT09P6u3tbXcZ/Ob3f+Dyf7+H2dMmMrGjTLkUfOHik3nvcaN7ByxJGg0RsSql1FNvv3E9ot/faUdM56NnHcW23QMArHh4A7c+9ByHdHUwoVzixDnTKLvcsaScMehHmDShzBcvOWXf6961L3LLA89xywNDNy/5+p+fxiWnz2tXeZLUEIO+hh9c8U6e2bydgcHEJ77fyw/uWctD67e+bp+T5kzjw2cc0aYKJak+g76Go2ZO4aiZUwA4c8FhPLXxFVZvem1d+52vDlIKDHpJY5onY5vw7TvX8E8/f4rpkzvfcE/FY7un8t+fPJuhe6hL0ujzZOxBcPHb59L/ym4G95t3/8TzL3P/s1tY+fhGJnaWa35HRynoWTCDiR2195OkRhn0TZg/YzLX/OnJb9j+00c2cP+zW1j6n6sO6Hu+9Gen8JF3HjXa5UkSYNC3xAWnzOF/PjWFPYN76+yZuPS7v+EH96xj1dotDR/v0j86grOPndnw5yUVm0HfAqVS8Lb5hx7Qvued8hYe6XuJ+555saFjbXx5Fzv3DBr0kjJ5MjbnPvzd33Dfsy/SMcYu5PrUOW/lsx84rt1lSIXW9pOxEXEe8A2gDFybUvpyq441nv3Necdz51Ob2l3G6/x41XOsfHwjCw8/pN2lSGPeMd1TWr62VkuCPiLKwLeAc4E+4P6IWJFS8rZOo+yMBYdxxoLD2l3G6zz7hx387yPPc+V/PdDuUqQx75PvPTafQQ+cCaxJKT0NEBE/BBYDBv048C+Xvp3PLFrY7jKkXJgxeULLj9GqoJ8HrB/xug9458gdImIpsBTgyCOPbFEZaoeuzjLHzbZtI40VrbrxSLUzg68765tSWpZS6kkp9XR3uwywJLVKq4K+Dxi5AMx8YEOLjiVJqqFVQX8/sDAijo6ICcBlwIoWHUuSVENLevQppYGI+BRwO0PTK69PKT3WimNJkmpr2Tz6lNJtwG2t+n5J0oFpVetGkjRGGPSSVHBjYq2biOgH1jb48VnA5lEsZywbL7/V31k84+W3HuzfeVRKqe789DER9M2IiN4DWdSnCMbLb/V3Fs94+a1j9XfaupGkgjPoJangihD0y9pdwEE0Xn6rv7N4xstvHZO/M/c9eklSbUUY0UuSajDoJangDHpJKjiDXpIKzqCXpIL7f47LD0NG3aSvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1304afa2048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['Name'].value_counts().plot() #how common are some names vs others"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "fd_names = FreqDist(df['Name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAE8CAYAAADaGCZFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4VOXZ+PHvnZ1shJAAQZAAAoKISuIC2talVmtttWpd2ioulbb6+mpt37q0/altbbW7r2+rdalarda9Ci7V4r4gJuyrIIsEwr4kkJAQcv/+eM4kkzBJzmyZhLk/1zVXMmfmPueZZGbu86xHVBVjjDGmvZREF8AYY0zPZAnCGGNMSJYgjDHGhGQJwhhjTEiWIIwxxoRkCcIYY0xIliCMMcaEZAnCGGNMSJYgjDHGhJSW6AJEo6ioSEtLSyOKra+vp0+fPhEfO9nje0IZLN7iLT6y+MrKyi2qWtzlE1W1197Kyso0UhUVFRHHWnzPKIPFW7zFRwaoUB/fsdbEZIwxJiRLEMYYY0KyBGGMMSYkSxDGGGNCsgRhjDEmJEsQxhhjQrIEYYwxJqSkTBDPza7iDzN3MGvVtkQXxRhjeqy4JQgR+ZuIbBKRhUHbfisiS0Vkvog8LyIFQY/dJCIrRGSZiJwWr3IBzFu7g/fX7mF+1Y54HsYYY3q1eNYgHgZOb7ftdWC8qk4APgFuAhCRccCFwGFezF9EJDVeBRtelAPAyi2743UIY4zp9eKWIFT1HWBbu22vqWqTd3cmMMT7/Szgn6raoKqrgBXAMfEqW6mXIFZbgjDGmA6JW5YjTjsXKQWmq+r4EI9NA55U1cdE5P+Amar6mPfYg8ArqvpMiLipwFSAkpKSsmnTpoVdrg27mrj6lS3075PCfWcOCDseoK6ujuzs7IhiD4T4nlAGi7d4i48svry8vFJVy7t8op8FmyK9AaXAwhDbfwI8T2uC+jPw7aDHHwTO7Wr/kS7Wt7dpn468cboOu2G61jU0RbSP3rxQVyzie0IZLN7iLT4y9NTF+kRkCnAm8C2voABVwNCgpw0B1serDGmpKQzMdV0cq7daM5MxxoTSrQlCRE4HbgC+pqp1QQ+9CFwoIpkiMhwYBcyKZ1lKct2lMFZZP4QxxoQUtwsGicgTwIlAkYhUAbfgRi1lAq+LCLh+h++p6iIReQpYDDQBV6vqvniVDWBwXipUW4IwxpiOxC1BqOpFITY/2Mnzbwduj1d52rMahDHGdC4pZ1IDlOR5fRCWIIwxJqSkTRCD86wGYYwxnUnaBFGYlUKf9FS27m5kZ/3eRBfHGGN6nKRNECJiM6qNMaYTSZsgAIYXuVmI1sxkjDH7S/IE4WoQliCMMWZ/SZ0gSvtbgjDGmI4kdYIYUez1QdhyG8YYs5+kThDDi3IBWLV5N63LQhljjIEkTxD9stPJz0qjtqGJLbsaE10cY4zpUZI6QYgIw4tdLcKamYwxpq2kThAAw/t7Q103W4IwxphgliAC/RBWgzDGmDaSPkGUFlkNwhhjQkn6BDGiyPogjDEmlKRPEKVBy200N9tQV2OMCUj6BJGXlU5RbiYNTc1sqNmT6OIYY0yPkfQJAmCErclkjDH7sQRBazPTSksQxhjTwhIErUNd7boQxhjTKm4JQkT+JiKbRGRh0LZviMgiEWkWkfJ2z79JRFaIyDIROS1e5QrFrgthjDH7i2cN4mHg9HbbFgLnAO8EbxSRccCFwGFezF9EJDWOZWvDahDGGLO/uCUIVX0H2NZu2xJVXRbi6WcB/1TVBlVdBawAjolX2dob1j8bEfhsWx1N+5q767DGGNOj9ZQ+iIOAtUH3q7xt3SIrPZXBffvQ1KxUba/vrsMaY0yPJvG8DoKIlALTVXV8u+1vAT9S1Qrv/p+BD1X1Me/+g8DLqvpsiH1OBaYClJSUlE2bNi2istXV1ZGdnd1y/7a3tzF/UyM3n9CPspLMsOOjPX5vi+8JZbB4i7f4yOLLy8srVbW8yyeqatxuQCmwMMT2t4DyoPs3ATcF3f83MKmr/ZeVlWmkKioq2tz/yfPzddgN0/WBd1dGFB/t8XtbfE8og8VbvMVHBqhQH9/hPaWJ6UXgQhHJFJHhwChgVncWwDqqjTGmrbR47VhEngBOBIpEpAq4BddpfTdQDLwkInNV9TRVXSQiTwGLgSbgalXdF6+yhWJDXY0xpq24JQhVvaiDh57v4Pm3A7fHqzxdabkuhCUIY4wBes4opoQb0q8PqSnC+p317NnbrZUXY4zpkSxBeNJTUzi4MBtVNx/CGGOSnSWIIMO9VV1X2tXljDHGEkSw0v627LcxxgRYgggyvNglCBvqaowxliDaGG41CGOMaWEJIkigBrFqqyUIY4yxBBGkJD+LzLQUNtc2ULtnb6KLY4wxCWUJIkhKirR0VK/ZakNdjTHJzRJEOy1DXa0fwhiT5CxBtFPqJYhVNhfCGJPkLEG0M8JLEKuto9oYk+QsQbRTak1MxhgDWILYz/CWJqZdgYsXGWNMUrIE0U5Rbga5mWnU7Glie50NdTXGJC9LEO2ISGstwpqZjDFJzBJECJYgjDHGEkRILUNdt+xKcEmMMSZxLEGE0DLUdYvNpjbGJC9LECHYUFdjjLEEEVJg2e/VW3bbUFdjTNKKW4IQkb+JyCYRWRi0rVBEXheR5d7Pft52EZH/FZEVIjJfRCbGq1x+9M1OpzAng/q9+9hY05DIohhjTMLEswbxMHB6u203AjNUdRQww7sP8GVglHebCtwTx3L5YiOZjDHJLm4JQlXfAba123wW8Ij3+yPA2UHb/67OTKBAREriVTY/LEEYY5KdxLONXURKgemqOt67v0NVC4Ie366q/URkOnCHqr7nbZ8B3KCqFSH2ORVXy6CkpKRs2rRpEZWtrq6O7OzsDh9/dskuHl+4i6+NzmbKEflhx0d7/J4e3xPKYPEWb/GRxZeXl1eqanmXT1TVuN2AUmBh0P0d7R7f7v18CTghaPsMoKyr/ZeVlWmkKioqOn38pfnrddgN0/WKhz+OKD7a4/f0+J5QBou3eIuPDFChPr7Du3sU08ZA05H3c5O3vQoYGvS8IcD6bi5bG4Ery9lkOWNMsuruBPEiMMX7fQrwQtD2S7zRTMcBO1W1upvL1kZpkau6fbatjn3NNtTVGJN84jnM9QngQ2CMiFSJyBXAHcCpIrIcONW7D/AysBJYAdwPXBWvcvmVnZHGoPws9u5T1m2vT3RxjDGm26XFa8eqelEHD50S4rkKXB2vskRqeFEOG2r2sGrrbg7uH12HrjHG9DY2k7oTpUEXDzLGmGRjCaITI2wuhDEmiVmC6ETLZLmttqqrMSb5WILohF0XwhiTzCxBdOLgwmxSBNZtr6ehaV+ii2OMMd3KEkQnMtJSGNIvm2aFtdusmckYk1wsQXShddE+SxDGmORiCaILw60fwhiTpCxBdMFqEMaYZGUJogtWgzDGJCtLEF2wCwcZY5KVJYguDC7oQ0ZqChtrGtjd0JTo4hhjTLexBNGF1BRpWahv9VarRRhjkoclCB8CzUyrraPaGJNELEH4YB3VxphkZAnCBxvqaoxJRmEnCBHpJyIT4lGYnspqEMaYZOQrQYjIWyKSLyKFwDzgIRH5Q3yL1nPYUFdjTDLyW4Poq6o1wDnAQ6paBnwxfsXqWQbkZZKdkcr2ur3sqGtMdHGMMaZb+E0QaSJSApwPTI9jeXokEaG0v9UijDHJxW+CuA34N7BCVT8WkRHA8kgPKiLXishCEVkkItd52wpF5HURWe797Bfp/uNheLE31NXmQhhjkoTfBFGtqhNU9SoAVV0JRNQHISLjgSuBY4AjgDNFZBRwIzBDVUcBM7z7PcbwQA1isyUIY0xy8Jsg7va5zY+xwExVrVPVJuBt4OvAWcAj3nMeAc6OcP9xYdenNsYkm7TOHhSRScBkoFhErg96KB9IjfCYC4HbRaQ/UA+cAVQAA1W1GkBVq0VkQIT7jwu7PrUxJtmIqnb8oMgXgBOB7wH3Bj1UC0xT1Yj6IUTkCuBqYBewGJcoLlPVgqDnbFfV/fohRGQqMBWgpKSkbNq0aZEUgbq6OrKzs30/v7ahmUtf3ERWmvDY2QOor68PKz7a4/e0+J5QBou3eIuPLL68vLxSVcu7fKKqdnkDhvl5XiQ34FfAVcAyoMTbVgIs6yq2rKxMI1VRURF2zBG3/VuH3TBdN9bURxQf7fF7UnxPKIPFW7zFRwaoUB/fz502MQXJFJH7gFKCmqVU9WT/OauViAxQ1U0icjBubsUkYDgwBbjD+/lCJPuOp9L+Ocyt28Gqzbt9/+GMMaa38vs99zSuiekBYF8Mjvus1wexF7haVbeLyB3AU17z02fAN2JwnJgaUZTD3LU7WL11N4fYKlbGmAOc3wTRpKr3xOqgqvq5ENu2AqfE6hjxEOioXrllN4f0qC50Y4yJPb/nwdNE5CoRKfEmtBV66zIlldbrQthcCGPMgc9vDWKK9/N/grYpMCK2xenZ2i7al5vYwhhjTJz5ShCqOjzeBekNAk1Mq7fW0aw5CS6NMcbEl68EISKXhNquqn+PbXF6ttzMNAbkZbKptoEtdc2JLo4xxsSV3yamo4N+z8J1Js8GkipBgKtFbKptoHpXU6KLYowxceW3iema4Psi0hd4NC4l6uFGFOUwa9U2qmtjMdrXGGN6rkhH89cBo2JZkN4i0A+x3moQxpgDnN8+iGm4UUvgFukbCzwVr0L1ZIGRTFaDMMYc6Pz2Qfwu6PcmYI2qVsWhPD3ecKtBGGOShK8mJlV9G1gK5AH9gKS9MPPBhdmIwKbd+9i7z0YyGWMOXL4ShIicD8zCrY90PvCRiJwXz4L1VFnpqRxU0IdmhbXb7OJBxpgDl98mpp8AR6vqJgARKQb+AzwTr4L1ZMOLcqjaXs+qLbsZUWwzqo0xBya/o5hSAsnBszWM2APOIQNcUnjwvVU0NFlntTHmwOT3S/5VEfm3iFwqIpcCLwEvx69YPdvlxw+nIDOFDz7dyg+fmkdzc8dX5TPGmN6q0wQhIoeIyPGq+j/AX4EJwBHAh8B93VC+HmloYTY/+Vw/cjPTmD6/mp9PXxy4Op4xxhwwuqpB/Al3/WlU9TlVvV5Vf4CrPfwp3oXryUb0S+e+i8vISE3h4Q9Wc8/bnya6SMYYE1NdJYhSVZ3ffqOqVuAuP5rUJh9SxB8vOBIR+M2ry3iqYm2ii2SMMTHTVYLI6uSxPrEsSG/1lQkl3PrVwwC46bkFzFiyMcElMsaY2OgqQXwsIle23+hdN7oyPkXqfaZMLuWakw9hX7Ny9eOzqVyzLdFFMsaYqHU1D+I64HkR+RatCaEcyAC+Hs+C9TbXnzqazbUN/PPjtVz+cAXPfG8SowbmJbpYxhgTsU5rEKq6UVUnA7cBq73bbao6SVU3xL94vYeI8Muzx/PFsQPZWb+XS/42i/U76hNdLGOMiZjftZjeVNW7vdsb0R5URH4gIotEZKGIPCEiWSIyXEQ+EpHlIvKkiGREe5zulpaawv998yiOLu1H9c49TPnbLHbUJe2yVcaYXq7bZ0OLyEHAfwPlqjoet3z4hcCdwB9VdRSwHbiiu8sWC1npqTxwydGMHpjL8k27uOKRCuobbba1Mab3SdRyGWlAHxFJA7KBauBkWtd2egQ4O0Fli1rf7HQeufwYBvfNonLNdq55YjZNtvKrMaaXkUTMABaRa4HbgXrgNeBaYKaqHuI9PhR4xathtI+dCkwFKCkpKZs2bVpEZairqyM7OzuyF+AzvqqmiZ+8uZVdjcrJpX24qjwfEem248czvieUweIt3uIjiy8vL69U1fIun6iq3XrDXU/iDaAYSAf+BVwMrAh6zlBgQVf7Kisr00hVVFREHBtOfOWabTrmpy/rsBum629eXdLtx49XfE8og8VbvMVHBqhQH9/XiWhi+iKwSlU3q+pe4DlgMlDgNTkBDAHWJ6BsMTfx4H785VsTSU0R/vzmpzz8/qpEF8kYY3xJRIL4DDhORLLFtbecAiwG3gQCFyGaAryQgLLFxcmHDuTOcycAcNv0xUybd0DkPmPMAa7bE4SqfoTrjJ4NLPDKcB9wA3C9iKwA+gMPdnfZ4um8siHc+OVDUYXrn5rL/I0NiS6SMcZ0yu8V5WJKVW8Bbmm3eSVwTAKK022++/kRbKpp4G/vr+LOD3awvGEBY0vyGVeSx6GD8snJTMi/wxhjQrJvpG4kIvz0K2PZtruBf81dz+Mffdbm8dL+2YwtyQ+65XFQQZ+WkU/GGNOdLEF0s5QU4Y8XHMnEgj005gxkSXUtS6prWL6pltVb61i9tY5XFrauYpKflcahJfmM825jS/IZNdCug22MiT9LEAkgIhxWnEFZ2YiWbY1NzXy6eRdLqmu8m0scW3c3MmvVNmatal0hNjVFKMlNZeInc1pqGuMG5zMgr7PV2Y0xJjyWIHqIjLSUlqalAFVlU20Di9sljZWbd1FV00TVvPW8GDQiqig3o03z1LiSvowoziE9NVET5o0xvZkliB5MRBiYn8XA/CxOGjOgZXt94z5efPtjtO/gNoljy65G3l2+hXeXb2l5bkZqCqMG5rYkjkBTlTHGdMUSRC/UJyOVQwrTKSs7uGWbqlK1vZ4l1TVtahyfbatj0foaFq2vabOP4uwUzly/iHMnDuGwwfnWEW6M2Y8liAOEiDC0MJuhhdl86bBBLdtr9+xl2YZaL3HUsri6hmUbathc18xD76/mofdXM3pgLudOHMLZRx3EwHzrxzDGOJYgDnB5WemUlxZSXlrYsm1fs/LMjI9YUp/Hi/PW88nGXfz6laXc+epSThhVzLkTD+JL4wbRJyM1gSU3xiSaJYgklJoiHFKYzgVlh3HzGWN5+5PNPFtZxYylG3nnk82888lmcjPT+PL4QZxbNoRjSgtJSbEmKGOSjSWIJJeRlsKp4wZy6riBbN/dyPQF1TxbWcXctTt4urKKpyurOKigD+dMPIhzJg5heFFOootsjOkmliBMi345GVx83DAuPm4Yn27exXOzq3h+9jrW7ajn7jdWcPcbK5h4cAHnTBzCVycMTnRxjTFxZgnChDSyOJf/Oe1QfnjqGGau2sqzlet4ZWE1sz/bwezPdvDz6YuZMiGXsrJEl9QYEy+WIEynUlKEySOLmDyyiF+cfRj/XrSBZyvX8d6KLdw/u4acwk+49pRRNkzWmAOQTbE1vmVnpPH1o4bw2HeO5Y5zDicF+NN/lnPri4tobu7+S9caY+LLEoSJyIXHHMwPJxWQkZrCIx+u4bon59LY1JzoYhljYsgShInYcUOyePiyo8nJSOXFeeu58u8V1DU2JbpYxpgYsQRhojL5kCKemHochTkZvP3JZr79wEfsqGtMdLGMMTFgCcJEbcKQAp7+3iQG981i9mc7uOCvM9lYsyfRxTLGRMkShImJkcW5PPP9yRwyIJdlG2s5954PWL1ld6KLZYyJgiUIEzODC/rw9HcnccTQAqq213PevR+wcN3ORBfLGBOhbk8QIjJGROYG3WpE5DoRKRSR10VkufezX3eXzUSvX04Gj3/nWD43qogtuxq56L6ZfLRya6KLZYyJQLcnCFVdpqpHquqRQBlQBzwP3AjMUNVRwAzvvumFcjLTeGBKOV85vITahiYu+dssXl+8MdHFMsaEKdFNTKcAn6rqGuAs4BFv+yPA2QkrlYlaZloq/3vRUXzz2INpaGrme49V8kxlVaKLZYwJQ6ITxIXAE97vA1W1GsD7OaDDKNMrpKYIt589nmtOPoR9zcqPnp7HA++uTHSxjDE+iWpilkgQkQxgPXCYqm4UkR2qWhD0+HZV3a8fQkSmAlMBSkpKyqZNmxbR8evq6sjOzo6s8BYf9j6mf7Kbh+bVAnDOoTl8c3wu9fX1vfpvYPEW31vjy8vLK1W1vMsnqmpCbrgmpdeC7i8DSrzfS4BlXe2jrKxMI1VRURFxrMVHto9nK9fqiJte0mE3TNcbn52nH338cbce3+It3uIdoEJ9fE8nsonpIlqblwBeBKZ4v08BXuj2Epm4OmfiEO67uIzMtBSemLWWP3y4g4amfYkuljGmAwlJECKSDZwKPBe0+Q7gVBFZ7j12RyLKZuLrlLEDefSKY8nLSmPmugYuf/hjdjXY+k3G9EQJSRCqWqeq/VV1Z9C2rap6iqqO8n5uS0TZTPwdM7yQJ6dOoiAzhfdXbOWb989k225bv8mYnibRo5hMkho3OJ/bTy5kaGEf5lft5Lx7P2DdjvpEF8sYE8QShEmYQblpPPu9yRw6KI+Vm3dz3j0fsGJTbaKLZYzxWIIwCTUgP4snp06ifFg/qnfu4Rv3fsi8tTsSXSxjDJYgTA/QNzudR684lpPGFLO9bi8X3T+T95ZvSXSxjEl6liBMj9AnI5X7Linn60cdRF3jPi5/+GNeXlCd6GIZk9QsQZgeIz01hd9/4wgunVxK475mrn58No9/9Fmii2VM0rIEYXqUlBThlq+O44enjkYVbn5+AX9+c0Vgtr0xphtZgjA9johwzSmj+MXZ4xGB3/57Gb98aQnNzZYkjOlOliBMj3XxccO4+6KjSE8VHnxvFT96Zh579zUnuljGJA1LEKZHO3PCYB6ccjR90lN5bvY6vv9YJXv22vpNxnQHSxCmx/v86GL+ceWxFGSn858lm7jkwVnU7Nmb6GIZc8CzBGF6hYkH9+Pp705iUH4Ws1Zv44K/zmTWuj2s3VZnHdjGxElaogtgjF+jBubxzPcnccmDs1hSXcOSarjzgzfJy0pj7KB8xg3OZ2xJHmNL8hk9MI+s9NREF9mYXs0ShOlVhvTL5unvTeLB91bx/pK1VO2CrbsbmbV6G7NWty4AnCIwojiXsSUuaYwryWdcST7FeZmISAJfgTG9hyUI0+v0z83kx6cfSmXxbiZOnMjm2gYWV9ewpLrWq1nU8OnmXazY5G7T5gXF5mQwtsTVNoamNDJR1RKGMR2wBGF6NRFhQH4WA/KzOHHMgJbte/bu45ONgYRR6yWQGrbubuS9FVt4b4Vb6+m++W9yzlFDOGfiQQzrn5Ool2FMj2QJwhyQstJTmTCkgAlDClq2qSpV2+tZUl1D5ZrtPP3xatZuq+euGcu5a8Zyyof149yyIZxxeAl9+6QnsPTG9AyWIEzSEBGGFmYztDCbLx02iFMG7KaxbynPza7ilYUbqFiznYo127nlxUWcOm4g500cwudGFZGWaoP9THKyBGGSVqoIJ4wq4oRRRfzi7CZeWbiB52ZX8eHKrbw0v5qX5ldTlJvJ2UcO5pyJQxg3OD/RRTamW1mCMAbIyUzjvLIhnFc2hHU76vnXnHU8O7uKlZt388B7q3jgvVUcOiiP88qG8LUjBzMgLyvRRTYm7ixBGNPOQQV9uPqkQ7jqxJHMq9rJc7OreHHeepZuqOWXLy3hVy8v4XOjijk4s55deZsZW5JnCcMckBKSIESkAHgAGA8ocDmwDHgSKAVWA+er6vZElM8YcH0WRw4t4MihBfzkK2N5c+lmnptdxRtLN/H2J5sBeHTBLACKcjO8OReBeRd9GVGcQ7r1X5heLFE1iLuAV1X1PBHJALKBm4EZqnqHiNwI3AjckKDyGdNGZloqp48fxOnjB7FtdyP/WbKRt+auYEtTFkuqa9iyq5F3l2/h3aBLpWakpjBqYG67xJFPQXZGAl+JMf51e4IQkXzg88ClAKraCDSKyFnAid7THgHewhKE6YEKczI4v3woI2UTZWVlbYbPujkXO1lSXctn2+pYtL6GRetr2sQP7pvFmEF5NOyupd+y2RGVIVWEQal1HDJ2L32zbUiuiQ/p7oXORORI4D5gMXAEUAlcC6xT1YKg521X1X4h4qcCUwFKSkrKpk2bFlE56urqyM7OjijW4ntGGXp6fN3eZj7b2cTqHU2s2rGXNTubWLNzL40xXK08LQWOHpzJF4b14ahBmaSl+J8V3tP/fhYfv/jy8vJKVS3v6nmJSBDlwEzgeFX9SETuAmqAa/wkiGDl5eVaUVERUTkqKyspKyuLKNbie0YZemP8vmZl9dbdLN+4i09WrGD48BERHbt2TxNPfrCM+ZsaCXyE++dk8LUjB3PuxCEcNji/yyVEeuPfz+JjEy8ivhJEIvogqoAqVf3Iu/8Mrr9ho4iUqGq1iJQAmxJQNmPiKjVFGFmcy8jiXIob1lF2xOCI9zUmbTODDxnHv+as59nZVazYtIuH3l/NQ++vZvTAXM6ZOISvH3UQA/NthJWJTLcPsVDVDcBaERnjbToF19z0IjDF2zYFeKG7y2ZMb1PStw/fP3Ekr//g80z7rxO4dHIphTkZfLJxF3e8spRJv57BxQ9+xL/mrKOusSnRxTW9TKJGMV0D/MMbwbQSuAyXrJ4SkSuAz4BvJKhsxvQ6IsLhQ/py+JC+3HzGWN7+xA3JnbFkU8voqpyMVM44vIRzJg7h2OGFiS6y6QUSkiBUdS4Qqv3rlO4uizEHmoy0FE4dN5BTxw1kR10j0+ZX89zsKuZ8toOnK6t4urKKwX2z6Ju+j9yPP4joGIKQrXWcULeScd4w3n45Nnz3QGMzqY05gBVkZ3DxccO4+LhhfLp5F8/PXsfzc9axbkc96wG2RDcX9a01S1p+L+mb1TLfIzD3o7R/DqlhjKwyPYslCGOSxMjiXH502hiuP3U0i6trmLNgMWPGjOk6MISmfc28OXsJu9MLWFJdw7INtVTv3EP1zj28sbR1fEmf9FRGDwpc0c8ljkNL8snNtK+e3sD+S8YkmZQUYfxBfWnYkEFZFH0RmTuzKSs7HIDmZmXNtjqWVNeweH1Ny5X91u/cw7y1O5i3dkeb2KGFfUhr3kuf996N6NgikEMDk7Z94q4QWJLP0MI+dnXAGLMEYYyJWkqKMLwoh+FFOZxxeEnL9h11jW0uBbu4uoblG3exdlu994SaDvboz6z1y1t+z8tM49Cg5q2xJfmMGZhHn4zUqI6RzCxBGGPipiA7g0kj+zNpZP+WbXv3NbNmax1z5i9k7NixEe13X7PyxscL2ZPVv+V65Ft2NfDx6u18vLq1XyVFoLQop6WWEehQH5ifGfVrSwaWIIwx3So9NYVDBuSys1864w/qG/F+mjb1oawWIvoRAAAejElEQVSsNcFsrm1oqaUEaiyfbt7NSu/20vzqluf2y05nSK5wTPXiluRxyIBcMtJs9d1gliCMMQeE4rxMivOK+fzo4pZte/buY8WmXSxu1zeyvW4v2+tgwaZVLc9NT3Wz3McFNVGNLcmjf27y1jYsQRhjDlhZ6amMP6hvm5qKqrJ+5x6mvTubvdkDWLLBJY812+pYuqGWpRtqYc66lucPzM9s068xriSP4UW5iXg53c4ShDEmqYgIBxX04ejBWZSVjWrZvruhiaUbWjvUl1TXsHRDLRtrGthYs5m3lm1ueW5mWgpZqZD+yusRl6NvejPlq+YzbnC+N/w3j/ysnrV0uyUIY4zBXZe8bFg/yoa1LiLd3Kx8tq2uTb/Gkupa1u2op6EJaGiM+HhbgE8r1rbZNqRfnzY1lbEl+Qztl01KgiYbWoIwxpgOpKQIpUU5lLYbvlu7Zy+zKucwYcIREe23WZV/fzCH5rxB3kWmali2sZaq7fVUba/n9cUbW56bm5nGoYPy2vSLjBmUF/Vr88MShDHGhCkvK52CrFSK8yLvwD6sOIOysuEt95v2NbNqy+6WYbuBEVmbaxuoWLOdijWtw3dFYFxROi9Fd0mXLlmCMMaYHiAtNYVRA/MYNTCPs45s3b5lV0Ob5q3F62v4dPMuMlLj3+xkCcIYY3qwotxMPjeqmM+Nah2+29C0j3c/qoz7sW1WiDHG9DKZaan0y4r/EiKWIIwxxoRkCcIYY0xIliCMMcaEZAnCGGNMSJYgjDHGhGQJwhhjTEiWIIwxxoQkqproMkRMRDYDayIML8KtlxWpZI/vCWWweIu3+MgMU9XiLp+lqkl5Ayos3v6GFm/xyRrv52ZNTMYYY0KyBGGMMSakZE4Q91l81BJdBou3eIuPo17dSW2MMSZ+krkGYYwxphOWIIwxxoRkCcIYY0xIliDCICKFIbYND/XcDuLHR3n84/1s6yT+5+3up4rIPyIoR76IhH3V9Ghfv7ePySLyTRG5JHCLdp+m+4jIPBG5WURGJuDYvxORw7r7uEHHzxGRFO/30SLyNRFJ9xmb7/0sDHWLW5mTqZNaRM4B7gQGAOLdVFXzfca/D3xZVWu8++OAp1TV1xefiLwHZAAPA4+r6o4wyz9bVSd2ta2T+IeBZar6axHJBJ4GZqvqrT7jy4GHgDzc324HcLmq+rr2YQxe/6PASGAusM/brKr63z7ji4ErgVKCLrerqpf7jM8CrgAOA7IiiI/4+CJSC3T4YQ3jPdwfuBU43tvfe8DPVXVrJzF3d3FsX39/b1/DgAu8WzPwJO4z9JnP+EzgXPb/G/68o5ig2O8Al3lxDwFPqOpOn8e9vrPHVfUPPvZRCXwO6AfMBCqAOlX9lo/Y6ap6poiswv0vgi9Irao6oqt9RCLZrkn9G+CrqrokwvhfAdNE5CvAGODvQJf/3ABVPUFERgGXAxUiMgt4SFVf7yxORCYBk4Hidm/UfCCc6w5eBvxDRG4CTgJeUdU/hhH/N+AqVX3XK9cJuA/aBD/Bkb7+IOXAOI38rOYF4F3gP7QmmHA8CiwFTgN+jvvfh/Neivj4qpoHLbXADV5ZxCtDOLW5fwLv4L5k8eKfBL7YSUyF9/N4YJz3fIBvAGFdGFlV1+A+h7/x3gs/w520+X0fvwDs9I7bEOaxHwAeEJExuM/CfO+k735VfbOL8MDfeAxwNPCid/+ruL+nH6KqdSJyBXC3qv5GROb4LPuZ3k/fLRYxEe+p2j3pBrwfg32cDXwALABGRbiPVNwHdB3uC2YpcE4nz/8CcAtQ7f0M3K73UwZgYtDtWNwZ+J8D26L5+0XyNw339QfFPQ2URPG/mxvl/36O93O+9zMdeKO7ju/t4yM/2zqJrwyxzdeSDcCbQHrQ/XTgzQheQynwY9yX/Czgh2HELozy75cKnAX8yzv+DcA04J8+418D8oLu5wGv+n3/AJNwtYfDvG0Lwiz/C8BFQHa07yU/t6SoQXhNS+DOWp/EvTlazj5U9bku4ttXsfOBlcA1IoL6b+KYgDtz+QrwOq42M1tEBgMfAiHLoapvA2+LyMPqzsDC9ft297fjzgR/j3tdJ/vczywR+SvwhBd3AfCWiEz0yjm7s+BIX7+ITPOOlwcs9moewf+/r/ks/3QROUNVX/b5/Pb2ej93eP0pG3Bfdn5Fe3yAfSLyLVxNQHFfFuHURt4UkQuBp7z75wEv+YwdjPsfbPPu53rbfBORj3CJ5WngG6q6Mpx44AMROVxVF4QZh4j8AfgaMAP4larO8h66U0SW+dzNwUBj0P1G/L8HrgVuAp5X1UUiMgKXdMPxB9zn7g7vc/AkMF1V94S5H1+Sog9CRB7q5GHVLtqARWRKZ4+r6iM+y/EOcD/wjKrWt3vsYlV9tIO4P6nqdUFflO2P7/cLMioi0tmbWVW100QTxev/Qmf79RJoZ8cNtN8LkINLLnsJvw/qO8CzwOG4fpRc4P+p6r3dcXxvX6XAXbT2IbwPXKeqq33G13plaPbiU4Hd3sOdlkVELsP1XwTeB18AbvX7/vf2caiqLvX7/BDxi4FDgFW4v2Pgb9hlM6eIXI6rKdSFeKyv+uiPEJGfAOcDz+P+fl/H9aH8KqwXEiURScWd2F0JnB7Oeyis4yRDgoglEckARnt3l6nq3s6eH6NjlqlqZUdflF19QQbtZyCuH2Wwqn7Z62SfpKoPxrC4cSMiOUC9qjaLyGjgUFw/Stz/B97xM1W1od22QlXd1lHMgUZEBuGaKRWYpaobwoyP6j3odXLvx2/NWkT6AaNoO8jAbx9CYB9lwAne3XdU1Vc/gjdI4cfsP8jBbw0+sJ8+uL6PC3DNxNNV9Zpw9uFbd7Rj9ZQbrnMsH1fFnYFbS/3bYcSfiLv+xNu4jqlVwOfDiB8FPAMsxjVRrQRWduPrfwV39jPPu59GGG2gwEDgQdyXMrhmqiu66/Xj2oyzgYOAtbizuH+EEX88kOP9/m1cdf3gMOJfom0b/CBCtOnH6/he3GjvvbvQuz8B+GkY8eId+2fe/aHAMWHEfw34nXf7ane9B4F872dhqJvPY38H13e4HVcLqieMPqSg/aTimtYODtx8xr2GGwW3BFf7+htwZ5jHfhJYDdyLq0GkhFv+sI4Xz533tBteJyGuWviI9+aaF0Z8JTAm6P7oML8g3gNOAeYDw3DV9dvCiD8T19G1DagBaoGaMOI/9n7Oaf838RkfbYKJ9vXP9n5eA/w4gvLP974gj/B+vxZ4O4z4K3H9V6m4duf5wJe66/jePt4Gjmn3P/TdcQvcgxugsMS73y/wvvARewcuOV3u3V4Hfh1m+SN6D+LOksGdlK30fgZuvk4ycMkhK+h74FDgyTDLfw3uxHKR9z9cgDdowUdsZeB9EPz/DPP4pwOp4cREc0uKTuoggUkpZ+DGQG8Tkc6ev1+8qrZ0ZqnqJ34nunj6qOoMERF1VeJbReRd3IgkP/4EnIP7Uo6kbXC3Nw7enUqKHIcbMuhXkao+5Q2TRVWbRCScDtJoX794Q36/hTsTg/CG+TapqorIWcBdqvpgV/1LwVT1fq+J8V+4BPFdVf2gu47vyVbVWe3et01hxB+rqhMDwytVdbv3mvw4AzhSVZsBROQR3AnLTWEcP6L3oMZmmOceVd0jIoHmwqXekNdwXIs7Sexw3kgnAk2h1d5Q+fXAkHB2oKqvish4r2kuuJnq7xGUp0vJliCmichSXNXyKq9NMJze/woReRA3Bh3cF1U448D3iJtJuVxE/gs3zHNAGPFrcWeLkXYcXY8bvz3SG/9djBvF4le0CSba138d0Y0CqfWS28XA57yOvi4TfLu5J4JrlpkLHCcix6mPSVLRHL+dLeJmIQf+B+fhhj/7tdc7biC+GNdh7VcBraOY+oYRFxDVe1BEZqjqKV1t60CViBTgEvzrIrId9yUdjrWE954P9ksR6Qv8ELgb19z9g3B2ICK34Jq6xwEvA1/G1czjkiCSrpPa66SqUdV9XqdnnvrsaBM3i/NqXAeV4Poh/qLtOi47iT8a1/5YAPwC9wH7jarODCP+F7hmhuBhnn6/oBCRNNxkHyHMTnZvOOvdwHhgId6HW1Xnh1H+iF9/0H7ycCNXdoUZNwj4Jq6Z410RORg4sauzL+9D2SFVvS2ex2+3jxG46wBMxrWlr8L1o632Gf8tXOdmGW4k1nm4PoynfcRehGtmehP3/vk8cJOq/tNv+b39hP0eFDeLPds79om0ziTOx/WJjQ2zDF/Avf9eVdVGH88PnCQc5pX9JcL8DIYa0CAiw1V1VRjlXoBropyjqkd4nf4PqOpX/e4jHEmVIEQkG3cGc7CqThU3k3OMqk5PcNF8EZHXgF24ds+Wsz6/X1DePiaz/zIF4XxBRZxgoiUih+POlAq9428GLlHVRWHsYxhucuF/vPdDqqrWxqXAcTy+d3KTEmHsobi+IAFmaBgrC4hICW4mseAm6IU7iinwGRymqlf6/QyKyLW4GuRg2p711+BmQv9fJ7H5qlojodcsUrwTxi6OH/VJguy/VM9Y4Gn1uVSPF/Oxqh4tbtmOk3D9kAtVNS5rTCVbE9NDuCahyd79KtyEHV8JQtzCeLfiOliDv2A7XQdFOp/HoLgq+199nEkXquqX/JS1g3KEXMsIn9XTUB9uEfHz4Y7V6/8rcL16yyKIyIm4eRWTOwsKKseVwFRcghmJGw11L+7L0k/8aOBH7J9gfQ1TjPb43j4KgEsCZQj0RWgY6yEBRbg1gB4SkeIwz2JTcJ20acBoERmt4Q0TDXwGJ3n3fX0GVfUu4C4RuUZV7w7jeACP4wZ4VNI6HyXwPhQgV0TuV9WbOzl+mwQQYS02qqV6xP2z53vvgfu917MLNxs9LpItQYxU1Qu8qjKqWi8SVi/1g7g2w0rCm70a6LP4XQePF+GGvI3rYj//EZEvqeprYRw7WLRrGUX04SZ2rz9Hg9bMUdW3vDNpv67GjQD6yItfLiLh9IE8jftCf4DI1nKK9vjg2p1n0q4W6Zd3JlyO+4J6CNcH8hhuCG5XsXfimqcWBR1b8b8WEUT4GRSRk1X1DWCdtK6M0EI7WQ0huIPbq0UEz4MITDZcCHSYIILKMR73fi707m/BZy1WVV8SN6jlNdyM9LNVdXlXcUHxKiJHqlvk8l4ReRU3/NdXE28kki1BNIqbZBLooBtJeAt+7VTVV8I9qHqrnWonE9pEpMt2UNwXzI9FJNKZuAtxY/fD6dQMFtGHO4avf6WI/IzWhPNtXBu8Xw2q2hgostdcFk6ybFLVe8J4fqyPD5Clqp2uLNqFrwNHAbMBVHW9+F+6/Wxcc1BYi+S1E+ln8AvAG7gJYtC2BqB0sExLMHEz4a/FjRyaCxwHfOB1cPvtw7iPMGuxEqOlejwzReRoVf3Yb79TNJItQdwCvAoMFXcdhOOBS7sK8jpnwa1j81vcmzG4g6qrNYieUtXzvQ6m4DdKyzIBqjqtq3Koal6IM6AuSezWMooqwYZoogu8/hF+Xj9u7P1ttH4ZvIOP/1+Qt0XkZqCPiJwKXIVbqM2vaSJyFW6CXvDfz+9M6miPD/Co11Q1PcIyNHpnooH/YTg1sJW4Gkc0CSKiz6CqBvoAvs/+y337TbLX4vpPZqrqSV5fjO/+O08ktdiKdvfDWgG3nZOA74rIGtwSKb6XGolEUnVSA4gbpnkc7g87U1W3+IjpbCildtUGLSIlqlrtjYSYhRsqF7wDv8sEdHYG1FncFzp7vLMz+3b7ORX4Ka4p6DW8D7eqvuUzfikhmujU55hyEflG+9E2obZ1Ep+Cmz/xJdz//9+4ESC+PgTi1uJvT7vqg+rs+Kp6v5/YoH1cDdyOuxZHoNzhlOFHuBOMU4Ff45Lu437a9UXkWdwImhm0TU7hnAFH9BkMin0V99pn0/aaIH5GEQU6eOfi5oM0iMhcVT0yjOM/7x07uBZbrqpn+91HNCTKpUbCPl4yJAjxFggLqgkEU2BbvP7A7cpxC24m8jbcapzPqOrGMOIX0HoGdGTgDEhVL/AZH/VaRlF+uD9S1WP9Pj9EfMQXTBI39v8RVf12pMePlnhrarXb9lWftafA8z/Ffbn5/ruH2MeptE1Svq7HIR1M6lMfi/XF6jMoIgs1jFE/7WKfx60mfB1umYrtuMmvZ4Sxj364Wsfx0DLU/Vbt5OJXnbQgABCvs/9YSJYEcZ+6Ya0d1QT645aPuLiL/VyL69irxbU7TgRuDLfTWNyy1xfgqspVqtrZxVqC46I6A5IIr2jVwYe6RVdNbEH7uQM38zncJrov42bxnk/rxWrAteWOU9VjfB7/37j1g/z0d3S0j/G4GlTYs1hFZDYwRb2lqr2+nOvCSZoi8iJwoYZYkbQni+Fn8D7cxXbCXu673X7CmgcRFFcO/IR2TVydfcnHqgUhEZKiD0JVp3o/T+roOeLmGHTlclW9S0ROw80AvgyXMMIdVbQJdy2BrYQ3kzjamaChrmg110dc4HoSWbgRMPNwZ08TcCNyTuggrr3AF2F50Dal6+tRrMcls6/Rtv22lvBmoq4G3ve+ZANLXPueaCjRz2I9D3hG3GS1E3DDVcMdtrwPmOt90fpu5pEoLlkaizPgGH4GTwAu9Zr7wlruu115fDWrhvAP3FDnhfgcRaaqgUEhebih2hG1ICRCUtQggkV5BjhfVSeIyF3AW6r6vIjMUdWjfMZ/H1dzKMatavqkqi4O+0UQ2RmQuPV3rgL+iFuFdZGILFDVw33G/xO4PegMeDzwI1W9NJLXEC4RSQ+nOSxEfMjJTup/JnTUs1i9pr1/4c4iz9Z218XwER9xM48XH/KSpar6m05iYnoGHOVnsFvb4EMc/z1V9XtC1NE+ImpBSISkqEEExOAMsNI7yxkO3CRueGA4Y9GH4ZoU/Jy1dyrCM6Bo1zI6NLhqr6oLRcR3Bx+AuElC7dfD7/KC855jRORWQoyC8hPsNxF0ItB/0yQi+biaYJfHDnHmXYhravtI3DBH32e/fhNBJ05r16R1j7irvHWYIGJ5BhztZ7AHNMfcIiIPsH9HfZfDbINE2oLQ7ZKqBhHtGaA3CuVI3FC/TNwEr4P8jAA5EIjIE7immcdwX3jfBnJV9SKf8ffi1tM5CTfZ7DzcRWeu6DSwNT7aUVBvErqJxO9M6L/gJlNdiFtwbRdu6ejLuogLedYbdHzfX3oiciZuHav2SdLvVfE+wC33HXzJ0qtV1ddsdG8fEZ8Bx6IWlkgi8hhucEebyYLaxVUpvdiYtSB0l6SqQeCW+w37DDDI5ew/zPRD3AJ2PZbE7pKll+HGoV/r3X8Hd30BvyZ7TXTzVfU2Efk9PiY4BYloomKQHwX9noX7gvO9VLaqXuX9GtYs1kACELc4X7SiXfL9m7hLlt5F6yzib4a5j2jOgKP9DCbaEX6bZEOIWQtCd0maBCESk3VMYjHRJhG6WurCF3UXRv+jd4tEYGn1OhEZjGuqCGd9/4gmKgY9r/0EpfdFxHdTnYg8gxuU8IpGNov1JVrXAsrCvfZluCY3v6Ja8t0r91mRxIY4A74ynDPgGH0GE22miIyL5MxfVW+MR4HiKWkShGpM1jGJxQVHup36WOrCD4lwscIg07wvh9/iJhsp7ovCr0hHQQEgbVfzTPH2MyiM49+Lq0X9r4g8DTysqkv9Brc/8/SGD383jOODu6bxy15iC3vJd3HLZl/B/v1AXTaREOUZcIw+g4l2AjAl2lFUvUXSJAhPtOuYxOKCIwkT4gs+rE5eIl+sMGApsE9VnxV3RayJuL+lL50NkfQpeDXPvbhhr776P7zj/we3YGJfXNv96yKyFpfkHgt3hJWqzhZ3jYxw3I47684C/F4JLtijuP/DacDPcaOYfC33HaMz4G5dSygOTk90AbpTsnVSL8ZdRzrqdUwinWiTSDHo5I12JnRgmPAJuKWPfw/cHM4+oxkFJSLn4/5fNeIW/ZsI/MJvE5W3j/64zvmLcScH/8CdVR6uqid2ERu8yF4K7qI9hap6WhjHr1DV8q6f2WH8HFU9Kuh/kY6bTe2rFhatWH4GTfwlWw3iy7HaUbTNNQkSbSdvVH0AtCalrwD3quoL3rBVXzoaBeU3HnfltKe8BHUqLkHdQ2vTVVfHfw43guVR3IzswPDPJ0Wk/YJsoeTROkigCbdQ37NhlB+iX/I9UMvZ4c1H2ICbFdxdYvYZNPGXVDWIZCWtS2WcTwRLXQTtJ9ScCQ1jmOh03HWov4g7e67HDXM9wmd84Kw38DMXeE59XkQp6Oz517hRQI9LeBMdA9ckiIjXnHQzYSzTEGIftUAO7v8X9pLv4hZ8fBY4HHfJ0VzgZ6r6V79lMMnDEkQS6OCLPcD3F3wMypGNa8NdoO5iOSW4phlfZ8MiMktVjxGRmbihntu8fY3yGR9RggokBglxoRrwP0lKRJYRYpmGcOZBRMObx3Oeqj7VHcczvV+yNTElpRh07raIpg9A3QJzzwXdrya8ixdFOwrqfFyC+p2q7vAS1P/4iGt/sZpgvi5W49msYazc2hFxK4q2uSaI+rjspzf/4L8ASxDGF6tBJBGvg/UWXKeq4pY4+HkYndRRzYSOloh8A9fJXBtpJ3MUx4767FtETsGNfop4mQYJfU2QD8No5vsZrub0JG0XLPR7wSGTRCxBJBEReR03+/kxb9O3gBPV/3LjUfUBRCsWo6CiPP47qvr5KOIjXqYhaB/RXhNkFaFn0/em2cymm1gTU3IpVNVfBN3/pYiEcyWsaGdCRyuqUVAx8Lq4K7JFevYdzTINAdFO1hyHW9E3UIt8FzcB0Jj9WIJILm+KyIW0tkGfh1v+wa9o+wCitU5E/orrZL5TRDJx8wm6y+W413xVu+1+z74jXqYhSLSTNR8BaoD/9e5f5G07P4oymQOUNTElAWm9WIzghkgGzsRTgV1hDJFMWB+Ad/yoRkHF4Ph9CHH2rT6v6SAiS4CRQEyWaYhksqaIzGs/aivUNmPAEkTS8dYjaj8Cxtekv0T3ASSaiDyFO/v+h7fpIqBAVX2dfUuMLnYj7vraA2m7HtZnPmMfxiW1md79Y3GXQW1fKzLGEkQy6WAEzAeqeorP+KgmmvV2PeHsW0SuwY1E20jbjm5ftRCvFjMGCCSUg3FrMTWHsx+THKwPIrlEu1x5ovsAEm2OiBzX7uz7/W4uw7XAGL9Dk0NIqsXmTHQsQSSXaEfARDrR7EBxLHCJiLQ5+/aGnnbX2fdaYGekwd01a9scGCxBJJeoRsDEYCZ0b5ews++glWBXAm+JyEtEcD0IY8JhfRBJqjcuV57MROSWzh5X1d5wZUPTy1iCMKYXEpE8XLPWrkSXxRy4kqmD0ZheT0TGi8gc3Iqwi0SkUkTCuaa1Mb5ZgjCmd7kPuF5Vh6nqMOCHdO9sdpNELEEY07vkqGrL9T1U9S3c7HhjYs5GMRnTu6z0ljl51Lv/bdzSHcbEnNUgjOldLgeKcZcNfQ4oAi5NZIHMgcsShDG9y0hgKO6zmw6cgrvGhzExZ8NcjelFEn1da5NcrA/CmN4lJte1NsYPq0EY04vE4rrWxvhlNQhjepfLcNe1TidouW+C1sgyJlYsQRjTu8TiutbG+GKjmIzpXWaKyLhEF8IkB+uDMKYXifV1rY3pjCUIY3qRWF3X2hg/LEEYY4wJyfogjDHGhGQJwhhjTEiWIIzxiMhPRGSRiMwXkbkicmwcj/WWiJTHa//GxILNgzAGEJFJwJnARFVtEJEiICPBxTImoawGYYxTAmxR1QYAVd2iqutF5P+JyMcislBE7hMRgZYawB9F5B0RWSIiR4vIcyKyXER+6T2nVESWisgjXq3kGRHJbn9gEfmSiHwoIrNF5GkRyfW23yEii73Y33Xj38IYwBKEMQGvAUNF5BMR+YuIfMHb/n+qerSqjgf64GoZAY2q+nngXuAF4GpgPHCpiPT3njMGuM+bp1ADXBV8UK+m8lPgi6o6EagArheRQuDrwGFe7C/j8JqN6ZQlCGMAVd0FlAFTgc3AkyJyKXCSiHwkIguAk4HDgsJe9H4uABaparVXA1mJu2YDwFpVfd/7/THghHaHPg4YB7wvInOBKcAwXDLZAzwgIucAdTF7scb4ZH0QxnhUdR/wFvCWlxC+C0wAylV1rYjcCmQFhQRWU20O+j1wP/DZaj/RqP19AV5X1Yval0dEjsFdEOhC4L9wCcqYbmM1CGMAERkjIqOCNh0JLPN+3+L1C5wXwa4P9jrAwS3T/V67x2cCx4vIIV45skVktHe8vqr6MnCdVx5jupXVIIxxcoG7RaQAaAJW4JqbduCakFYDH0ew3yXAFBH5K7AcuCf4QVXd7DVlPSEimd7mnwK1wAsikoWrZfwggmMbExVbasOYOBGRUmC618FtTK9jTUzGGGNCshqEMcaYkKwGYYwxJiRLEMYYY0KyBGGMMSYkSxDGGGNCsgRhjDEmJEsQxhhjQvr/sLaNZqGSBM8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1304e08f898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fd_names.plot(20) #20 most common names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking at surnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "singh          1393\n",
       "jain            412\n",
       "das             270\n",
       "patel           232\n",
       "shah            207\n",
       "kumar           183\n",
       "verma           183\n",
       "mishra          150\n",
       "ghosh           148\n",
       "gupta           137\n",
       "arora           128\n",
       "sharma          126\n",
       "srivastava      125\n",
       "roy             124\n",
       "shaikh          108\n",
       "saha            105\n",
       "tiwari          102\n",
       "pandey           99\n",
       "nair             99\n",
       "rao              93\n",
       "yadav            90\n",
       "saxena           84\n",
       "mukherjee        83\n",
       "kumari           83\n",
       "sinha            80\n",
       "jaiswal          78\n",
       "agarwal          78\n",
       "prakash          77\n",
       "agrawal          77\n",
       "khan             74\n",
       "               ... \n",
       "de.               1\n",
       "chawbey           1\n",
       "keshwani          1\n",
       "dhaliwal          1\n",
       "baghel            1\n",
       "singania          1\n",
       "balani            1\n",
       "rafi              1\n",
       "dasari            1\n",
       "sultania          1\n",
       "maji              1\n",
       "bhoite            1\n",
       "shhu              1\n",
       "jamadar           1\n",
       "sain              1\n",
       "ranju             1\n",
       "azhar             1\n",
       "choudhaury        1\n",
       "ramiah            1\n",
       "christian         1\n",
       "bit               1\n",
       "nayek             1\n",
       "nandhakumar       1\n",
       "shahu             1\n",
       "karole            1\n",
       "bhardwat          1\n",
       "kapila            1\n",
       "naveen            1\n",
       "parashar          1\n",
       "avlani            1\n",
       "Name: Surname, Length: 2736, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Surname'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1304b2a4358>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAADuCAYAAAAwTtAhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAE6FJREFUeJzt3X2MHdV5x/Hv4117jTHYxl7AsU3tBPclVG1DV0AbtYpKQyBNYlQVCfqClSJZVelrWiWklWqpVaVUfaGN1CK5sRvSRqQpSYQVuU0tkiptVQhL0kCANKxIgxcbvGAwrh3brPfpHztOLvbO3PW9++I99/uRVjtz5tw7Z/7Z385zzp0bmYkkqfcsmu8BSJLmhwEgST3KAJCkHmUASFKPMgAkqUcZAJLUowwASepRBoAk9SgDQJJ6VP98D6DJmjVrcuPGjfM9DElaUB599NEXM3OwXb/zOgA2btzI8PDwfA9DkhaUiPjWdPpZApKkHmUASFKPMgAkqUcZAJLUo9oGQETsioiDEfG1KY79bkRkRKyp9iMiPhwRIxHxWERc3dJ3a0Q8Xf1sndnLkCSdq+ncAXwUuPHMxojYALwdeLal+SZgc/WzDbin6nsJsB24FrgG2B4Rq7oZuCSpO20DIDO/CBya4tDdwPuB1q8U2wJ8LCc9BKyMiLXAO4C9mXkoM18G9jJFqMyUzGRiIvHbziSpXkdzABHxHuC5zPzqGYfWAfta9kertrr2qd57W0QMR8Tw2NhYJ8Pj0NGTvPH39vD3D01rKawk9aRzDoCIWAb8PvAHUx2eoi0b2s9uzNyRmUOZOTQ42PaDbJKkDnVyB/AmYBPw1Yj4X2A98OWIuJzJ/+w3tPRdD+xvaJ9VVoAkqd45B0BmPp6Zl2bmxszcyOQf96sz83lgN3B7tRroOuBwZh4APgfcEBGrqsnfG6q2WREx1Q2HJKnVdJaB3gf8F/B9ETEaEXc0dN8DPAOMAH8L/CpAZh4C/gh4pPr5w6pNkjRP2j4MLjNva3N8Y8t2AnfW9NsF7DrH8XXFVUCSVK/ITwJbAJKk9ooMAElSe0UHgAUgSapXZAC4CEiS2isyACRJ7RUdAC4CkqR6RQZAuA5IktoqMgAkSe0VHQBWgCSpXpkBYAVIktoqMwAqPgpCkuoVGQB+DkCS2isyACRJ7RkAktSjigwAK0CS1F6RASBJaq/oAHARkCTVKzIA/E5gSWqvyACQJLU3nS+F3xURByPiay1tfxoRX4+IxyLiMxGxsuXYByNiJCL+JyLe0dJ+Y9U2EhF3zfylnC19GIQk1ZrOHcBHgRvPaNsL/GBm/hDwDeCDABHxZuBW4KrqNX8TEX0R0Qf8NXAT8GbgtqrvrLAAJEnttQ2AzPwicOiMtn/NzPFq9yFgfbW9BfhEZp7IzG8CI8A11c9IZj6TmSeBT1R9JUnzZCbmAH4Z+Odqex2wr+XYaNVW136WiNgWEcMRMTw2NtbVwFwFJEn1ugqAiPh9YBz4+OmmKbplQ/vZjZk7MnMoM4cGBwc7HFdHL5OkntLf6QsjYivwLuD6/O5jN0eBDS3d1gP7q+269lnjDYAk1evoDiAibgQ+ALwnM4+1HNoN3BoRAxGxCdgMfAl4BNgcEZsiYgmTE8W7uxt6w/icBpakttreAUTEfcDbgDURMQpsZ3LVzwCwt/rQ1UOZ+SuZ+UREfBJ4ksnS0J2Zeap6n18DPgf0Absy84lZuB5J0jS1DYDMvG2K5p0N/f8Y+OMp2vcAe85pdF1yEliS6hX5SWAngSWpvSIDQJLUXtEB4KMgJKle0QEgSapnAEhSjyo6AFwFJEn1igwAVwFJUntFBoAkqT0DQJJ6VJEB4LOAJKm9IgPgtHQWWJJqFRkATgJLUntFBoAkqb2iA8AKkCTVKzIArABJUntFBoAkqb2iA8AKkCTVKzIAwmVAktRWkQEgSWqvbQBExK6IOBgRX2tpuyQi9kbE09XvVVV7RMSHI2IkIh6LiKtbXrO16v90RGydnct5PVcBSVK96dwBfBS48Yy2u4AHM3Mz8GC1D3ATsLn62QbcA5OBAWwHrgWuAbafDo3ZYAFIktprGwCZ+UXg0BnNW4B7q+17gZtb2j+Wkx4CVkbEWuAdwN7MPJSZLwN7OTtUJElzqNM5gMsy8wBA9fvSqn0dsK+l32jVVtc+q/xOYEmqN9OTwFNVX7Kh/ew3iNgWEcMRMTw2NtbZIKwBSVJbnQbAC1Vph+r3wap9FNjQ0m89sL+h/SyZuSMzhzJzaHBwsMPhSZLa6TQAdgOnV/JsBR5oab+9Wg10HXC4KhF9DrghIlZVk783VG2zylVAklSvv12HiLgPeBuwJiJGmVzN8yHgkxFxB/AscEvVfQ/wTmAEOAa8FyAzD0XEHwGPVP3+MDPPnFieMX4QTJLaaxsAmXlbzaHrp+ibwJ0177ML2HVOo+uSNwCSVM9PAktSjzIAJKlHlR0AzgJLUq1iA8B5YElqVmwASJKaFR0AFoAkqV6xAWAFSJKaFRsAkqRmRQeAi4AkqV6xAeDjICSpWbEBIElqVnQA+IUwklSv2ACwACRJzYoNAHASWJKaFBsAzgFLUrNiA0CS1KzoALACJEn1ig2AcBpYkhoVGwCSpGZFB4CrgCSpXlcBEBG/HRFPRMTXIuK+iFgaEZsi4uGIeDoi/jEillR9B6r9ker4xpm4gPrBzeq7S9KC13EARMQ64DeAocz8QaAPuBX4E+DuzNwMvAzcUb3kDuDlzLwSuLvqJ0maJ92WgPqBCyKiH1gGHAB+Cri/On4vcHO1vaXapzp+fczyE9t8FIQk1es4ADLzOeDPgGeZ/MN/GHgUeCUzx6tuo8C6ansdsK967XjVf/WZ7xsR2yJiOCKGx8bGOh2eFSBJaqObEtAqJv+r3wS8AbgQuGmKrqf/DZ/qb/JZ/6Jn5o7MHMrMocHBwU6HJ0lqo5sS0E8D38zMscx8Dfg08OPAyqokBLAe2F9tjwIbAKrjK4BDXZy/PStAklSrmwB4FrguIpZVtfzrgSeBLwA/V/XZCjxQbe+u9qmOfz5z9hZq+iwgSWrWzRzAw0xO5n4ZeLx6rx3AB4D3RcQIkzX+ndVLdgKrq/b3AXd1Me7pjXG2TyBJC1h/+y71MnM7sP2M5meAa6boexy4pZvznQsfBSFJzYr+JLAkqV7RATCLUwyStOAVGwBOAktSs2IDQJLUrOgAsAIkSfWKDQArQJLUrNgAkCQ1KzoArABJUr1iA2CWnzQtSQtesQEgSWpWdAC4CkiS6hUbABaAJKlZsQEAfiWkJDUpOgAkSfXKDQBrQJLUqNwAwElgSWpSbAB4AyBJzYoNAElSMwNAknpUVwEQESsj4v6I+HpEPBURPxYRl0TE3oh4uvq9quobEfHhiBiJiMci4uqZuYTasc3m20vSgtftHcBfAf+Smd8P/DDwFHAX8GBmbgYerPYBbgI2Vz/bgHu6PLckqQsdB0BEXAz8JLATIDNPZuYrwBbg3qrbvcDN1fYW4GM56SFgZUSs7Xjk0+B3AktSvW7uAN4IjAF/FxFfiYiPRMSFwGWZeQCg+n1p1X8dsK/l9aNV2+tExLaIGI6I4bGxsY4HZwVIkpp1EwD9wNXAPZn5FuAo3y33TGWqP8ln/YuemTsycygzhwYHB7sYniSpSTcBMAqMZubD1f79TAbCC6dLO9Xvgy39N7S8fj2wv4vzt2UBSJLqdRwAmfk8sC8ivq9quh54EtgNbK3atgIPVNu7gdur1UDXAYdPl4pmgxUgSWrW3+Xrfx34eEQsAZ4B3stkqHwyIu4AngVuqfruAd4JjADHqr6SpHnSVQBk5n8DQ1Mcun6Kvgnc2c35zpWLgCSpXrGfBPaDYJLUrNgAAL8QRpKaFBsA/v8vSc2KDQBJUrOiA8BJYEmqV2wAOAcsSc2KDQBJUrOiA8AKkCTVKzgArAFJUpOCA0CS1KToAHAVkCTVKzYAXAUkSc2KDQBJUrPCA8AakCTVKTYArABJUrNiAwCcBJakJsUGgJPAktSs2ACQJDUrOgAsAUlSvWIDIJwGlqRGXQdARPRFxFci4rPV/qaIeDgino6If4yIJVX7QLU/Uh3f2O25JUmdm4k7gN8EnmrZ/xPg7szcDLwM3FG13wG8nJlXAndX/WaV3wksSfW6CoCIWA/8DPCRaj+AnwLur7rcC9xcbW+p9qmOX1/1nxWuApKkZt3eAfwl8H5gotpfDbySmePV/iiwrtpeB+wDqI4frvq/TkRsi4jhiBgeGxvrcniSpDodB0BEvAs4mJmPtjZP0TWncey7DZk7MnMoM4cGBwc7HV71Xl29XJKK1t/Fa98KvCci3gksBS5m8o5gZUT0V//lrwf2V/1HgQ3AaET0AyuAQ12cv5EVIElq1vEdQGZ+MDPXZ+ZG4Fbg85n5C8AXgJ+rum0FHqi2d1f7VMc/n+n/6JI0X2bjcwAfAN4XESNM1vh3Vu07gdVV+/uAu2bh3K9jukhSvW5KQN+Rmf8G/Fu1/QxwzRR9jgO3zMT5pmMWFxhJUhGK/SQwOAksSU2KDgBJUj0DQJJ6VNEB4KMgJKlesQHgHLAkNSs2ACRJzcoOACtAklSr2ACwBCRJzYoNAElSs6IDwAqQJNUrNgD8TmBJalZsAEiSmhUbAIsCTk1YBJKkOsUGwJL+RZwcn2jfUZJ6VNkBcMoAkKQ65QZAn3cAktSk3ACwBCRJjQoOgD5LQJLUoNwAsAQkSY06DoCI2BARX4iIpyLiiYj4zar9kojYGxFPV79XVe0RER+OiJGIeCwirp6pi5jKgJPAktSomzuAceB3MvMHgOuAOyPizcBdwIOZuRl4sNoHuAnYXP1sA+7p4txtOQcgSc06DoDMPJCZX662jwBPAeuALcC9Vbd7gZur7S3Ax3LSQ8DKiFjb8cjbsAQkSc1mZA4gIjYCbwEeBi7LzAMwGRLApVW3dcC+lpeNVm1nvte2iBiOiOGxsbGOx7S4PywBSVKDrgMgIpYDnwJ+KzNfbeo6RdtZz2rIzB2ZOZSZQ4ODgx2Pa0lfH68cO9nx6yWpdF0FQEQsZvKP/8cz89NV8wunSzvV74NV+yiwoeXl64H93Zy/yamJCSYSjhx/bbZOIUkLWjergALYCTyVmX/Rcmg3sLXa3go80NJ+e7Ua6Drg8OlS0WzYuOZCAF45ZgBI0lT6u3jtW4FfAh6PiP+u2n4P+BDwyYi4A3gWuKU6tgd4JzACHAPe28W527r0oqUAHDt5ajZPI0kLVscBkJn/wdR1fYDrp+ifwJ2dnu9cLVvSB8DRk+NzdUpJWlCK/STw6QD4tncAkjSlYgPgwoHJm5sjx70DkKSpFBsAl140AMDYkePzPBJJOj8VGwCrlw/Qvyg4cNgAkKSpFBsAfYuCy1cs5fHnDs/3UCTpvFRsAABcu2k1T+5v+nCyJPWuogNg7YqlvHzsJBMTZz1xQpJ6XtEBsGb5EiYSxv7vxHwPRZLOO0UHwI9+zyUA/OfIi/M8Ekk6/xQdAFe94WIGLxrg818/2L6zJPWYogNg0aLgbd87yJ7HD/Ca3w0gSa9TdAAAvP3NlzGR8Of/+o35HooknVd6IgB+YvMa/uGhb7Hv0LH5Ho4knTeKD4CIYPu7r+K1UxP87j99laMnfDaQJEEPBADAlZcuZ/u7r+Lhbx7iF3c+7OcCJIkeCQCAn7/2Cn726nV85dlXeP+nHpvv4UjSvOuZAAD481t+mOveeAn3PzrKF78xNt/DkaR51VMBEBHc8ws/ytoVS7l915f4pZ0P8+kvj7pEVFJPislvajw/DQ0N5fDw8Iy/76GjJ/nrL4zwma88x6GjJ9m4ehk3XHU5Vw4u58evXM36Vctm/JySNFci4tHMHGrbb64DICJuBP4K6AM+kpkfqus7WwFw2qmJ5L4vPctH/v0Z/vel7y4RXbtiKW8aXM7my5azdsVSNq6+kA2XLOMNKy/ggsV9LOnvqRsnSQvMeRkAEdEHfAN4OzAKPALclplPTtV/tgOg1fHXTvHUgVf57GMHeOHV4zz+3GFGX/42p85YMRQBg8sH6FsUrF2xlCX9i3jDigtYNtDH2hUXEAEXDfSzevkAAVxe9QFYu+ICli4+OzwG+vvoWxRzcZmSesB0A6B/LgbT4hpgJDOfAYiITwBbgCkDYC4tXdzHW65YxVuuWPWdttdOTfDKsdf4+vOvMnbkBAePnGDsyAmOnhjnyPFxXjp6giPHx/mvl17ipaMnOTne2VzCQP8iLr5g8RnjWcTlFy8laA6Gxf3B5RdfwEzmx8Dpc8f5H0oXLulj8KKl8z0MacatXLaYt165ZlbPMdcBsA7Y17I/Clw7x2OYtsV9ixi8aIDBiwbb9j01kd+ZTD5w+Dgnxyf49muneOHVya+kPDE+wfOHv82ZN1ynMjnwynHGz7jTOPjqcY6dPNX2vC8eOck3x45O84qm58UuwkzSzPiRDSuLC4Cp/qV83V++iNgGbAO44oor5mJMM6JvUdC3qA+ATWsunOfRdOfURJ5V+jpfPX/4OMfH2weltNAs7e+b9XPMdQCMAhta9tcD+1s7ZOYOYAdMzgHM3dB02mSYnf/lH4ArVrtiS+rUXC9neQTYHBGbImIJcCuwe47HIEliju8AMnM8In4N+ByTy0B3ZeYTczkGSdKkuS4BkZl7gD1zfV5J0uv5iSZJ6lEGgCT1KANAknqUASBJPeq8fhpoRIwB3+riLdYAL87QcM43JV8beH0LWcnXBgvj+r4nM9s+wuC8DoBuRcTwdB6ItBCVfG3g9S1kJV8blHV9loAkqUcZAJLUo0oPgB3zPYBZVPK1gde3kJV8bVDQ9RU9ByBJqlf6HYAkqYYBIEk9ygCQpB5lAEhSjzIAJKlH/T94A2azLD7JOQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1304b2bb9e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['Surname'].value_counts().plot() #how common are some surnames vs others"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "fd_surnames = FreqDist(df['Surname'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEzCAYAAAAmUOTXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8XHW9//HXJ3vTdC9tI5SWpZalsjRhUVERFMENvQrI1St6UbyKiper4oa4/sT9uqIoKF68LHJRaEWgssgOTUrLjpS2lNJ9oVuapEk+vz++Z5JpOs2cM0sm6byfj8c8MnPmfM75ZjKZz3yX8/2auyMiIhJXRakLICIiw4sSh4iIJKLEISIiiShxiIhIIkocIiKSiBKHiIgkosQhIiKJKHGIiEgiShwiIpJIVakLUAwTJ0706dOn5xy/Y8cORowYoXjFK17xZRXf2tq63t33ybqju+91t6amJs9HS0uL4hWveMWXXTzQ4jE+Y9VUJSIiiShxiIhIIkocIiKSiBKHiIgkUrTEYWZXmtlaM3siw3OfNTM3s4nRYzOzn5rZYjN7zMxmp+17jpk9F93OKVZ5RUQknmLWOH4PnNp/o5lNBd4MLE/bfBowI7qdB1wW7TseuAQ4DjgWuMTMxhWxzCIikkXREoe73wNszPDUj4HPA+lLD54O/CEaEfYQMNbMGoG3APPcfaO7bwLmkSEZFcqm7Z08tGQDSzbtLNYpRESGPfMiLh1rZtOBue4+K3r8TuBkd7/AzJYBze6+3szmApe6+33RfncAFwEnAnXu/q1o+8XADnf/QYZznUeordDY2Ng0Z86cxOW9e9kOfjZ/M69+RRWffe3ExPEpbW1t1NfXK17xilf8sIpvbm5udffmrDvGudgj1xswHXgiul8PPAyMiR4vAyZG9/8KnJAWdwfQBHwO+Era9ouB/8p23lwvALzvuXU+7aK5ftr3b8spPmU4XwCkeMUrvnzjGYIXAB4EHAAsimob+wELzGwKsAKYmrbvfsDKAbYXxZQxdQBs3NFTrFOIiAx7g5Y43P1xd5/k7tPdfTohKcx299XAzcAHo9FVxwOb3X0VcBtwipmNizrFT4m2FcWU0SFxbNjRnarhiIhIP8UcjnsN8CAw08xWmNm5A+x+C7AEWAz8BvgEgLtvBL4JzI9u34i2FcXI2ipG1VWxswdeblMHuYhIJkWbHdfdz87y/PS0+w6cv4f9rgSuLGjhBjBldB1b27exanM740bWDNZpRUSGDV053k+qn2PNlvYSl0REZGhS4ugn1c+xarMSh4hIJkoc/TRGNY7VqnGIiGSkxNHP5FTi2LyjxCURERmalDj66atxdJS4JCIiQ5MSRz+TR6vGISIyECWOfhrHhEXeV6tzXEQkIyWOfsbVV1NdAVvau2jr7Cp1cUREhhwljn7MjPEjKgHVOkREMlHiyGD8iPCyKHGIiOxOiSODCakah67lEBHZjRJHBqkah64eFxHZnRJHBhPqQ41D81WJiOxOiSODVFOVahwiIrtT4shgQtRUpRqHiMjulDgyGK8ah4jIHilxZDC2roIKg/XbOtjZrfXHRUTSKXFkUFVhTGyoxR3WbtVkhyIi6ZQ49qB3llw1V4mI7EKJYw/6ZslV4hARSafEsQdaCVBEJLOiJQ4zu9LM1prZE2nbvm9mz5jZY2b2ZzMbm/bcF81ssZk9a2ZvSdt+arRtsZl9oVjl7U8rAYqIZFbMGsfvgVP7bZsHzHL3I4B/Al8EMLPDgPcBh0cxvzSzSjOrBH4BnAYcBpwd7Vt0WglQRCSzoiUOd78H2Nhv2+3unlrk4iFgv+j+6cC17t7h7kuBxcCx0W2xuy9x907g2mjfotNKgCIimZm7F+/gZtOBue4+K8Nzc4Dr3P1qM/s58JC7Xx09dwXwt2jXU939I9H2fwOOc/dPZjjeecB5AI2NjU1z5szJudxtbW283F3Dp25dz6SRlVz21n0Sx9fX1+d1fsUrXvGKH+z45ubmVndvzrqjuxftBkwHnsiw/cvAn+lLXL8APpD2/BXAe4AzgN+mbf834GfZztvU1OT5aGlp8baOLp920Vyf8aVbvKenJ3F8vudXvOIVr/jBjgdaPMZn+6CPqjKzc4C3A++PCgqwApiattt+wMoBthfdiJpKxoyoprO7h43bOwfjlCIiw8KgJg4zOxW4CHinu7elPXUz8D4zqzWzA4AZwCPAfGCGmR1gZjWEDvSbB6u8U6J+Ds1ZJSLSp5jDca8BHgRmmtkKMzsX+DkwCphnZgvN7FcA7v4kcD3wFHArcL67d3voSP8kcBvwNHB9tO+gmBKNrNIsuSIifaqKdWB3PzvD5isG2P/bwLczbL8FuKWARYtNNQ4Rkd3pyvEBqMYhIrI7JY4BpBKHahwiIn2UOAagGoeIyO6UOAagPg4Rkd0pcQwgNV/VGiUOEZFeShwDGDOimtqqCrZ2dLGtoyt7gIhIGVDiGICZaSVAEZF+lDiy0EqAIiK7UuLIQisBiojsSokjC60EKCKyKyWOLBpHq8YhIpJOiSOLKeocFxHZhRJHFlPGjABU4xARSVHiyGKKRlWJiOxCiSOLiQ01VBis39ZJZ1dPqYsjIlJyShxZVFVWMGmUJjsUEUlR4ohhsmbJFRHppcQRg4bkioj0UeKIQUNyRUT6KHHEoMQhItJHiSOG3gWd1FQlIlK8xGFmV5rZWjN7Im3beDObZ2bPRT/HRdvNzH5qZovN7DEzm50Wc060/3Nmdk6xyjuQKVrQSUSkVzFrHL8HTu237QvAHe4+A7gjegxwGjAjup0HXAYh0QCXAMcBxwKXpJLNYNISsiIifYqWONz9HmBjv82nA1dF968C3pW2/Q8ePASMNbNG4C3APHff6O6bgHnsnoyKLlXjWLu1nZ4eH+zTi4gMKYPdxzHZ3VcBRD8nRdv3BV5M229FtG1P2wdVXXUlY+ur2dntbNjeOdinFxEZUsy9eN+gzWw6MNfdZ0WPX3b3sWnPb3L3cWb2V+A77n5ftP0O4PPASUCtu38r2n4x0ObuP8xwrvMIzVw0NjY2zZkzJ+dyt7W1UV9fv8u2C29fzwubu/j+myZw4LjqxPH5nl/xile84osd39zc3OruzVl3dPei3YDpwBNpj58FGqP7jcCz0f1fA2f33w84G/h12vZd9tvTrampyfPR0tKy27ZzrnzYp100129/cnVO8fmeX/GKV7ziix0PtHiMz/bBbqq6GUiNjDoHuClt+wej0VXHA5s9NGXdBpxiZuOiTvFTom2DTkvIiogEVcU6sJldA5wITDSzFYTRUZcC15vZucBy4Ixo91uAtwKLgTbgwwDuvtHMvgnMj/b7hrv373AfFJNHawlZEREoYuJw97P38NTJGfZ14Pw9HOdK4MoCFi0nvTWOzR0lLomISGnpyvGYemscW1TjEJHypsQRU2NqCVldBCgiZU6JIyYtISsiEihxxDR6RBUjqivZ3tnN1vadpS6OiEjJKHHEZGaaXl1EBCWORKZoJUARESWOJFI1Ds2SKyLlTIkjAa3LISKixJGIVgIUEVHiSEQ1DhERJY5EtBKgiIgSRyKp+arWqKlKRMqYEkcCExpqqawwNmzvpKOru9TFEREpCSWOBCorjMmjagFYu0Wz5IpIeVLiSGiyruUQkTKnxJGQVgIUkXKnxJGQVgIUkXKnxJGQVgIUkXKnxJGQVgIUkXKXOHGY2TgzO6IYhRkOtBKgiJS7WInDzO42s9FmNh5YBPzOzH5U3KINTVoJUETKXdwaxxh33wL8C/A7d28C3lS8Yg1dk0ZH13Fs7aC7x0tcGhGRwRc3cVSZWSNwJjA335Oa2X+a2ZNm9oSZXWNmdWZ2gJk9bGbPmdl1ZlYT7VsbPV4cPT893/Pno666kvEja+jqcTZsUwe5iJSfuInj68BtwGJ3n29mBwLP5XJCM9sX+DTQ7O6zgErgfcB3gR+7+wxgE3BuFHIusMndDwZ+HO1XUloJUETKWdzEscrdj3D3TwC4+xIgnz6OKmCEmVUB9cAq4CTghuj5q4B3RfdPjx4TPX+ymVke586bVgIUkXJm7tnb6c1sgbvPzrYt9knNLgC+DewAbgcuAB6KahWY2VTgb+4+y8yeAE519xXRc88Dx7n7+n7HPA84D6CxsbFpzpw5uRQNgLa2Nurr6/f4/K9bN3P7kh185OhRnHbwyMTx+Z5f8YpXvOKLEd/c3Nzq7s1Zd3T3Pd6AVwP/BbwIXJh2+xqwaKDYAY45DrgT2AeoBv4C/BuhGSy1z1Tg8ej+k8B+ac89D0wY6BxNTU2ej5aWlgGf/8nf/+nTLprrl/7t6Zzi8z2/4hWveMUXIx5o8Rif49maqmqABkLT0qi02xbgvTESWCZvApa6+zp33wncCLwGGBs1XQHsB6yM7q+IEgnR82OAjTmeuyC0EqCIlLOqgZ50938A/zCz37v7CwU653LgeDOrJzRVnQy0AHcRktG1wDnATdH+N0ePH4yevzPKjCWjlQBFpJwNmDjS1JrZ5cD09Bh3PynpCd39YTO7AVgAdAGPApcDfwWuNbNvRduuiEKuAP7HzBYTahrvS3rOQtNKgCJSzuImjj8BvwJ+C+S99J27XwJc0m/zEuDYDPu2A2fke85CSl+Tw90p8SAvEZFBFTdxdLn7ZUUtyTAyqraKkTWVbO/sZkt7F2NGVJe6SCIigybudRxzzOwTZtZoZuNTt6KWbAgzs95ah+asEpFyE7fGcU7083Np2xw4sLDFGT4ax9SxZN12Vm9pZ+aUUaUujojIoImVONz9gGIXZLjRSoAiUq5iJQ4z+2Cm7e7+h8IWZ/jQSoAiUq7iNlUdk3a/jnDtxQKgbBPHFK0EKCJlKm5T1afSH5vZGOB/ilKiYWKKVgIUkTKV65rjbcCMQhZkuNHV4yJSruL2ccwhjKKCsH7GocD1xSrUcDB5TFgJUFePi0i5idvH8YO0+13ACx5Nc16uJo6sparC2NS2k/ad3dRVV5a6SCIigyJWU1U02eEzhJlxxwGdxSzUcFBRYb1DclXrEJFyEitxmNmZwCOEOaPOBB42s1ynVd9raCVAESlHcZuqvgwc4+5rAcxsH+Dv9C31WpamqMYhImUo7qiqilTSiGxIELvXUo1DRMpR3BrHrWZ2G3BN9Pgs4JbiFGn46L0IUIlDRMrIgInDzA4GJrv758zsX4ATACOsxvfHQSjfkDZFM+SKSBnK1tz038BWAHe/0d0vdPf/JNQ2/rvYhRvqehOH+jhEpIxkSxzT3f2x/hvdvYWwjGxZU1OViJSjbImjboDnRhSyIMNR6jqOdds66OruKXFpREQGR7bEMd/MPtp/o5mdC7QWp0jDR01VBRMbaujucdZvK/trIkWkTGQbVfUZ4M9m9n76EkUzUAO8u5gFGy4mj65j/bZOVm9p7+3zEBHZmw1Y43D3Ne7+GuDrwLLo9nV3f7W7r871pGY21sxuMLNnzOxpM3t1tI75PDN7Lvo5LtrXzOynZrbYzB4zs9m5nrcY+hZ00rocIlIe4s5VdZe7/yy63VmA8/4EuNXdDwGOBJ4GvgDc4e4zgDuixwCnEaZwnwGcB1xWgPMXzGR1kItImRn0q7/NbDTweuAKAHfvdPeXgdOBq6LdrgLeFd0/HfiDBw8BY82scZCLvUe9NY4tWkJWRMqDuXv2vQp5QrOjgMuBpwi1jVbgAuAldx+btt8mdx9nZnOBS939vmj7HcBF0ZDg9OOeR6iR0NjY2DRnzpycy9jW1kZ9fX2sfe9c1sYv5m/h9fvXccFxYxPH53t+xSte8YovVHxzc3Oruzdn3dHdB/VG6FzvAo6LHv8E+Cbwcr/9NkU//wqckLb9DqBpoHM0NTV5PlpaWmLve+8/1/m0i+b6Wb9+IKf4fM+veMUrXvGFigdaPMbneCkmKlwBrHD3h6PHNwCzgTWpJqjo59q0/aemxe8HrByksmY1JVoJUH0cIlIuBj1xeBiN9aKZzYw2nUxotroZOCfadg5wU3T/ZuCD0eiq44HN7r5qMMs8kCljwnWQq7e0p2pEIiJ7tbiz4xbap4A/mlkNsAT4MCGJXR9dXLicsGgUhHmx3gosBtqifYeMhtoqGmqr2NbRxeYdOxlbX1PqIomIFFVJEoe7LyT0dfR3coZ9HTi/6IXKw5QxdSxeu43VW9qVOERkr1f2izEVQmqyQy3oJCLlQImjAFJTjaxR4hCRMqDEUQCqcYhIOVHiKIDeGocWdBKRMqDEUQCqcYhIOVHiKADVOESknChxFEAqcajGISLlQImjAMbX11BTWcHmHTvZ0dld6uKIiBSVEkcBVFQYk0ZHc1apuUpE9nJKHAXStxKgEoeI7N2UOAqkdyXALVpCVkT2bkocBdJX49BKgCKyd1PiKJC+tcdV4xCRvZsSR4E0pq3LISKyN1PiKBCtBCgi5UKJo0CmqMYhImVCiaNAJo2qxQzWbe2gu0dLyIrI3kuJo0CqKyuY2FBLj8PL7T2lLo6ISNEocRRQapbcDTs07YiI7L2UOAooNdnhhh2qcYjI3kuJo4BU4xCRcqDEUUCpGsdG1ThEZC9WssRhZpVm9qiZzY0eH2BmD5vZc2Z2nZnVRNtro8eLo+enl6rM2aRqHC+8vLPEJRERKZ5S1jguAJ5Oe/xd4MfuPgPYBJwbbT8X2OTuBwM/jvYbko47cDx11RUsXNPJDa0rSl0cEZGiKEniMLP9gLcBv40eG3AScEO0y1XAu6L7p0ePiZ4/Odp/yNlvXD3feOcsAC7+yxMsXrutxCUSESk8cx/8i9XM7AbgO8Ao4LPAh4CHoloFZjYV+Ju7zzKzJ4BT3X1F9NzzwHHuvr7fMc8DzgNobGxsmjNnTs7la2tro76+PqdYd+dHD2zkgZU72X9MFZeePIHaymR5Lp/zK17xild8rvHNzc2t7t6cdUd3H9Qb8Hbgl9H9E4G5wD7A4rR9pgKPR/efBPZLe+55YMJA52hqavJ8tLS05BV/74OP+Infv8unXTTXv3jjY4N+fsUrXvGKzwXQ4jE+x0vRVPVa4J1mtgy4ltBE9d/AWDOrivbZD1gZ3V9BSCREz48BNg5mgZMaUV3Bz//1aGqqKvjfh5cz97GV2YNERIaJQU8c7v5Fd9/P3acD7wPudPf3A3cB7412Owe4Kbp/c/SY6Pk7o8w4pB3+ijFc/LZDAfji/z3OCxu2l7hEIiKFMZSu47gIuNDMFgMTgCui7VcAE6LtFwJfKFH5EvvA8dM4bdYUtnZ08cn/fZSOLl0YKCLDX1X2XYrH3e8G7o7uLwGOzbBPO3DGoBasQMyMS99zBI+/tJnHX9rMd//2LF99x2GlLpaISF6GUo1jrzRmRDU//9fZVFUYV96/lHlPrSl1kURE8qLEMQiOmjqWL5x2CACf/dMiXnpZ65KLyPClxDFIzj3hAE46ZBKbd+zk09c8ys5uzWclIsOTEscgMTN+cMaRTBldR+sLm/jRvH+WukgiIjlR4hhE40fW8NOzj6bC4LK7n+cf/1xX6iKJiCSmxDHIjj1gPBe++ZUAXHjdQtZsaS9xiUREklHiKIGPn3gwJxw8kQ3bO7ng2kfp7hny1zOKiPRS4iiBygrjR2cdycSGWh5aspGf3flcqYskIhKbEkeJTBpVx3+fdRRm8JM7nuOB59dnDxIRGQKUOErohBkT+eQbD8YdPnPtQtZv6yh1kUREslLiKLELTp7BsdPHs3ZrBxdev4ge9XeIyBCnxFFiVZUV/OTsoxhXX809/1zH5fcuKXWRREQGpMQxBDSOGcEPzzwSgO/f9iwPrminS1eWi8gQpcQxRJx0yGTOe/2BdPc4P3jwZV596Z18c+5TPPHSZobB8iMiUkZKOq267Opzb5nJuPoa/nDfc6za2sEV9y3livuWMmNSA+86el9OP+oV7Dcu97WIRUQKQYljCKmurODjJx7EMQ2bqJp8MH959CVuXrSS59Zu4/u3Pcv3b3uW4w4Yz7uP3pfTXtXImBHVpS6yiJQhJY4hyMw4aupYjpo6li+/7VDufW4dNy54iXlPreHhpRt5eOlGvnrzk7zp0Em866h9OXHmJGqq1OooIoNDiWOIq66s4KRDJnPSIZPZ2r6TW59YzZ8ffYkHl2zglsdXc8vjqxlbX83bj2jk3Ufvy+z9x5W6yCKyl1PiGEZG1VVzRvNUzmieyqrNO7h54Ur+/OhLPLN6K1c/tJyrH1rO/uPrmTUe1tSsonn6OCaNqit1sUVkL6PEMUw1jhnBx95wEB97w0E8vWoLf3n0Jf6y8CWWb2xj+Ua4ZfECAKZPqOeY6eM55oDxHDt9PNMm1GNmJS69iAxnShx7gUMbR3No42g+f+ohzF+2kT/f/wQvddSxYPkmlm1oY9mGNv7UugKAfUbVcsz0cSGZTB/PoY2jqaxQIhGR+AY9cZjZVOAPwBSgB7jc3X9iZuOB64DpwDLgTHffZOHr8U+AtwJtwIfcfcFgl3s4qKwwjj9wAtWbGmhqaqKru4enVm3hkaUbmb9sIy3LNrFua0dv3whAQ20Vs6eN49gomRw5dWyJfwsRGepKUePoAv7L3ReY2Sig1czmAR8C7nD3S83sC8AXgIuA04AZ0e044LLop2RRVVnBEfuN5Yj9xvKR1x2Iu7Nk/XbmL93I/GWbmL9sI8s3tnHPP9dxT7QaYU1lBQeMreT1q5+iadp4mqaNY59RtSX+TURkKBn0xOHuq4BV0f2tZvY0sC9wOnBitNtVwN2ExHE68AcPl08/ZGZjzawxOo4kYGYctE8DB+3TwPuO3R+A1Zvbo9rIRh5ZtolnVm/h2Q09PHvvUn5z71IA9h9fT/O0ccyeNo6maeN45eRRat4SKWNWyukszGw6cA8wC1ju7mPTntvk7uPMbC5wqbvfF22/A7jI3Vv6Hes84DyAxsbGpjlz5uRcrra2Nurrc79CezjHb+/s4fFV21i2zXhm/U6e27CT9u5d3yP1VcaMCdUcMqGGmROreeX4akZU911HMpx/f8Urvpzjm5ubW929Odt+JescN7MG4P+Az7j7lgFG+mR6Yrds5+6XA5cDNDc3e1NTU85la21tpZzjR7a2cn4U39Xdw7NrttL6wiZaX9hEy7JNvPTyDhat6WTRmk4AKgxmThlN07SxNE0bx6aty5kxZf+cz9+xYymvG8avn+IVP5zj4yhJ4jCzakLS+KO73xhtXpNqgjKzRmBttH0FMDUtfD9g5eCVtrxVVVZw+CvGcPgrxvDBV08HQvPWguUhibQu38STL23m6VVbeHrVFq5+aHkIvOeRvM571KL7ecvhUzjl8MkctE9Dnr+FiBRSKUZVGXAF8LS7/yjtqZuBc4BLo583pW3/pJldS+gU36z+jdKaMqaOt76qkbe+qhGA9p3dLHrxZVqXb2Lh8pdZvX4To0ePzunYnd09LHxhIwtffJmFL77Md299hoP2Gckph0/hlMMmc+R+Y6lQ/4pISZWixvFa4N+Ax81sYbTtS4SEcb2ZnQssB86InruFMBR3MWE47ocHt7iSTV11JccdOIHjDpwA5F9Vvv/h+WwdOZXbn1rNHU+v5fl127ns7ue57O7nmTy6ljcfNplTDpvC8QdO0BxdIiVQilFV95G53wLg5Az7O3B+UQslQ0pdVQWvnTWFU2dNYWd3D/OXbeT2J9dw+5OrWbm5vXd6lVG1VbzxkEmccvhkTpw5iYZaXc8qMhj0nyZDWnVlBa85aCKvOWgil7zjMJ5cuYXbn1zN7U+t4ZnVW7l50UpuXrSSmsoKXnvwBE45fApdGzux5ZuoqjCqKiqorjSqKiuoqjCqKyuorLCM20QkHiUOGTbMjFn7jmHWvmO48JSZvLBhe6iJPLWalhc2cdez67jr2XAhI3c/kPDY9CaakVXOfg/dz5TRdUwZU8fk0XVMHl3LlNF1TB5Tx5TRdYxU7UbKmN79MmxNmzCSj77+QD76+gNZv62DO55ewx1Pr+XFtRupHTGSrp4eurqdnd09dPc4O7t9l21dPR5u3T30OOzsdnZ2d7NjJ6x/8eUBz91QWxWSyZg6Jo/qSyiTR9excUMnEzdsZ0JDLSNrKjWppOx1lDhkrzCxoZazjtmfs47ZP6fO+Z4eZ2dPDzu7nXsebmXS/jNYvaWd1ZvbWbOlndVbOlizJbq/uZ1tHV1sW9fF8+u2Zz7gnXcDUFNVwcSRNUxoqGVCQw0TRtYysaGG8WnbJo4MP8ePrKGuujLPV0Kk+JQ4RICKCqO2opLaKpg8soqm6eP3uK+7s3nHTtZs6WD1lnbWbG4PSSa6v3ztJnZ4Feu3ddC+s4eVm9tZubk9Vjkaaquoth7q591JVaX1Np9VRX0y1RVGVWXol6mq6OunSX9u08bNTFr+OJUVRoUZZlBpRkX0uMLChJhmFrYbuzy3cuV2Fu5YSmW0PdN+lRVE+/fFpZ5bt6GTg9o6GVtfU6g/jwwxShwiCZkZY+trGFtfw8wpo3Z7Pr3G09bZxYZtnWzY3smGbR1s2NbJ+u3h54ZtHdH2TjZE27Z1dAGwqX1HfoVcsjy/+EVP5RX+pTvnMX5kDQdMHMmBE0dy4D4NHLjPSA7aZyT7jx+pYdTDnBKHSBHV11RRP76KqeOzzx3k7mzZ0cXDCx7l0MNm9fa/9PbN9Dhd3dG2fs/19uF097B02QvsN3UqPQ7dPU6PO+7Q7eF+T4/3Puce3U97btXqNUzcZxLuHm0PZeuO4nqiY3Y7acfrO+bS1RtZ0+Zs3N7Jxu2dtL6waZffs7LCmDpuREgqUUI5cGIDB+0zUjMxDxNKHCJDhJkxpr6aCSMqYyWaPWmtXEdT0/Tc41t30NR0eB7xrcyePZvVW9pZum47z6/fzpJ121iybjtL1m9jxaYdvQuM9Y6CizTUVjGq2hl9zz3UVFVQW1VBTXQL9yupqaygtroi/Nxtn0pWvdTGMlZkjK+tqgyPKyt2e76mskIDGWJS4hCRgjMzGseMoHHMCF5z8MRdnmvf2c0LG9pCMlm/vTehLFm3nc07drKtA1Zt25pfAVoX5RRWU1lBlTkjbpmXIfFUZkw4tVWVfY8rK1i7Ziv3bvpnzkVft3obT3YuY1RdFaNqq2moq2JUXRWj66ppqK2ioa6K6srGWpNOAAAbNElEQVTSNvUpcYjIoKqrrmTmlFG79Q+5h+at+1sWMmPmoXR09dCZunV307Gzh87unt7te3p+5eq1jB47no7UPt09dOzsprO7b//e2LTndnZ72Ado6+rM75d86rn84p94csCn66orGFVXHSWXKkZFSWVUXRU7tmzhsFd1M6KmeCP0lDhEZEgwMyY01LLvqCoObcxtkkxIDU44KnFcT09IHI+0LODQWUfslnDSE1VHV3e/5NP3c8XKl2hsfEXO5V++YiUjx05ga3sX2zq62Nq+k63tXdFtJ9s6umjf2UP7zg7Wbe3IeIwfFrlCosQhIkIYTlxXUcnImoq8OulbW7fQ1PTKPOK30tT0qj0+7+60dXb3JpUt7V1sixLLto6dPLN4GbVVxb0eSIlDRGQYMTNG1lYxsraKyaPrdnu+tWJdhqjC0mBqERFJRIlDREQSUeIQEZFElDhERCQRJQ4REUlEiUNERBJR4hARkUTM3UtdhoIzs3XAC3kcYiKwXvGKV7ziyyx+mrvvk3Uvd9et3w1oUbziFa/4coyPc1NTlYiIJKLEISIiiShxZHa54hWveMWXaXxWe2XnuIiIFI9qHCIikogSh4iIJKLEISIiiWghpwIxs9cA00l7Td39DyUrkIhIkShxRPL54Dez/wEOAhYC3alwIHHiMLNxwFR3fyzm/v8y0PPufmPM4+wLTGPX3/+eOLH5MrMK4DF3nzUY5ysWM5sE9C7J5u7LS1icQWVmPwB+5+5PlroscZnZaHffYmbjMz3v7htjHqcS+LS7/ziPsvwfcCXwN3fvySG+Gvg48Ppo0z+AX7n7zlzLNOD5NKpqzx/87v7pmPFPA4d5ji+mmd0NvJPwob0QWAf8w90vjBH7uwGednf/9xjH+C5wFvAUu/7+78wWG8UfD/wMOBSoASqB7e4+Ok58dIw/Al/M58PWzN4GHM6uH97fiBFXC7yH3b84ZI2N4t8J/BB4BbCWkICfdvfDY8bXAedmKHvWv10Uvw9wEXBYv/iTBuP80TE+AnyY8Pr9DrjG3TcniH8t8DX6vrxYKIIfGDM+8XvQzOa6+9vNbCnhi56lPR373NGx7nb3E+PunyH+TYTX73jgT8Dv3f2ZBPG/BaqBq6JN/wZ0u/tHci3TQFTjCJrJ44MfeAKYAqzKMX5M9M3nI4RvbZeYWawah7t/OMdzpnsXMNPdO3KM/znwPsIbvhn4IHBwwmM0Ak+a2SPA9tTGBMnrV0A98Ebgt8B7gUdinvsmYDPQCuTyGnyT8A//d3c/2szeCJydIP5/gGeAtwDfAN4PPJ0g/o/AdcDbgP8AziF8+Ris8+PuvwV+a2YzCR+Aj5nZ/cBv3P2uGIe4AvhPwt+gO8u+mSR+D0ZJw4A3FKB2eL+Z/Zzwd0h//y6IE+zufwf+bmZjCO+deWb2IvAb4OoYNYdj3P3ItMd3mtmiRL9BEsWe02Q43AhvtsY84u8CNgG3ATenbgniHyd8cN4evQEgNN0kKcNkwj/f36LHhwHnxoz9G9CQx+/f0r/MwAMJj/GGTLcE8Y/1+9kA3B4z9ok83z+p338RUBHdfyRB/KP9yl4N3JkgvjXD6/+PwTp/2nEqgdOBvxASwEXAHODaGLEPF+hvkPg9mHr98jz/XRluiV5DYAJwAdASfYacRahF3R0jdgFwUNrjA4EF+f5ee7qVdY3DzOYQqqijgKeib7u93zg95rddQhU7H98gJJ373H2+mR0IPJfwGL8nNBF8OXr8T8K3nyv2FGBmPyP8/m3AQjO7g11//1hNdUCbmdVEx/geoeY1Mknh3f0fSfbPoD2tLK8ANgAHxIx9wMxe5e6P53jul82sAbgH+KOZrQW6EsSnvk2+bGazgNWEZrOk8aui5rqVwH6DeH7M7EeE5tY7gP/n7qna3nfN7NkB4mZHd+8ys+8DN7LrezDWN3byew8+ZGbHuPv8mPvvxt3fmGssgJndCBxCqP29w91TrRfXmVlLjEN8jvAaLiE0uU0j1PyKoqz7OMzsDQM9X4APs0FjZvPd/Rgze9Tdj462LXT3owaIOWegY7r7VQM9n3acaYS2/WpCc8MY4JfuvjhB+fPqJzGzi6P4k4FfEBLib9z9qwPEPB7tVwXMAJYQPrRS7etHxDz3SGAHYXj7+wm//9Uev3P1I8D/Aa8ifAFoAC5291/HjH87cC8wlfAajAa+7u43Jzz/EYQvH4nOHx3j3wk1i7YMz43xPfR3mNlAzVju8ftppgFrCO+dRO9BM3sKeCVhKYbtJPj7m9kH3P1qM8vYH+nuP4pZ/pPc/c44+w5wjFpgJqH8z3juTc/Zz1XOiSNfZnafu59gZlsJH0C9TxHeeHE/9ArROXk3oYN3nrvPjj6Iv+vuAybHoSL6VtW/jXqGu38pRmwFcLy7PxA9rgXq9vRhlRY3baDn3T3Wmi5m9l13vyjbtj3EVgDvdffr45yrGMys0t1z6Vfof5x3kjaqx93nJIitc/f2ftsmuPuGfMsV49wZ3wdx/v5m9jF3/7WZXbKHY3w9ZhnqgQuB/d39PDObQeh3nBsnPjrGLHYfIFGUSwKUOIAMH/wQOktbgP9y9yVFPv+fCJ2T/0pa56S7X5DgGLMJ3zZnETrr9wHOcPesHWTRm/Q77P6mizui5e2EDuL+I2KSjKpqcfdmM3ss9U3PzB5w99fEjH/Q3V8d93z9Yg8CVrh7h5mdSPjm/Qd3fzlm/AJ3n91vW+/vESP+Hnd/ffY99xh/APApdh8VFndgwXLgVkLT5p2ew4eCmX0HOJbQUQ+hg7fF3b8YM/6vwOnu3hU9ngL81d2bYsb3H5UFxH8PR8co2XBqM7uO0C/0QXefZWYjgAcHajHoF38JcCLhf/gW4DRC0/d7i1LgYnWeDKcb8HXgY4S+jtHAecBXCZ1TWTum0o4zCdg/dUsQl3fnJFBL+Ic5nJA8qoHamLH3EZp4HiP8432N0NQR99yLCR+2lsff4B5CM8MfgO8RmhsWJfwbvieXMhCGQFcRRuE8D/wYuCVG3McJAxu2R6/dY9HjpYSmqrjnvxj4LKGpaXzqliB+EfBpwoiyXAYWjADOJPQvvEAYoXRCwtfwMaKBAdHjShIM8AA+SuhUryQkwMeAUxLEP0P4sJxE6GSeAEyIGftOQp/i9uhv1wM8mfD3rwPOB35JuB7jSuDKBPGpzv1H0/+uCeIfJzSVLooeTwbmJP1fiH2+Yh14ON3IMKIDeCjuHy/fNx7RCJzow3MWYenHJQl/h91GUGTatofY1Kicx9O23Zvg3Helf2jk+DeYFn2AjQYuAX4EHJwgfmv0uncCW6LHW5K8dsDngU9F9x+NETcm+pC7Jip/6hb7Qz86ztIMt9h//0zv3zz+DuMIybs7Ydxj6b83IfklHRl4PmEU1uPAaxLG5vwaEBLvBPq+wL0RuDzhMf5EqHU/TxgOfTvwkwTxD0Tv/9R78SCSjcybH/1sjf6HLMlnUNJbWY+qStNjZmcCN0SP06t3cart+Y7jv9zCFeNfIQzDayB8C80qqtLvC4wws6Ppu4hpNOG6hjjao7b258zsk8BLhG9ucX0euMXM/sGuI2JidQxG+6bak3cQag+JuPuopDFpdprZ2YR+lXdE26pjnHMzoUnz7Kip8ATC++V+IFbHeHScuKO/9uQnUVPF7eQ2Iik1UOQswrf2+YQaSNxYA34APBp1dhuhryNrM1W/TmUj1LoWAseb2fHZ3kMFGpW10903mFmFmVW4+10WLopN4mB3P8PMTnf3q8zsfwkjJeO6hNBcONXCxbCvBT4UJzB6/R8zs7GE6z5agW3Ev44pMSWO4P3ATwjVTAceAj4QtTN+MkZ8Tm+8fv80qaFzv4h+xh1K+BbCG2w/wrf0lK1A1o7lyGcISebThCR4EuFbU1zfJrxR6wjNTYkVqJ8k12lTPky4cO7b7r406jO4OsF5L6avqQfgd2b2J3f/Vsz4/tNF3A382uNPF/EqwpXCJxFqXRDex3FHJC0lfFhfD3zO3bdnCdmFu7uZXUD48nQM4W93kbuvjhHeP+H/eQ/b9+SH/R43pxeNeK9BvsOpIc8hze4+z8wWEF5DAy5w9/UxY93MjvLQJ/crM7sVGO0xpy3KhTrHC8DM/k64+vpSQpV3LeFCvgE7dtNGYswk/MOlhk++A7jHE0wXYGbvcff/S1r2Qkh1bOd5jMXAvxCay3LpnM132pQawpBMgGcTfGinppw52qNRQdEXjgXufmjM+LymizCzZ4Aj3L0zbpn7xY929y25xKYd4xeEaTJyvhaiVPYwnPqPnmBEV65DqtNqTBnFrTUO9uuvxEHvXD8fZfdRKXHnCqonXIBmwAcIzUR/9Pjj+G8H3uPuW6PHo4A/ufupCX6NfOZqeiXhAqL+39bjfmO9lNCZf3uS8vY7xl3AyZ7DBG9R/LOED8/EY9ejkVRXAcvoay45J2ZtBTP7G3B29I2PqMngand/e8z4Rb7rdBEZtw0Qfx2hb2ZtnP0zxBdiOHhO10KY2X+7+2es72LcXcRN/NGxcn3//zuhTy/pRbfpx0if7yzVzOnZzm9917HUEWpLiwiv3RGEfpsTYp4/52tRcqGmquAmwgVUfyfBPDkWXcdBuPAo9aZP9TF8y8w2At93919mOdT+hE7dlE6SX7mbz1xNfwJ+RWgfzWU8//nA582sg1BlT9zMRP79JEsI/7C5XPT0Q8IInmehN5FeA8QaChqd80kzm0d4H7wZuM/MfgqxrsDvNrOD3P356PwHkuzvMBl4xszmk9vMB3nPVUXoG8nF/0Q/f5BjPJD3+386oWl6OmEI/r2ERLIwQRFymu/MoyvOzexa4DyPZi+Imrs+m+D8ub7+OVGNA7JeYZ3HcScQ5suZmWW/LxPayP9M+OB5N3Cdu38nwbkec/cj0n42ADe6+ykxYls95nj5DLFGmAY+rzHvUa1rG2FETW+tw7NcQGV906bsCxxJmPIi0bQpma65SHgdRl5X4JvZyYQrtneZLsLjTQ6Y6tjOdN5YMx9YNNtA2nunGrgtbo1zKMjn/Z92jBGElofPAvu6e2WC2Cc8j2UBMn0GFetzqRBU4wjmmtlb3f2WQh406jA/McZ+346aO14Xbfqwuz+a8HQ7op+x52qyvnUI5pjZJwiJK/1DN2tTW9Qx92fifzvfk/FJ/snTpObxaaWvjyjxMczsCvq+/b4/Ol4s2RLDQCyMZttBmPIk8XQRFtaCuNjd35RrGSjAXFX5sjwvQmX3uco2kuX9n3burxBGMTUAjxISx70xz5uS73xnT0d9XVcTvgh9gOS1vkGjxBFcAHwpz6aWjLxvsrJs+y0gzHCZq7lR2/r36PvQ+22WmFZ2XYfgc+zazhz3nzbvSeIIU0qfkrSfJNOHtiVcDIswoul8wqgyI4yuyda8mH6+pWRun8/6+rl7j5n90MNV74lHwbh7t5m12QDzQcWQ83DwAvodYUjqjwnNTR+m730Zx5zo/f99wv+RE5pe4/gXwiiqvxIWQHrI+01/EsMJwIei90Li+c4Iv+/HCZ9FEN6DlyUsw6BRU9VeIqpmf5xQa3HCN6bL4vwDWLiG5VYPa4JcDMwGvplgRMdThG/Ly8ixY87CtC8jCf90iZO35bEYVr6iJsmUOuAMQg1qjxMs9ov/OiFp3JjjiLLrCcM457HrWhBxFyI7wN2XZttWTKnmUjN73N1fFW27191fly022vcMwnt4a47v4VGED/8TCM3Ga+J2TEfx0zJt95jznQ03ZZ04zOwQd39mT0Pi4r7phoLow2MrfdcfnA2MdfesF3KltQufAPw/Qmfxl9z9uJjnLvk/TVo7/UcItY1L4vZTWAHmOcpwzPvifvCkJc0u+kbnJUmaGftY4jahWea5tnLu98qFhUWfXke4CPdOwkWol2brH0yLz/k9HDXPvY4wVUsz8CKhczxW4i+EYrwHi6ncm6ouJMxLlX4RUXomHTadg4SZNNOHb95l8VcAS43geRthneKbzOxrcU/s7i9E/7Az3P13FoY3N8SNBzCzjJP8efx1z6vMrJHwbfHL2XbuJ6/V5/p98aggfPjEvpLd3UdF/U0zSGvfTxCfUx+LmR1CGL46xnZdu350LuXIU74XoebzHv4uoWnop4SpO4qyTncW+a6AOKjKOnG4+3nR3cvI0FRTupLl5FELUzQ8BGBmxxGmvojjJTP7NfAmwsI7tYQPwFgsXMjYTGiu+h1hWOzVhA7HuD6Xdr+OMNNqK/GTdz6LYW1297/FLunufkjfF44uQpPdGXGDo1rSBYSr/xcSmp0eIEw8GSc+147lmcDbgbH0TbUCoeb60ZjFL4i0/rFt5LYAUc7vYXd/Ww7nK7R834ODqqybqlLybaoZCixcvTwTSA2L3Z8wKqOHLP0NFi5gPJVw1fZz0Tf3V8XtqDazhcDRhKulU4tIxR7OuodjTgW+5+5J5vxKeo5UTeFMwqysOa0+Z2b/xa6DDJxoTL/HuBbAwoJSxxA6ZY+KagJfd/ezYp7/Pvo6lt9B1LHs7pcMGNgX/2p3fzDOvsVi+V+Emvg9bGbXu/uZ1regV+9TFPHiuT2U5VLyeA8OtrKucaTJq6lmiEh0lXk6D6u23Zj2eBVh6c24OqNhuQ6kpnDI1wrCTMEDMrPPu/v3rO96jl1k6SDuP89Rqk3fSDDXUxTXTBiRZIT30XzgPyzMWfW9LPHt7t5uZphZbdTvFqttPzLC3e8wM4v6lb5mZvcSkkkc7zazJwnDgm8lXA/zGXePPV9XAeR1EWqO7+HUCKan2bXGa4TRiYMp9SU1l7m2Bp0SR5BXU81QUOLRG9dHr99YM/so8O/EHwoJ7HIhH4TX/mhCs002qbHuLWRIHAPxvqt2M9UYtliYOC5OGSYAs919W3S8SwidvK8nNLdl+xBaEQ0l/Qswz8w2EdYNjyvf2Y1PcffPm9m7CQn7DMJU+YOZOLrcfVCHn6YNlT+4//9PVOsbzLLktWb5YFPiCM4kfGP/gbu/HFVzP5clRvr0EIb/biHMl/NVd5+X8BgvEIbQQugnuIawpsOAvG950qcIswFPp+997YS1JbLZU43hYzFrDP2njNkJTHP3HdG1Qdl+h3dHd79mYe6iMYRv/nHl27GcmlvprcA17r7RLMklFLmzAlyEmse5Pw58AjjQzNKvoRlF/P7BQpYnp7m2SkF9HJK36Bv2mYSrda8FbnD3NQmPsQD4kEcX7VlYH+MzCYYEP0tI9v2nLImzbvRthEkmUzWGBkKN4d2EforDssRfHO17U7TpHYQk9EPCgkDvj/M7lErUvv4uQlPVsYTO8rmD0cdnfRdPpmeq3g+lYg5HNbMxhIWrvgN8Ie2prcVMWHsoS8a5ttz93MEsR1xKHFIwZnYEYWrz9xDW8I49DUY0CuoGwnQfJxAWVXq7x7waOsl1ExlinwaO9Gha8qipcqG7H5q6PiTGMZqichthZFdLlpCCybdjOTrGOMKKid1RR/Noj7eeRkFYnhehDndWgLm2BpOaqqSQ1hLmOdpAsjZ23H2Jmb2P0M7/IqHdfUeWsHSXWJjrp/8khzfuOaTX/xKmTUmvMVwTdfI/FbP8rSSY36rA8p3dGMIkkW+2MMV6SpxmvkL5irtfH41sfDOhtnYZfZ3Ge7vEc82VkhKH5C1qKz4L2IdQa/iou8f6wM0wFHI8YVjiw2ZGgiGRHwYOIbTXp6+ClzVxuPs3zewW+moM/5FWYxjSzUyRvDqWo6bGEwnXgdxCmKL7PgY3cewNIxvzkctccyWjpirJW9RGfm3MEUj9YzNOV5ISd7SYpc1xVC7SOpY/Tajt5dSxHCXvI4FH3f1IM5sM/Nbd35EltGDMbC5hNNibCIMVdhDa+GMtZjXcWR5zzZWCEofsFczsN8CP49Z09gaF6lg2s0fc/VgzayV0zm4FnnD3wwtZ3ixlyOsi1OHO8phrrhTUVCV7ixOAcyz3aa2HHXc/APbcsZzgUC1RM8lvCM0k24i/el5BFOAi1OEun7nmBp1qHLJX2FOTV4kvjBwUhZwyx8LyqaM9/lomUgBm9ntC3076XHPnuPsnSlqwPVDiEBnmrG9K+e8Qmnr+N+4w4ij+JuA64CZ3355tfym8fOaaKwUlDpFhLt+OZQtrlp9FGNH0CCGJzB2qHbN7o0INEhksShwiw1yhOpYtrF9+EmFK9VO9AEsny95JiUNEUsNB30Goecwm1Dg+VdpSyVClxCFS5szsOsIV2rcC1wN3u3vPwFFSzpQ4RMqcmZ0KzHP3Ib9kqQwNShwiZcrMTnL3O23X9cZ7xZznS8qQLgAUKV9vAO5k1/XGU2LN8yXlSTUOkTIWrRz4Xne/vtRlkeFjWC2PKiKFFXWCf7LU5ZDhRTUOkTIXzW+1g3DhX++V44O9Cp4MH0ocImUumhgyZVCWbZXhTU1VInIRYencA4DfAYsIa16LZKTEISJfiaZkTy3b+nvCsq0iGSlxiMhuy7YCNSUsjwxxShwi8pKZ/Ro4E7jFzGrRZ4MMQJ3jImWu3JdtleSUOEREJBFVR0VEJBElDhERSUSJQyQLM/uymT1pZo+Z2UIzO66I57rbzJqLdXyRQtDsuCIDMLNXA28HZrt7h5lNRENVpcypxiEysEZgvbt3ALj7endfaWZfNbP5ZvaEmV1uZga9NYYfm9k9Zva0mR1jZjea2XNm9q1on+lm9oyZXRXVYm6IRjbtwsxOMbMHzWyBmf3JzBqi7Zea2VNR7A8G8bUQAZQ4RLK5HZhqZv80s1+a2Rui7T9392PcfRYwglArSel099cDvwJuAs4HZgEfMrMJ0T4zgcvd/QhgC/CJ9JNGNZuvAG9y99lAC3ChmY0H3g0cHsV+qwi/s8iAlDhEBuDu24Am4DxgHXCdmX0IeKOZPWxmjwMnAYenhd0c/XwceNLdV0U1liXA1Oi5F939/uj+1cAJ/U59PHAYcL+ZLQTOAaYRkkw78Nto5b62gv2yIjGpj0Mki2gt7ruBu6NE8THgCKDZ3V80s68BdWkhHdHPnrT7qcep/7n+F1D1f2yEdcDP7l8eMzsWOBl4H2EtjZMS/koieVGNQ2QAZjbTzGakbToKeDa6vz7qd8hlJtn9o453gLOB+/o9/xDwWjM7OCpHvZm9MjrfGHe/BfhMVB6RQaUah8jAGoCfmdlYoAtYTGi2epnQFLUMmJ/DcZ8GzonmiHqOfrPRuvu6qEnsmmjuKAh9HluBm8ysjlAr+c8czi2SF005IjLIzGw6MDfqWBcZdtRUJSIiiajGISIiiajGISIiiShxiIhIIkocIiKSiBKHiIgkosQhIiKJ/H/+ulcuWUbuqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13050093780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fd_surnames.plot(20) #20 most common surnames"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Distribution of community in data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "hindi        2935\n",
       "punjabi      2208\n",
       "bengali      2073\n",
       "telugu       1770\n",
       "gujarati     1760\n",
       "marwari       674\n",
       "oriya         672\n",
       "tamil         661\n",
       "marathi       545\n",
       "urdu          446\n",
       "malayalam     364\n",
       "kannada       358\n",
       "english       302\n",
       "sindhi        131\n",
       "assamese       30\n",
       "konkani        26\n",
       "nepali         13\n",
       "kumaoni        10\n",
       "garhwali        6\n",
       "haryanavi       6\n",
       "tulu            4\n",
       "kashmiri        2\n",
       "pashto          1\n",
       "persian         1\n",
       "mizo            1\n",
       "Name: Community, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Community'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1304b427940>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAADuCAYAAAAwTtAhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHclJREFUeJzt3Xt4XHW97/H3dy5J2txvTUov9JamFJFSQukFthUQCl4KunGDj9hH4VQFtoi3g249gB7Ocbu3IroRrYKiIoiCm25OBWthI7a1kGKp9B56Db0kbXpJ2ibNzPzOH7OK0zZtJskkKzPzeT3PPGvNb35r5rseHvLpWr+1fsucc4iISPYJ+F2AiIj4QwEgIpKlFAAiIllKASAikqUUACIiWUoBICKSpRQAIiJZSgEgIpKlFAAiIlkq5HcBZ1JRUeHGjBnjdxkiImll5cqVe51zld31G9QBMGbMGOrr6/0uQ0QkrZjZtmT6dXsKyMzyzOwVM3vdzNaY2b1e+1gzW2Fmm8zs12aW47Xneu8bvM/HJHzXl732DWZ2Ve92TUREUiGZMYAO4DLn3PnAFGCOmU0H/hW43zlXA+wHbvb63wzsd85NAO73+mFmk4EbgHOBOcAPzCyYyp0REZHkdRsALq7Nexv2Xg64DPit1/4ocK23Ptd7j/f55WZmXvsTzrkO59wWoAGYlpK9EBGRHkvqKiAzC5rZKqAJWAy8CRxwzkW8Lo3ACG99BLADwPv8IFCe2N7FNom/Nd/M6s2svrm5ued7JCIiSUkqAJxzUefcFGAk8X+1n9NVN29pp/nsdO0n/9YC51ydc66usrLbQWwREemlHt0H4Jw7APw3MB0oMbPjVxGNBHZ6643AKADv82KgJbG9i21ERGSAJXMVUKWZlXjrQ4ArgHXAi8A/et3mAc946wu993ifv+Dijx1bCNzgXSU0FqgBXknVjpysvTNKe2e0v75eRCTtJXMEMBx40cxWA68Ci51zzwL/E/icmTUQP8f/sNf/YaDca/8ccBeAc24N8CSwFngOuM051y9/oXe0HOH8e//Awtd1gCEicjrd3gjmnFsNXNBF+2a6uIrHOdcOXH+a77oPuK/nZfbMyNIhFOSGWP7mPj5cN6r7DUREslBGzgVkZswYX87Shr3oofciIl3LyAAAmDWhgqbWDt5sPux3KSIig1LGBsDM8eUALH9zr8+ViIgMThkbAKPLhjKiZAhLG/b5XYqIyKCUsQFgZswcX87yzfuIxTQOICJysowNAICZE8o5eLSTtbsO+V2KiMigk9EBMGNcBQDL39RpIBGRk2V0AFQX5zGuMp+lGggWETlFRgcAwKzxFbyypYXOaMzvUkREBpWMD4CZ48s5cizK6sYDfpciIjKoZHwATB9Xjhks0+WgIiInyPgAKM3PYfLwIo0DiIicJOMDAOKngV7bdkDTQ4uIJMiSAKjgWDTGym37/S5FRGTQyIoAuGhsGaGAsbRBp4FERI7LigAoyA1x/qgSlumGMBGRt2VFAEB8HGB14wEOtXf6XYqIyKCQNQEwY3w5MQevbmnxuxQRkUEhawJg6uhSckMBTQ8tIuLJmgDICwepG1PKMt0PICICZFEAQPxy0PW7W9nX1uF3KSIivsuyAIg/JvIvmzUOICKSVQFw3ohiCnJDmhZCRIQsC4BQMMDFY8v0gBgREbIsACB+OeiWvYfZeeCo36WIiPiq2wAws1Fm9qKZrTOzNWZ2h9d+j5m9ZWarvNc1Cdt82cwazGyDmV2V0D7Ha2sws7v6Z5fObNaE+GMidVewiGS7ZI4AIsDnnXPnANOB28xssvfZ/c65Kd5rEYD32Q3AucAc4AdmFjSzIPAgcDUwGbgx4XsGTG1VIWX5ObocVESyXqi7Ds65XcAub73VzNYBI86wyVzgCedcB7DFzBqAad5nDc65zQBm9oTXd20f6u+xQMCYMa6c5W/uwzmHmQ3kz4uIDBo9GgMwszHABcAKr+l2M1ttZo+YWanXNgLYkbBZo9d2uvaTf2O+mdWbWX1zc3NPykvazAnl7DrYztZ9R/rl+0VE0kHSAWBmBcBTwGedc4eAh4DxwBTiRwjfPt61i83dGdpPbHBugXOuzjlXV1lZmWx5PTJzfHwcQNNDi0g2SyoAzCxM/I//Y865pwGcc3ucc1HnXAz4MX8/zdMIjErYfCSw8wztA25M+VCGF+fpclARyWrJXAVkwMPAOufcdxLahyd0uw54w1tfCNxgZrlmNhaoAV4BXgVqzGysmeUQHyhemJrd6BkzY8b4cpZv3kcsdspBiIhIVuh2EBiYBdwE/M3MVnltXyF+Fc8U4qdxtgKfBHDOrTGzJ4kP7kaA25xzUQAzux14HggCjzjn1qRwX3pk1vgKnn7tLdbvbmXyWUV+lSEi4ptkrgL6M12fv190hm3uA+7ron3RmbYbSDO8eYGWvblXASAiWSnr7gQ+7qySIYytyNc4gIhkrawNAIjPDrpiSwuRaMzvUkREBlyWB0AFbR0RVr910O9SREQGXFYHwPRxZQA6DSQiWSmrA6C8IJdzhhdpXiARyUpZHQAQHweo37qf9s6o36WIiAwoBcD4cjoiMV7bvt/vUkREBlTWB8C0sWUEA6ZxABHJOlkfAIV5Yd45slgPiBGRrJP1AQDx00Cv7zhAW0fE71JERAaMAoD4/QCRmOPVLS1+lyIiMmAUAMCFZ5eSEwroclARySoKACAvHOTC0aUaBxCRrKIA8MwcX87aXYf4Tf0OnNMzAkQk8ykAPDfNOJuLx5bxxd+u5vbH/8rBo51+lyQi0q8UAJ6SoTk8dst0vjSnluff2M01D7zMKxoUFpEMpgBIEAwYt86ewFOfnkk4aNywYDnf+cMGTRctIhlJAdCF80eV8OxnLuWDU0fyvRca+PCPlrOj5YjfZYmIpJQC4DQKckP8+/Xn8/0bL2BTUxtXP/Ayz6x6y++yRERSRgHQjfeffxa/v+NSJlUXcscTq7jz16tobdcAsYikPwVAEkaWDuWJ+dO584qJLHx9J9d872XNHioiaU8BkKRQMMAdV9Tw5Cen4xxc/8PlfH/JJqIx3TMgIulJAdBDF55dxqI7LuV97xzOtxdv5NbHVurGMRFJSwqAXijKC/PADRfwxatqeX7NHv7f33b5XZKISI8pAPrgU+8az3kjirn3v9ZySAPDIpJmug0AMxtlZi+a2TozW2Nmd3jtZWa22Mw2ectSr93M7Htm1mBmq81sasJ3zfP6bzKzef23WwMjGDD+z3Xnsa+tg397boPf5YiI9EgyRwAR4PPOuXOA6cBtZjYZuAtY4pyrAZZ47wGuBmq813zgIYgHBnA3cDEwDbj7eGiks/NGFvOxGWP45YptrNpxwO9yRESS1m0AOOd2Oede89ZbgXXACGAu8KjX7VHgWm99LvBzF/cXoMTMhgNXAYudcy3Ouf3AYmBOSvfGJ5+/ciLDCnP5ytN/07QRIpI2ejQGYGZjgAuAFUCVc24XxEMCGOZ1GwHsSNis0Ws7XfvJvzHfzOrNrL65ubkn5fmmMC/MPe8/l7W7DvGzZVv9LkdEJClJB4CZFQBPAZ91zh06U9cu2twZ2k9scG6Bc67OOVdXWVmZbHm+m/OOai6bNIzvLN7IWweO+l2OiEi3kgoAMwsT/+P/mHPuaa95j3dqB2/Z5LU3AqMSNh8J7DxDe0YwM+79wLnEnOOehWv8LkdEpFvJXAVkwMPAOufcdxI+Wggcv5JnHvBMQvvHvKuBpgMHvVNEzwNXmlmpN/h7pdeWMUaVDeWzV0xk8do9/GHNbr/LERE5o2SOAGYBNwGXmdkq73UN8E3gPWa2CXiP9x5gEbAZaAB+DNwK4JxrAb4BvOq9vu61ZZSbLxnLpOpC7lm4hsMdEb/LERE5LRvM0xjU1dW5+vp6v8vosZXbWvjQQ8u55ZKxfPV9k/0uR0SyjJmtdM7VdddPdwL3gwvPLuPGaaP56bKtrNl50O9yRES6pADoJ3fNmUTp0DBf+d0bmjFURAYlBUA/KR4a5qvvnczrOw7wqxXb/C5HROQUCoB+NHfKWVwyoYJvPbeBpkPtfpcjInICBUA/MjO+ce076IjG+Pqza/0uR0TkBAqAfja2Ip/bZk/g2dW7eGljekxtISLZQQEwAD41exzjKvP52n++QXtn1O9yREQABcCAyA0F+d/XvoPtLUf4/gub/C5HRASAkN8FZIuZ4yv44NQRLPjTZsLBACVDwuTnhijMC5GfG6LgpPX8nBCBQFfz54mIpIYCYAD9yzXnsGr7Ab77x+SOAvJzghTkhTirZAi/uPliCnL1n0tEUkd/UQZQeUEuL3xhNh2RKIc7ohzuiNDaHqGtIxJf74jQ1n7i+vaWI/xx3R5W7zjAzAkVfu+CiGQQBYAPckNBckNByvJzuu3b1NrOH+/bw4Y9rQoAEUkpDQIPcpUFuZTl57Bhd6vfpYhIhlEADHJmxsSqAjbsUQCISGopANJAbVUhG3e3Mpin7haR9KMASAO11UUcPhalcb+eNSwiqaMASAO11QUAbNRpIBFJIQVAGqipKgTQOICIpJQCIA0U5YUZUTJEVwKJSEopANLExKoCBYCIpJQCIE3UVhfxZnMbndGY36WISIZQAKSJ2uoCOqOOrXsP+12KiGQIBUCamKiBYBFJMQVAmhhfWUAwYBoHEJGU6TYAzOwRM2syszcS2u4xs7fMbJX3uibhsy+bWYOZbTCzqxLa53htDWZ2V+p3JbPlhYOMKR+qABCRlEnmCOBnwJwu2u93zk3xXosAzGwycANwrrfND8wsaGZB4EHgamAycKPXV3qgtrpQN4OJSMp0GwDOuT8BLUl+31zgCedch3NuC9AATPNeDc65zc65Y8ATXl/pgdqqIra1HOHIsYjfpYhIBujLGMDtZrbaO0VU6rWNAHYk9Gn02k7Xfgozm29m9WZW39zc3IfyMk9tdQHOQUNTm9+liEgG6G0APASMB6YAu4Bve+1dPcTWnaH91EbnFjjn6pxzdZWVlb0sLzPVVhcBsF7jACKSAr16Iphzbs/xdTP7MfCs97YRGJXQdSSw01s/XbskaXTZUHJDATYqAEQkBXp1BGBmwxPeXgccv0JoIXCDmeWa2VigBngFeBWoMbOxZpZDfKB4Ye/Lzk7BgFGjh8OISIp0ewRgZo8Ds4EKM2sE7gZmm9kU4qdxtgKfBHDOrTGzJ4G1QAS4zTkX9b7nduB5IAg84pxbk/K9yQK1VUW8vEljIyLSd90GgHPuxi6aHz5D//uA+7poXwQs6lF1cora6gKeeq2R/YePUZrEQ+VFRE5HdwKnGU0JISKpogBIM5O8K4F0Q5iI9JUCIM1UFeVSlBfSlBAi0mcKgDRjZkyqLlIAiEifKQDS0MTq+KWgznV5L52ISFIUAGmotqqQ1vYIuw+1+12KiKQxBUAa0pQQIpIKCoA0VOtdCqopIUSkLxQAaah4aJjqojwNBItInygA0tTE6kLdDCYifaIASFO1VQVsamojGtOVQCLSOwqANFVbXcSxSIyt+w77XYqIpCkFQJrSQLCI9JUCIE1NGFaAmSaFE5HeUwCkqSE5QcaU5+tKIBHpNQVAGpuop4OJSB8oANJYbXURW/cepr0z6ncpIpKGFABprLaqkJiDhqY2v0sRkTSkAEhjtdUFgB4OIyK9owBIY2PK88kJBjQQLCK9ogBIY6FggPHDNBAsIr2jAEhztVUFuhlMRHpFAZDmaquL2HmwnYNHO/0uRUTSjAIgzR0fCN6k00Ai0kMKgDSnp4OJSG91GwBm9oiZNZnZGwltZWa22Mw2ectSr93M7Htm1mBmq81sasI287z+m8xsXv/sTvY5qziPgtyQLgUVkR5L5gjgZ8Cck9ruApY452qAJd57gKuBGu81H3gI4oEB3A1cDEwD7j4eGtI3ZhafEkJHACLSQ90GgHPuT0DLSc1zgUe99UeBaxPaf+7i/gKUmNlw4CpgsXOuxTm3H1jMqaEivVRbXcSGPa04p4fDiEjyejsGUOWc2wXgLYd57SOAHQn9Gr2207Wfwszmm1m9mdU3Nzf3srzsUltVwIEjnTS3dvhdioikkVQPAlsXbe4M7ac2OrfAOVfnnKurrKxMaXGZSgPBItIbvQ2APd6pHbxlk9feCIxK6DcS2HmGdkmBiVWaE0hEeq63AbAQOH4lzzzgmYT2j3lXA00HDnqniJ4HrjSzUm/w90qvTVKgvCCXioJcDQSLSI+EuutgZo8Ds4EKM2skfjXPN4EnzexmYDtwvdd9EXAN0AAcAT4O4JxrMbNvAK96/b7unDt5YFn6YFJ1oeYEEpEe6TYAnHM3nuajy7vo64DbTvM9jwCP9Kg6SdrEqkJ+9co2YjFHINDVkIuIyIl0J3CGqK0uoL0zxo79R/wuRUTShAIgQ+hKIBHpKQVAhqgZ5l0JpAAQkSQpADJEfm6I0WVDWa+BYBFJkgIgg0ysKtQRgIgkTQGQQWqrC9iy9zAdkajfpYhIGlAAZJDa6iIiMcfm5sN+lyIiaUABkEFqqwoBTQkhIslRAGSQsRX5hAKmKSFEJCkKgAySEwowvlIPhxGR5CgAMsxEzQkkIklSAGSYSdWFNO4/SltHxO9SRGSQUwBkmIkaCBaRJCkAMszbVwJpHEBEuqEAyDAjS4cwNCeoSeFEpFvdPg9A0ksgYNRUFfL7N3bR3NpBIGAEDQJm3np8GTAIBoyAGcFA/DUkHKQwL0RRXpjCvBCF3rIgL/R2e24ogJmeNyCSCRQAGej6C0fy8+VbWb/7EM5B1DmiMRdfjzmizuG8tmjMEfPaj3Z2P4VEOGhvB8Owwlxue/cEZtcO6/+dEpGUs/hDvAanuro6V19f73cZWSMac7R1RGjriNDa3klr+9+Xh9pPbXt9xwG27jvCe88bztfeN5nq4jy/d0FEADNb6Zyr666fjgDkbcGAUTwkTPGQMDCk2/4dkSgLXtrM919s4KWNzXz+yoncNP1sQkENLYmkA/2fKr2WGwryz5fXsPjOf2Dq2aXc+19rmfvgUlbtOOB3aSKSBAWA9NnZ5fk8+vGLePAjU2lu7eC6Hyzla//5BgePdvpdmoicgQJAUsLMeO87h7Pk8+9i3owxPLZiG5d/+yWeWfUWg3mcSSSbKQAkpQrzwtzzgXNZePslnFWSxx1PrOKjD69gc3Ob36WJyEkUANIv3jGimN/dOotvzD2X1TsOMue7L3P/4o20J3GpqYgMDAWA9JtgwLhpxhiWfOFdXH1eNQ8s2cR77n+Jp19rJBrTaSERv/UpAMxsq5n9zcxWmVm911ZmZovNbJO3LPXazcy+Z2YNZrbazKamYgdk8BtWmMcDN1zAL2++mMLcMJ978nWueeBl/rBmt8YHRHyUiiOAdzvnpiTcdHAXsMQ5VwMs8d4DXA3UeK/5wEMp+G1JI5fUVPDsP1/C92+8gGPRGPN/sZLrfrCMZQ17/S5NJCv1xymgucCj3vqjwLUJ7T93cX8BSsxseD/8vgxigYDx/vPPYvGd/8A3P3geew6185GfrOCjP1nB67p/QGRA9TUAHPAHM1tpZvO9tirn3C4Ab3l8opgRwI6EbRu9thOY2Xwzqzez+ubm5j6WJ4NVKBjghmmjefELs/nqe89h7a5DzH1wKZ/8RT2b9CwDkQHR16kgZjnndprZMGCxma0/Q9+uppA85QSwc24BsADicwH1sT4Z5PLCQW65dBz/dNEoHvnzVn788mYWr/0T114wgjuvmMiosqF+lyiSsfp0BOCc2+ktm4DfAdOAPcdP7XjLJq97IzAqYfORwM6+/L5kjsK8MHdcUcOfvvRubr5kLM+u3sVl3/5v7n7mDXYfbPe7PJGM1OsAMLN8Mys8vg5cCbwBLATmed3mAc946wuBj3lXA00HDh4/VSRyXFl+Dv/y3sm89MXZ/OOFo/jliu1c8q8v8JnH/6o5hkRSrNfTQZvZOOL/6of4qaRfOefuM7Ny4ElgNLAduN4512Lxp4j8BzAHOAJ83Dl3xrmeNR20bN93hEeXb+XXr+6grSPC1NElfHzWWOa8o5qwZh0V6VKy00HreQCSFlrbO/ntykZ+tmwr2/YdYXhxHjfNOJsbLxpNaX6O3+WJDCoKAMlI0ZjjxfVN/HTZFpY27CMvHOC6C0byiVljqKkq9Ls8kUFBASAZb/3uQ/xs6VZ+99e36IjEuLSmgk/MGsu7JlYSCOi5xZK9FACSNVoOH+PxV7bz8+Vb2XOog5GlQxhenEduKEheOEBuKEhuKEBu4nooQG44+PayqjCXmRMqKMjVQ/Ik/SkAJOsci8T4/Ru7eHb1LtraI3REonREYt4rSntnjI7Ov7edLBw0po8r57JJw7h8UhWjy3UPgqQnBYDIGTjnOBb1wqEzxpvNbbywvokl6/bwZvNhACYMK+DyScO4/Jwqpo4u0bOOJW0oAER6aevew7ywvokX1jexYss+OqOO4iFhZtdWctmkYcyeOIzioWG/yxQ5LQWASAq0tnfy8qa9LFnXxIsbmmg5fIxgwKg7u5RPzx7P7Nph3X+JyABTAIikWDTmeL3xAEvW7eHZ1bto3H+U/3vdeXz4olHdbywygJINAJ3UFElSMGBMHV3KF6+axKLPXMrM8eV86anVPPhigx5sI2lJASDSC/m5IR6edxEfOP8s/u35DXz92bXE9JhLSTO66Fmkl3JCAb77T1MoL8jhp0u3sq/tGP9+/fnkhPTvKkkPCgCRPggEjP/1vslUFubyrec2sP/IMX740QvJ1w1lkgb0TxWRPjIzbp09gW996J0sbdjLR378F/a1dfhdlki3FAAiKfLhi0bxo5vqWL+7let/uJwdLUf8LknkjBQAIin0nslV/PKWi9nb1sGHHlrGul2H/C5J5LQUACIpdtGYMn7zqZmYwYd/tJxXtrT4XZJIlxQAIv2gtrqQpz49k8rCXD768AqeX7Pb75JETqEAEOknI0uH8ttPzeSc4UV8+pcreeKV7X6XJHICBYBIPyrLz+FXt1zMJTWV3PX03/jkL+ppaGrzuywRQAEg0u/yc0P85GN13HnFRP68aS9X3v8Sdz21mt0H2/0uTbKcJoMTGUB72zr4jxcaeGzFNgJmfHzWWD79rvGaXlpSSrOBigxi2/cd4TuLN/DM6zspygtz6+zxzJs5hrxw0O/SJAMoAETSwJqdB/nWcxt4aWMzw4vzuPOKiXxw6gg9fUz6RNNBi6SBc88q5tFPTOPx/zGdYUV5fOmp1cx54GWeX7NbU0xLvxvwIwAzmwM8AASBnzjnvnm6vjoCkGzinOP5Nbv51nMb2Lz3MFNHl3DLpeMoygsTDhqhYICcYIBQ0AgHA4S9ZShoXnu8LScYwMz83h3x0aA8BWRmQWAj8B6gEXgVuNE5t7ar/goAyUaRaIzfrGzk/sUbaWrt+aRyoYBRMjRM8ZD4q2RoDiVDwhQNCVMyNEzJkDDFQ8OUDMmheGiYorwwuaEAwYARCpi3DBAMJr43hUoaSTYABnrO2mlAg3NuM4CZPQHMBboMAJFsFAoGuHHaaK67YATrdh2iM+qIRGMci8aIRB2d0RidMUdnJEYkFqPTa4tEHceiMQ53RDhwtJODRzs5eKSTptZ2Nu5p5eDRTlrbI72uK2DEg8ELhHAoEF+ecDQSIMc7WgkFjJyEPsFAzwMkYIZZ/GlsQTMCASPgvQ+Yvb2Mr+N9flLft9e9PpawrdfH8JYWn93ViC+PtwW88DteT7xH/yoZGmbWhIp+/Y2BDoARwI6E943AxYkdzGw+MB9g9OjRA1eZyCCTFw5ywejSlH5nJBrjUHuEA0eOcfBoJweOdnLoaCedUUc0FiMSc0RjjkjUW8ZOao/Fw6gz6ojEjgeSF0CxGMcizguleJ+2jsjboRXr4dkG58ABsZgj6uK/71z82cxR54jFHDGvPZbQ7hLa0tmUUSUZFwBdxeYJ/5mccwuABRA/BTQQRYlki1AwQFl+DmX5OX6XMiCOh0fMOWIxEoLEJQTG8bCJh4Y7TVs8UAYuWPJC/X9J8EAHQCMwKuH9SGDnANcgIlkiEDACA3C6Jl0N9GWgrwI1ZjbWzHKAG4CFA1yDiIgwwEcAzrmImd0OPE/8MtBHnHNrBrIGERGJG/AnVzvnFgGLBvp3RUTkRLoTWEQkSykARESylAJARCRLKQBERLLUoJ4O2syagW19+IoKYG+Kykk32vfslc37n837Dn/f/7Odc5XddR7UAdBXZlafzIRImUj7np37Dtm9/9m879Dz/dcpIBGRLKUAEBHJUpkeAAv8LsBH2vfslc37n837Dj3c/4weAxARkdPL9CMAERE5DQWAiEiWUgCIiGQpBYCISJZSAIiIZKn/D9sSePFM9bSlAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1304b3299b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['Community'].value_counts().plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ":D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Surname</th>\n",
       "      <th>Community</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1506</th>\n",
       "      <td>anusha</td>\n",
       "      <td>venkat</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13809</th>\n",
       "      <td>anusha</td>\n",
       "      <td>nama</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Name Surname Community\n",
       "1506   anusha  venkat     tamil\n",
       "13809  anusha    nama     tamil"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.Name=='anusha']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Surname</th>\n",
       "      <th>Community</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Name, Surname, Community]\n",
       "Index: []"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.Surname=='lihala']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filtering out names that have non alphabetic characters or are less than 3 characters long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valid_name(name):\n",
    "    valid_chars=\"abcdefghijklmnopqrstuvwxyz\"\n",
    "    if(len(name)<=2):\n",
    "        return False\n",
    "    for ch in name:\n",
    "        if ch not in valid_chars:\n",
    "            return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "firstnames_mask = df['Name'].apply(valid_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Surname</th>\n",
       "      <th>Community</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>m</td>\n",
       "      <td>mouli</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>b</td>\n",
       "      <td>rao</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>k</td>\n",
       "      <td>panda</td>\n",
       "      <td>oriya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>md</td>\n",
       "      <td>alam</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>sujan.</td>\n",
       "      <td>dutta</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>rk</td>\n",
       "      <td>mishra</td>\n",
       "      <td>marwari</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>gp</td>\n",
       "      <td>mishra</td>\n",
       "      <td>marwari</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>a</td>\n",
       "      <td>.</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285</th>\n",
       "      <td>j</td>\n",
       "      <td>singh</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527</th>\n",
       "      <td>km</td>\n",
       "      <td>khader</td>\n",
       "      <td>hindi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>780</th>\n",
       "      <td>dr</td>\n",
       "      <td>gupta</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>915</th>\n",
       "      <td>mr</td>\n",
       "      <td>sandeep</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>935</th>\n",
       "      <td>bv.</td>\n",
       "      <td>chandra</td>\n",
       "      <td>marwari</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>982</th>\n",
       "      <td>dr.nikita</td>\n",
       "      <td>singh</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>991</th>\n",
       "      <td>m</td>\n",
       "      <td>rizvi</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1182</th>\n",
       "      <td>o</td>\n",
       "      <td>prakash</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1221</th>\n",
       "      <td>md</td>\n",
       "      <td>ali</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1299</th>\n",
       "      <td>ks</td>\n",
       "      <td>prakash</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313</th>\n",
       "      <td>md</td>\n",
       "      <td>pasha</td>\n",
       "      <td>urdu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1334</th>\n",
       "      <td>mr.a</td>\n",
       "      <td>raja</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1335</th>\n",
       "      <td>ma</td>\n",
       "      <td>xiaohua</td>\n",
       "      <td>marathi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1361</th>\n",
       "      <td>dr</td>\n",
       "      <td>bhatia</td>\n",
       "      <td>sindhi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1749</th>\n",
       "      <td>om</td>\n",
       "      <td>chabadia</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1821</th>\n",
       "      <td>v</td>\n",
       "      <td>venugopal</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1833</th>\n",
       "      <td>md.</td>\n",
       "      <td>akram</td>\n",
       "      <td>urdu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2118</th>\n",
       "      <td>r.p.</td>\n",
       "      <td>singh</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2211</th>\n",
       "      <td>md</td>\n",
       "      <td>ali</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2266</th>\n",
       "      <td>a</td>\n",
       "      <td>azeem</td>\n",
       "      <td>urdu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2362</th>\n",
       "      <td>md</td>\n",
       "      <td>alam</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2403</th>\n",
       "      <td>b</td>\n",
       "      <td>banerjee</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12726</th>\n",
       "      <td>c</td>\n",
       "      <td>karthikeyan</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12740</th>\n",
       "      <td>ak</td>\n",
       "      <td>ganapathy</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12896</th>\n",
       "      <td>a.kalyana</td>\n",
       "      <td>sundaram</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12917</th>\n",
       "      <td>s</td>\n",
       "      <td>mansoor</td>\n",
       "      <td>urdu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12955</th>\n",
       "      <td>n</td>\n",
       "      <td>krishna</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13122</th>\n",
       "      <td>v</td>\n",
       "      <td>karthikeyan</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13133</th>\n",
       "      <td>p</td>\n",
       "      <td>raghavan</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13192</th>\n",
       "      <td>c</td>\n",
       "      <td>basha</td>\n",
       "      <td>urdu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13202</th>\n",
       "      <td>o</td>\n",
       "      <td>yadav</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13280</th>\n",
       "      <td>r.k.</td>\n",
       "      <td>mishra</td>\n",
       "      <td>marwari</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13360</th>\n",
       "      <td>dr:</td>\n",
       "      <td>chaudhury</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13407</th>\n",
       "      <td>p.c.srivastava</td>\n",
       "      <td>srivastava</td>\n",
       "      <td>marwari</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13428</th>\n",
       "      <td>sk</td>\n",
       "      <td>ali</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13506</th>\n",
       "      <td>s</td>\n",
       "      <td>biswas</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13625</th>\n",
       "      <td>ak</td>\n",
       "      <td>singh</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13677</th>\n",
       "      <td>s</td>\n",
       "      <td>krishnamoorthy</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13808</th>\n",
       "      <td>m</td>\n",
       "      <td>nill</td>\n",
       "      <td>urdu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13875</th>\n",
       "      <td>g</td>\n",
       "      <td>dash</td>\n",
       "      <td>oriya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13895</th>\n",
       "      <td>md</td>\n",
       "      <td>khan</td>\n",
       "      <td>urdu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13902</th>\n",
       "      <td>g</td>\n",
       "      <td>rao</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13945</th>\n",
       "      <td>md</td>\n",
       "      <td>siddiqule</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13966</th>\n",
       "      <td>r</td>\n",
       "      <td>chouhan</td>\n",
       "      <td>hindi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13995</th>\n",
       "      <td>m</td>\n",
       "      <td>parmar</td>\n",
       "      <td>gujarati</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14005</th>\n",
       "      <td>g</td>\n",
       "      <td>jain</td>\n",
       "      <td>gujarati</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14087</th>\n",
       "      <td>b.j</td>\n",
       "      <td>singh</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14422</th>\n",
       "      <td>b.c.</td>\n",
       "      <td>srivastava</td>\n",
       "      <td>marwari</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14597</th>\n",
       "      <td>n</td>\n",
       "      <td>.</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14689</th>\n",
       "      <td>k</td>\n",
       "      <td>singh</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14808</th>\n",
       "      <td>a</td>\n",
       "      <td>prabhu</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14957</th>\n",
       "      <td>m</td>\n",
       "      <td>manivannan</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>201 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Name         Surname Community\n",
       "25                  m           mouli    telugu\n",
       "125                 b             rao   kannada\n",
       "160                 k           panda     oriya\n",
       "191                md            alam   bengali\n",
       "198            sujan.           dutta   bengali\n",
       "230                rk          mishra   marwari\n",
       "250                gp          mishra   marwari\n",
       "254                 a               .     tamil\n",
       "285                 j           singh   punjabi\n",
       "527                km          khader     hindi\n",
       "780                dr           gupta   bengali\n",
       "915                mr         sandeep    telugu\n",
       "935               bv.         chandra   marwari\n",
       "982         dr.nikita           singh   punjabi\n",
       "991                 m           rizvi   bengali\n",
       "1182                o         prakash   kannada\n",
       "1221               md             ali   bengali\n",
       "1299               ks         prakash    telugu\n",
       "1313               md           pasha      urdu\n",
       "1334             mr.a            raja    telugu\n",
       "1335               ma         xiaohua   marathi\n",
       "1361               dr          bhatia    sindhi\n",
       "1749               om        chabadia     tamil\n",
       "1821                v       venugopal   kannada\n",
       "1833              md.           akram      urdu\n",
       "2118             r.p.           singh   punjabi\n",
       "2211               md             ali   bengali\n",
       "2266                a           azeem      urdu\n",
       "2362               md            alam   bengali\n",
       "2403                b        banerjee   bengali\n",
       "...               ...             ...       ...\n",
       "12726               c     karthikeyan     tamil\n",
       "12740              ak       ganapathy     tamil\n",
       "12896       a.kalyana        sundaram     tamil\n",
       "12917               s         mansoor      urdu\n",
       "12955               n         krishna    telugu\n",
       "13122               v     karthikeyan     tamil\n",
       "13133               p        raghavan    telugu\n",
       "13192               c           basha      urdu\n",
       "13202               o           yadav   kannada\n",
       "13280            r.k.          mishra   marwari\n",
       "13360             dr:       chaudhury   punjabi\n",
       "13407  p.c.srivastava      srivastava   marwari\n",
       "13428              sk             ali   bengali\n",
       "13506               s          biswas   bengali\n",
       "13625              ak           singh   punjabi\n",
       "13677               s  krishnamoorthy    telugu\n",
       "13808               m            nill      urdu\n",
       "13875               g            dash     oriya\n",
       "13895              md            khan      urdu\n",
       "13902               g             rao   kannada\n",
       "13945              md       siddiqule   bengali\n",
       "13966               r         chouhan     hindi\n",
       "13995               m          parmar  gujarati\n",
       "14005               g            jain  gujarati\n",
       "14087             b.j           singh   punjabi\n",
       "14422            b.c.      srivastava   marwari\n",
       "14597               n               .     tamil\n",
       "14689               k           singh   punjabi\n",
       "14808               a          prabhu   kannada\n",
       "14957               m      manivannan    telugu\n",
       "\n",
       "[201 rows x 3 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking at first names that will be filtered out\n",
    "df[[not x for x in firstnames_mask]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "surnames_mask = df['Surname'].apply(valid_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Surname</th>\n",
       "      <th>Community</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>jyothi</td>\n",
       "      <td>m</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>chetan</td>\n",
       "      <td>c</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>a</td>\n",
       "      <td>.</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>moksha</td>\n",
       "      <td>m</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>pallavi</td>\n",
       "      <td>p</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>859</th>\n",
       "      <td>anoop</td>\n",
       "      <td>g</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1504</th>\n",
       "      <td>hemanth</td>\n",
       "      <td>b</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1682</th>\n",
       "      <td>viswanath</td>\n",
       "      <td>g</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1895</th>\n",
       "      <td>lokesh</td>\n",
       "      <td>j</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2217</th>\n",
       "      <td>nagaraj</td>\n",
       "      <td>k</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2469</th>\n",
       "      <td>karthik</td>\n",
       "      <td>t</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2713</th>\n",
       "      <td>nagesh</td>\n",
       "      <td>n</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2814</th>\n",
       "      <td>manjunath</td>\n",
       "      <td>g</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2969</th>\n",
       "      <td>bhuvana</td>\n",
       "      <td>b</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3050</th>\n",
       "      <td>karthik</td>\n",
       "      <td>g</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3051</th>\n",
       "      <td>janaki</td>\n",
       "      <td>.</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3245</th>\n",
       "      <td>vp</td>\n",
       "      <td>.</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3272</th>\n",
       "      <td>dorine</td>\n",
       "      <td>de</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3478</th>\n",
       "      <td>arnab</td>\n",
       "      <td>de</td>\n",
       "      <td>bengali</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3634</th>\n",
       "      <td>farauk</td>\n",
       "      <td>exchange)</td>\n",
       "      <td>urdu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3742</th>\n",
       "      <td>vinaya</td>\n",
       "      <td>s</td>\n",
       "      <td>hindi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3797</th>\n",
       "      <td>nagesh</td>\n",
       "      <td>k</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3956</th>\n",
       "      <td>manjunath</td>\n",
       "      <td>d</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3967</th>\n",
       "      <td>smitha</td>\n",
       "      <td>p</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4093</th>\n",
       "      <td>ravi</td>\n",
       "      <td>g</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4191</th>\n",
       "      <td>santosh</td>\n",
       "      <td>vv</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4283</th>\n",
       "      <td>padma</td>\n",
       "      <td>c</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4443</th>\n",
       "      <td>raghunandan</td>\n",
       "      <td>k</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4453</th>\n",
       "      <td>shilpa</td>\n",
       "      <td>g</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4517</th>\n",
       "      <td>prabhakaran</td>\n",
       "      <td>b</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9695</th>\n",
       "      <td>manjunath</td>\n",
       "      <td>j</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9755</th>\n",
       "      <td>sajan</td>\n",
       "      <td>s</td>\n",
       "      <td>malayalam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9826</th>\n",
       "      <td>thahira</td>\n",
       "      <td>.</td>\n",
       "      <td>punjabi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9855</th>\n",
       "      <td>bharath</td>\n",
       "      <td>g</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9953</th>\n",
       "      <td>madhavi</td>\n",
       "      <td>g</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9954</th>\n",
       "      <td>bhaskar</td>\n",
       "      <td>de.</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10213</th>\n",
       "      <td>shivakumar</td>\n",
       "      <td>b</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10268</th>\n",
       "      <td>kumari</td>\n",
       "      <td>b</td>\n",
       "      <td>telugu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10586</th>\n",
       "      <td>karthik</td>\n",
       "      <td>m</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10773</th>\n",
       "      <td>manjunath</td>\n",
       "      <td>s</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10851</th>\n",
       "      <td>pradeep</td>\n",
       "      <td>bv</td>\n",
       "      <td>malayalam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11118</th>\n",
       "      <td>nithin</td>\n",
       "      <td>.</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11403</th>\n",
       "      <td>shiva</td>\n",
       "      <td>v</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11623</th>\n",
       "      <td>jithesh</td>\n",
       "      <td>at</td>\n",
       "      <td>malayalam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11637</th>\n",
       "      <td>ashray</td>\n",
       "      <td>n</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11661</th>\n",
       "      <td>prakash</td>\n",
       "      <td>m</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12631</th>\n",
       "      <td>aruna</td>\n",
       "      <td>r</td>\n",
       "      <td>english</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12784</th>\n",
       "      <td>venugopal</td>\n",
       "      <td>p</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13491</th>\n",
       "      <td>chandrasekar</td>\n",
       "      <td>b</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13512</th>\n",
       "      <td>riaz</td>\n",
       "      <td>9880279904</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13679</th>\n",
       "      <td>shivakumar</td>\n",
       "      <td>n</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13751</th>\n",
       "      <td>kannan</td>\n",
       "      <td>.</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13968</th>\n",
       "      <td>disha</td>\n",
       "      <td>s</td>\n",
       "      <td>hindi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14152</th>\n",
       "      <td>karthik</td>\n",
       "      <td>l</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14170</th>\n",
       "      <td>soumitra</td>\n",
       "      <td>.das</td>\n",
       "      <td>marathi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14295</th>\n",
       "      <td>prakash</td>\n",
       "      <td>p</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14355</th>\n",
       "      <td>greeshma</td>\n",
       "      <td>v</td>\n",
       "      <td>english</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14597</th>\n",
       "      <td>n</td>\n",
       "      <td>.</td>\n",
       "      <td>tamil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14838</th>\n",
       "      <td>venkatesh</td>\n",
       "      <td>g</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14936</th>\n",
       "      <td>manjunatha</td>\n",
       "      <td>g</td>\n",
       "      <td>kannada</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Name     Surname  Community\n",
       "53           jyothi           m    kannada\n",
       "209          chetan           c    kannada\n",
       "254               a           .      tamil\n",
       "304          moksha           m    kannada\n",
       "492         pallavi           p    kannada\n",
       "859           anoop           g    kannada\n",
       "1504        hemanth           b    kannada\n",
       "1682      viswanath           g     telugu\n",
       "1895         lokesh           j    kannada\n",
       "2217        nagaraj           k      tamil\n",
       "2469        karthik           t    kannada\n",
       "2713         nagesh           n     telugu\n",
       "2814      manjunath           g    kannada\n",
       "2969        bhuvana           b      tamil\n",
       "3050        karthik           g    kannada\n",
       "3051         janaki           .      tamil\n",
       "3245             vp           .    punjabi\n",
       "3272         dorine          de    bengali\n",
       "3478          arnab          de    bengali\n",
       "3634         farauk   exchange)       urdu\n",
       "3742         vinaya           s      hindi\n",
       "3797         nagesh           k     telugu\n",
       "3956      manjunath           d    kannada\n",
       "3967         smitha           p     telugu\n",
       "4093           ravi           g    kannada\n",
       "4191        santosh          vv     telugu\n",
       "4283          padma           c     telugu\n",
       "4443    raghunandan           k    kannada\n",
       "4453         shilpa           g    kannada\n",
       "4517    prabhakaran           b      tamil\n",
       "...             ...         ...        ...\n",
       "9695      manjunath           j    kannada\n",
       "9755          sajan           s  malayalam\n",
       "9826        thahira           .    punjabi\n",
       "9855        bharath           g    kannada\n",
       "9953        madhavi           g     telugu\n",
       "9954        bhaskar         de.     telugu\n",
       "10213    shivakumar           b    kannada\n",
       "10268        kumari           b     telugu\n",
       "10586       karthik           m    kannada\n",
       "10773     manjunath           s    kannada\n",
       "10851       pradeep          bv  malayalam\n",
       "11118        nithin           .      tamil\n",
       "11403         shiva           v    kannada\n",
       "11623       jithesh          at  malayalam\n",
       "11637        ashray           n    kannada\n",
       "11661       prakash           m    kannada\n",
       "12631         aruna           r    english\n",
       "12784     venugopal           p    kannada\n",
       "13491  chandrasekar           b      tamil\n",
       "13512          riaz  9880279904      tamil\n",
       "13679    shivakumar           n    kannada\n",
       "13751        kannan           .      tamil\n",
       "13968         disha           s      hindi\n",
       "14152       karthik           l    kannada\n",
       "14170      soumitra        .das    marathi\n",
       "14295       prakash           p    kannada\n",
       "14355      greeshma           v    english\n",
       "14597             n           .      tamil\n",
       "14838     venkatesh           g    kannada\n",
       "14936    manjunatha           g    kannada\n",
       "\n",
       "[92 rows x 3 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Looking at surnames that will be filtered out\n",
    "df[[not x for x in surnames_mask]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "fullnames_mask = [fn and sn for fn,sn in zip(firstnames_mask,surnames_mask)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploring n-grams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exploring name bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ngrams(words, n):\n",
    "    return [word[i:i+n]  for word in words for i in range(len(word)-n+1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mask_indices(mask):\n",
    "    return np.argwhere(mask).reshape(1,-1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "firstname_bigrams = get_ngrams(df['Name'][firstnames_mask], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sh    3661\n",
       "an    3638\n",
       "ra    2906\n",
       "ha    2417\n",
       "ar    1801\n",
       "ma    1518\n",
       "na    1375\n",
       "it    1350\n",
       "ee    1333\n",
       "as    1172\n",
       "am    1165\n",
       "nd    1157\n",
       "in    1150\n",
       "sa    1137\n",
       "hi    1114\n",
       "ta    1112\n",
       "ka    1087\n",
       "ni    1069\n",
       "al    1001\n",
       "vi     989\n",
       "es     986\n",
       "aj     901\n",
       "su     899\n",
       "ri     886\n",
       "ja     842\n",
       "is     810\n",
       "pr     796\n",
       "ya     769\n",
       "en     757\n",
       "de     744\n",
       "      ... \n",
       "wk       1\n",
       "jl       1\n",
       "zs       1\n",
       "jb       1\n",
       "kn       1\n",
       "wq       1\n",
       "xa       1\n",
       "vg       1\n",
       "hg       1\n",
       "cq       1\n",
       "fs       1\n",
       "dl       1\n",
       "hz       1\n",
       "vm       1\n",
       "vc       1\n",
       "mz       1\n",
       "cs       1\n",
       "eu       1\n",
       "bw       1\n",
       "qs       1\n",
       "gc       1\n",
       "cv       1\n",
       "fz       1\n",
       "rq       1\n",
       "gp       1\n",
       "ci       1\n",
       "gg       1\n",
       "uu       1\n",
       "vp       1\n",
       "zy       1\n",
       "Length: 484, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(firstname_bigrams).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEQCAYAAACnaJNPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8XXWd//HXJ9vN2qTpmi60xZaWlqU0YQeRgi3g0nEGFXQUBQd/IzgqMyiMIgIyLjMO4oaggCCogIK0pcMiq4VS2pTuC22hha7pmqRNs39+f5yT9rYkzb03ublZ3s/H4zxy7/eczzmfJPfezz3fs3zN3REREYlVWqoTEBGRnkWFQ0RE4qLCISIicVHhEBGRuKhwiIhIXFQ4REQkLiocIiISFxUOERGJiwqHiIjERYVDRETikpHqBJJh4MCBPnr06ITjDxw4QE5OjuIVr3jF96n48vLyne4+qN0F3b3XTaWlpd4RCxcuVLziFa/4PhcPLPQYPmPVVSUiInFR4RARkbiocIiISFxUOEREJC4qHCIiEhcVDhERiUuvvI4jUet37OOxhZtoqNzPjshWhvTLZmhhNoPyI2Skq8aKiIAKx2FWbqni1y+vB+DexYsOtqcZDCqIMLRf9sFiMqRfNkP7ZVNSmM2QwuBxXkR/ThHp/fRJF2X80AL+Y9pxLFv/Hk1ZBWyrqmVbZR279texvSqYoLLN+IJIBiVF2cw4NoPS0q7LW0SkK6lwRDluSAHHDSmgvLCK0qhP/oamZiqq69hWWRtMVbVsr3r/4+q6Rqq37+PR2gyumZHCX0REJIlUOGKQmZ7G8KIchhe1ff8Xd2fX/nrO+/GLbKxsZPPeA0ddXkSkp9IR305iZgzMj3DuuOD+YC+srkhxRiIiyaHC0cmmHj8YgBdWbU9xJiIiyaHC0ck+ND7Y43ht/S4O1DelOBsRkc6nwtHJBhdkM7Z/JnWNzby2fmeq0xER6XRJKxxmlm1mb5jZEjNbYWa3hO2/M7N3zGxxOE0O283MfmZm68xsqZlNiVrXFWa2NpyuSFbOnaV0WASA53WcQ0R6oWSeVVUHTHX3fWaWCcw1s/8L513v7n8+YvmLgXHhdDpwF3C6mRUDNwNlgAPlZjbT3fckMfcOKSuJ8MiKfby4ugJ3x8xSnZKISKdJ2h5HOKDUvvBpZjj5UUJmAA+Gca8DRWZWAkwHnnP33WGxeA64KFl5d4YxRRkMLoiwtbKWVVurU52OiEinSuoxDjNLN7PFQAXBh//8cNbtYXfUHWYWCduGA+9FhW8K29pq77bMjKkTwrOrVuvsKhHpXSwYZjbJGzErAp4AvgrsArYBWcA9wHp3v9XMngJ+4O5zw5jngW8CU4GIu38/bL8JqHH3nxyxjauBqwFKSkpKZ82alXC+NTU15Obmdih++Z40fvTaXo4rzuQHFwzo8u0rXvGKV3y8ysrKyt29rN0FYxmYvDMmguMU/3FE24eA2eHju4HLo+atAUqAy4G7o9oPW661qbS0NNGx2t29cwaL31fb4OO+PcdH3zDbd1bXdvn2Fa94xSs+XsBCj+HzPJlnVQ0K9zQwsxzgQmB1eNwCC44Y/wOwPAyZCXw+PLvqDKDS3bcCzwDTzKy/mfUHpoVt3VpeJIMzjh2AO7y0Zkeq0xER6TTJPMZRArxoZkuBBQTHOGYDD5vZMmAZMBD4frj8HOBtYB3wG+ArAO6+G7gtXMcC4Nawrdu74OBxDp2WKyK9R9JOx3X3pcAprbRPbWN5B65pY959wH2dmmAXmDphMDfPXMErb+2goamZTA0GJSK9gD7JkmhkcS7jBudTXdfIgg09YidJRKRdKhxJduimh+quEpHeQYUjyS6YMATQcQ4R6T1UOJJsyjFFFOZk8vbO/byzc3+q0xER6TAVjiTLSE/jvOM0uJOI9B4qHF3gguN1+xER6T1UOLrAeccNIs1g/tu7qa5tSHU6IiIdosLRBYpysygd1Z/GZufvazW4k4j0bCocXWSqzq4SkV5ChaOLtBzneHF1Bc3Nyb8jsYhIsqhwdJFxg/MZ0T+HXfvrWbJpb6rTERFJmApHFzl8cCd1V4lIz6XC0YVUOESkN1Dh6EJnHDuAnMx0VmypYltlbarTERFJiApHF8rOTOfssQMB7XWISM+lwtHFDl1FrsIhIj2TCkcXO398UDheXbeT2oamFGcjIhI/FY4uNrQwmxOG9+NAQxPz3t6V6nREROKmwpECU8drcCcR6blUOFJg6vGHbj8SDLUuItJzqHCkwEnDCxmYn8XmvQd4a/u+VKcjIhIXFY4USEszPhR2Vz2vMTpEpIdR4UiRCyboOIeI9EwqHClyzriBZKYbi97dw5799alOR0QkZiocKVKQncnpYwbQ7PDyWztSnY6ISMxUOFLo/AktxznUXSUiPYcKRwq1HOd4eU0FjU3NKc5GRCQ2KhwpNHpgHscOyqOqtpHyjXtSnY6ISEySVjjMLNvM3jCzJWa2wsxuCdvHmNl8M1trZo+YWVbYHgmfrwvnj45a141h+xozm56snFPh4FXk6q4SkR4imXscdcBUdz8ZmAxcZGZnAD8C7nD3ccAe4Kpw+auAPe4+FrgjXA4zmwhcBkwCLgJ+ZWbpScy7S009Xsc5RKRnSVrh8EDLZdGZ4eTAVODPYfsDwD+Ej2eEzwnnX2BmFrb/yd3r3P0dYB1wWrLy7mqnji6mIJLBuop9vLurJtXpiIi0K6nHOMws3cwWAxXAc8B6YK+7N4aLbAKGh4+HA+8BhPMrgQHR7a3E9HiZ6Wl8cPwgAF7QVeQi0gNYV9xkz8yKgCeA7wL3h91RmNlIYI67n2hmK4Dp7r4pnLeeYM/iVmCeuz8Utt8bxvzliG1cDVwNUFJSUjpr1qyE862pqSE3N7fL4l/acICfL6jk5CFZfPeDxV2+fcUrXvGKBygrKyt397J2F3T3LpmAm4HrgZ1ARth2JvBM+PgZ4MzwcUa4nAE3AjdGrefgcm1NpaWl3hELFy7s0vid1bU++obZPu4/53h1bUOXb1/xile84t3dgYUew+d5Ms+qGhTuaWBmOcCFwCrgReDScLErgCfDxzPD54TzXwh/kZnAZeFZV2OAccAbyco7FQbkRzhlZBH1Tc3MXbsz1emIiBxVMo9xlAAvmtlSYAHwnLvPBr4FXGdm6wiOYdwbLn8vMCBsvw64AcDdVwCPAiuBp4Fr3L3Xjbk6NbwY8EWdXSUi3VxGslbs7kuBU1ppf5tWzopy91rgk22s63bg9s7OsTuZOmEI//PsW7ywpoJLRxelOh0RkTbpyvFu4viSAkoKs9lRXcfbexrbDxARSREVjm7CzA52Vy3aWpfibERE2qbC0Y1cGI5F/vLGAzQ1ayxyEemeVDi6kQ8eN4iRxTls29/E86t0MaCIdE8qHN1IeprxhbPGAPDbue+kOBsRkdapcHQznyobQU6G8cY7u1m+uTLV6YiIvI8KRzdTkJ3JhcfmAHCv9jpEpBtS4eiGLhmbS5rBrCVb2FZZm+p0REQOo8LRDQ3Oy+DiE0pobHYenLch1emIiBxGhaObuvKc4CD5w/PfpaZeFwSKSPehwtFNlY7qz+SRRVQeaOAvizanOh0RkYNUOLqxL50b7HXcP/cdmnVBoIh0Eyoc3dhFk4YyvCiHt3fu58U1umuuiHQPKhzdWEZ6GlecNQrQqbki0n2ocHRznz71GPKy0nlt/S5WbqlKdToiIioc3V1hTiafLBsJaK9DRLoHFY4e4Itnj8bCCwIrqnVBoIiklgpHDzBqQB7TJg6hvqmZh+ZtTHU6ItLHqXD0EFedcywAD81/l9qGXjfkuoj0ICocPcSpo/tz0ohCdu+v54k3dUGgiKSOCkcPYWZcFd6G5N657+CuCwJFJDVUOHqQS04sYWi/bNZV7OPlt3akOh0R6aNUOHqQzPQ0rjhrNKBTc0UkdVQ4epjPnHYMOZnp/H3tTtZsq051OiLSB6lw9DCFuZlcWjoCgPu01yEiKaDC0QO1XBD4xOLN7NxXl+p0RKSPUeHogY4dlM8FEwZT39jMQ6/rgkAR6VoqHD1UywiBD72+URcEikiXSlrhMLORZvaima0ysxVm9rWw/XtmttnMFofTJVExN5rZOjNbY2bTo9ovCtvWmdkNycq5Jznz2AFMLOnHzn31zFyyJdXpiEgfksw9jkbg3939eOAM4BozmxjOu8PdJ4fTHIBw3mXAJOAi4Fdmlm5m6cAvgYuBicDlUevps6IvCLxPFwSKSBeKu3CYWX8zO6m95dx9q7svCh9XA6uA4UcJmQH8yd3r3P0dYB1wWjitc/e33b0e+FO4bJ/3sZOHMbggwupt1by6bleq0xGRPiKmwmFmL5lZPzMrBpYA95vZ/8a6ETMbDZwCzA+brjWzpWZ2n5n1D9uGA+9FhW0K29pq7/OyMtL4/JktIwS+neJsRKSvsFi6OMzsTXc/xcy+BIx095vNbKm7t7vnYWb5wMvA7e7+uJkNAXYCDtwGlLj7lWb2S2Ceuz8Uxt0LzCEobtPd/Uth++eA09z9q0ds52rgaoCSkpLSWbNmxfo3eJ+amhpyc3N7RHxVXTNfnl1BfTPcOX0gI/pl9Kj8Fa94xXef+LKysnJ3L2t3QXdvdwKWASXAs8CpYdvSGOIygWeA69qYPxpYHj6+Ebgxat4zwJnh9ExU+2HLtTaVlpZ6RyxcuLBHxd/4+FIf9a3ZfuPjS1OyfcUrXvG9Ix5Y6DHUhFiPcdwSfpCvc/cFZnYssPZoAWZmwL3AKnf/36j2kqjFPgEsDx/PBC4zs4iZjQHGAW8AC4BxZjbGzLIIDqDPjDHvPuHKs4OD5I8v2sTu/fUpzkZEeruMGJfb6lHdUu7+dgzHOM4GPgcsM7PFYdt/EpwVNZmgq2oD8OVwnSvM7FFgJcEZWde4exOAmV1LULjSgfvcfUWMefcJYwfn86Hxg3hpzQ7+MH8jZxamOiMR6c1iLRw/B6bE0HaQu88FrJVZc44Scztweyvtc44WJ/Clc47lpTU7eHDeRsqmFaU6HRHpxY5aOMzsTOAsYJCZXRc1qx/Bt3/pJs4eO4AJQwuCU3Pfq+WMU1OdkYj0Vu0d48gC8gkKTEHUVAVcmtzUJB5mdvBYxzPralKcjYj0Zkfd43D3l4GXzex37q676XVzHz25hO/NWsFbuxt4d1cNxwxI/JQ+EZG2xHpWVcTM7jGzZ83shZYpqZlJ3HKzMpg2cQgAs5bq/lUikhyxFo7HgDeB7wDXR03SzcyYHFxU/9c3N+v+VSKSFLGeVdXo7nclNRPpFOeMG0hBlrG2Yh+rt1VzfEm/VKckIr1MrHscs8zsK2ZWYmbFLVNSM5OEZKancdbIbACeXKzuKhHpfLEWjisIuqZeA8rDaWGykpKOOfeYHABmLdlCc7O6q0Skc8XUVeXuY5KdiHSe8QMyGV6Uw+a9Byh/dw+njtbOoYh0npgKh5l9vrV2d3+wc9ORzpBmxsdOHsavX17Pk4s3q3CISKeKtavq1KjpXOB7wMeTlJN0ghmThwHw1NKtNDQ1pzgbEelNYu2qOnLsi0Lg90nJSDrFhKEFHDckn7e272Pu2p2cP2FwqlMSkV4i0THHawhuey7dlJkdvKbjycWbU5yNiPQmsQ4dO8vMZobTU8Aa4MnkpiYd9fGTg+6qZ1dup6a+McXZiEhvEesFgP8T9bgR2Ojum5KQj3SikcW5TDmmiEXv7uVvqyoOFhIRkY6IaY8jvNnhaoI74/YHNMxcD9HSXTVT3VUi0kli7ar6FMEwrp8EPgXMNzPdVr0HuOTEEtLTjJfW7GCPhpUVkU4Q68HxbwOnuvsV7v554DTgpuSlJZ1lUEGEs8cOpLHZ+b/l21Kdjoj0ArEWjjR3r4h6viuOWEmxGeGxDZ1dJSKdIdYP/6fN7Bkz+4KZfQF4Co0B3mNMmzSESEYab2zYzZa9B1Kdjoj0cEctHGY21szOdvfrgbuBk4CTgXnAPV2Qn3SCguxMLjx+CO4wWwM8iUgHtbfH8VOgGsDdH3f369z9GwR7Gz9NdnLSeT4+uaW7SoVDRDqmvcIx2t2XHtno7guB0UnJSJLiQ+MHUZCdwYotVayrqE51OiLSg7VXOLKPMi+nMxOR5IpkpHPJCSUAzNReh4h0QHuFY4GZ/cuRjWZ2FcFgTtKDtNwx98klWzQeuYgkrL1bjnwdeMLMPsuhQlEGZAGfSGZi0vlOP3YAgwsibNxVw5JNlUweWZTqlESkBzrqHoe7b3f3s4BbgA3hdIu7n+nuupqsh0lPCwZ4Al3TISKJi/VeVS+6+8/D6YVkJyXJ09JdNWvJVpo0HrmIJCBpV3+b2Ugze9HMVpnZCjP7WthebGbPmdna8Gf/sN3M7Gdmts7MlprZlKh1XREuv9bMrkhWzn3BicMLGTMwj5376pi3fleq0xGRHiiZtw1pBP7d3Y8HzgCuMbOJwA3A8+4+Dng+fA5wMcHgUOOAq4G7ICg0wM3A6QT3yLq5pdhI/MzUXSUiHZO0wuHuW919Ufi4GlgFDAdmAA+Eiz0A/EP4eAbwoAdeB4rMrASYDjzn7rvdfQ/wHHBRsvLuC1rG5Xh6+TZqG5pSnI2I9DRdcqNCMxsNnALMB4a4+1YIigvQMhj2cOC9qLBNYVtb7ZKgsYPzOWF4P6rrGnlpTUX7ASIiUSzZ5/ObWT7wMnC7uz9uZnvdvShq/h537x8OSfsDd58btj8PfBOYCkTc/fth+01Ajbv/5IjtXE3QxUVJSUnprFmzEs65pqaG3NzcXh0/c81+HlhazRnDI1x/1uE9fz0hf8UrXvGdH19WVlbu7mXtLujuSZuATOAZ4LqotjVASfi4BFgTPr4buPzI5YDLgbuj2g9brrWptLTUO2LhwoW9Pn7L3hoffcNsH/ftOV55oL7Lt694xSu++8UDCz2Gz/ZknlVlwL3AKnf/36hZM4GWM6OuAJ6Mav98eHbVGUClB11ZzwDTzKx/eFB8WtgmHVBSmMPpY4qpb2zmGQ3wJCJxSOYxjrOBzwFTzWxxOF0C/BD4sJmtBT4cPofgjrtvA+uA3wBfAXD33cBtwIJwujVskw46OB75Et27SkRi194tRxLmwbEKa2P2Ba0s78A1bazrPuC+zstOAC4+YSjffXI5r67bSUV1LYMLjnZPSxGRgIZ/7cOKcrM477jBNDs8tXRrqtMRkR5ChaOPm6EBnkQkTiocfdyFxw8hNyudxe/tZeOu/alOR0R6ABWOPi4nK53pk4YCGuBJRGKjwiEHxyP/6+LNGuBJRNqlwiGcM3YgxXlZrN+xn5Vbq1Kdjoh0cyocQmZ6Gh85UeORi0hsVDgEOHR21cwlW2hWd5WIHIUKhwAw5Zj+DC/KYWtlLat2NqQ6HRHpxlQ4BIC0NDt4kPyFd2pSnI2IdGcqHHLQP00ZQUaa8dLGWp5erivJRaR1Khxy0NjB+dxw8QQArn9sKRt26oJAEXk/FQ45zFXnjOH04RGq6xr514cXaWhZEXkfFQ45jJlxzamFjBqQy6qtVdz85IpUpyQi3YwKh7xPXmYav/rsFCIZaTyy8D0eXfhe+0Ei0meocEirJg0r5LYZJwBw01+Xs3KLrigXkYAKh7TpU6eO5JOlI6hrbOaaPyyiulbXd4iICoe049YZJzBhaAHv7NzPt/6yVDdBFBEVDjm6nKx07vrnUvIjGcxZto37X92Q6pREJMVUOKRdYwbm8d+XngTAf81ZRfnGPSnOSERSSYVDYnLxiSVcefYYGpuda/+wiF376lKdkoikiAqHxOyGiycw5ZgitlbW8vVHFtPUrOMdIn2RCofELCsjjV98ZgrFeVn8fe1Ofv7C2lSnJCIpoMIhcRlWlMNPPz0ZM7jz+bW88taOVKckIl1MhUPi9sHjBvG1C8bhDl9/ZDFbKw+kOiUR6UIqHJKQr04dx7njBrJ7fz3XPLyIhqbmVKckIl1EhUMSkp5m/PTTkykpzGbRu3v54f+tTnVKItJFVDgkYQPyI/ziM1PISDPunfsO/7dMgz+J9AVJKxxmdp+ZVZjZ8qi275nZZjNbHE6XRM270czWmdkaM5se1X5R2LbOzG5IVr6SmNJR/bnxkuMBuP7PS3lHgz+J9HrJ3OP4HXBRK+13uPvkcJoDYGYTgcuASWHMr8ws3czSgV8CFwMTgcvDZaUbufLs0Vx8wlD21TXyrw+VU9eo6ztEerOkFQ53fwXYHePiM4A/uXudu78DrANOC6d17v62u9cDfwqXlW7EzPjxpScxZmAeq7dV89/z9lBRVZvqtEQkSVJxjONaM1sadmX1D9uGA9GjBW0K29pql26mIDuTX312CgWRDN7cVs+H73iFxxdt0t10RXohS+Yb28xGA7Pd/YTw+RBgJ+DAbUCJu19pZr8E5rn7Q+Fy9wJzCArbdHf/Utj+OeA0d/9qK9u6GrgaoKSkpHTWrFkJ511TU0Nubq7iE7CjpolfvrGbZTuCscpLSyJ8ubQfA3LSu2T7ile84hOPLysrK3f3snYXdPekTcBoYHl784AbgRuj5j0DnBlOz0S1H7ZcW1Npaal3xMKFCxXfAQsWLPBHFrzrJ9z8tI/61mw/4ean/ZE33vXm5uYu2b7iFa/4xAALPYbP9i7tqjKzkqinnwBazriaCVxmZhEzGwOMA94AFgDjzGyMmWURHECf2ZU5S/zMjE+VjeS5b5zH1AmDqa5t5Jt/WcoV9y9g815dZS7S0yXzdNw/AvOA8Wa2ycyuAn5sZsvMbClwPvANAHdfATwKrASeBq5x9yZ3bwSuJdgDWQU8Gi4rPcDQwmzuvaKMOz59MoU5mbzy1g6m3/EKD8/fqGMfIj1YRrJW7O6Xt9J871GWvx24vZX2OQTHO6QHMjM+ccoIzh47kO88sZxnV27n208s56mlW/nRP53EyOLE+3JFJDV05bh0icEF2dz9uVJ+dvkp9M/N5LX1u5j+01d4cN4GmjWuh0iPosIhXcbM+PjJw3juuvP4yIkl1NQ38d0nV3D5b15ng644F+kxVDikyw3Mj/DLz07hrs9OYWB+FvPf2c1Fd77CvXPf0aiCIj1A0o5xiLTn4hNLOP3YAdwyawVPLt7CbbNXMmfZVqYOc7KGVFJSlM2AvCzMLNWpikgUFQ5JqeK8LO687BQ+etIwvv3EMso37qF8I/z3vLkARDLSKCnMZlhRTjCFj0uKchhelE1JYQ55Eb2MRbqS3nHSLXx44hBOG13M3a+sZ/6aTez3LLZW1lJ5oIENu2rYsKumzdjCnExKCrMZXpRDSVE2jdX7WNmwkeLcLPrnZVKcl0VxbhZFuVlkZah3VqSjVDik2yjMzeSbF02gfNB+SktLAdhX18jWvQfYUlnLlr0H2Lr3AJv31rK18gBbwvbKAw1UHmhg9bbqQytbsbzVbRREMuifl0X/vCyKczPDn+HzvCz652YRqdNohiJHo8Ih3Vp+JINxQwoYN6Sg1fnuzq799UER2RsUlxXrN5JVMIA9++vZXVPPnv317KmpZ09NA9V1jVTXNfLu7rb3YAqyjJ8VV3D+hMHJ+rVEejQVDunRzIyB+REG5kc4aUTQVp6zm9LSE9+3bHOzU13byO6aenbvrz+ssLT8XLOtmiWbKvni7xbw5Q8ey39MH09murq3RKKpcEifkZZmFOZmUpibyZiBea0u09zs3PTHV/jTiv3c/crbvLFhNz+//BRG9NcV7iIt9FVKJEpamvGPE/J55OozKCnM5s1393LJnX/nmRXbUp2aSLehwiHSirLRxcz5t3O58PjBVNU28uXfl3PLrBXUNTalOjWRlFPhEGlD/7wsfvP5Mr7zkePJTDfuf3UDl941j427dHsU6dtUOESOwsz40rnH8tj/O4sR/XNYtrmSj/xsLrOXbkl1aiIpo8IhEoPJI4t46t/O5aJJQ9lX18i1f3iTbz+xjNoGdV1J36PCIRKjwpxM7vrnKdw2YxJZ6Wk8PP9d/uGXr7J+x75UpybSpVQ4ROJgZnzuzNE8/pWzGD0gl9XbqvnYz+fyxJubUp2aSJdR4RBJwAnDC5n9b+fy8ZOHUVPfxDceWcL1jy2hpr4x1amJJJ0uABRJUH4kgzsvm8xZHxjAzTNX8Fj5Jha/t5eLRqezI7KVSGY6kYw0IhnBz+yW55lBW3ZmGlnpabptvPQ4KhwiHWBmXHbaMUw+pohr//Amayv2sbYCeGNRzOs4sqh4YwP9X5tLTmY6kcw0cjLTyc4MCs2hx8GUk5l22PPszDTe3VFP+nt7w6KVRiQznezwZyQjTbdQkQ5T4RDpBBOG9mPmtWfz65ffZtFb75FbUEhdYzN1jU3UNTZT2xA+bmgO2huC9vqm8Hnj4Xfk3VRV2bGEXnq1zVnpaXaoqIR7PpGMoEhlZ6TTULuPEW+9SUF2Bv2yM8OfGfTLCR4XZGcebC/IziAvK4O0NO019SUqHCKdJDcrg+s+fBzlxdUHbwvfnuZmD4pHQzO1jU3UNjSxaMkyjh03gdqGJg40NFHb0ExtQ9Nhzw80NFF38HkTB6KW2bmnkqzs3EPFKixMLfObmp2a+iZq6puAhlbzenNb7NeppFnQbVeQnUm/nEwym2oZt34JQwsjDO2XzdDCHIb2y2ZIYYSBeREVmV5AhUMkhdLSjOy0oJupkEwAdhZmcvLIooTXWV5eftTC1Ri1l1Pb0HRozygsPktWrmbIiNFUHWigqraR6tpGqmobgp8HGqhueRz+rKlvoqq2karaRjbvPQDA0orWzzLLSDMGF0QYWpjN0MJshvTLDovLoZ9Vdc3sranHzEhPM9IM0szCKdhj0nGh1FLhEOljMtLTyEhPIy/Sxvw92ZROHh7z+hqamtkXVWDmvbmCgsEj2FZVy7bK2oM/t1fVsqemIRiUq7L26Cud+Vy7240uImkG6WFxiaQ3M2bBa5QUhsMNF2UzrDAYHXJ4UQ6FOZkqPB2kwiEiHZKZnnZwVEWAum0RSkuPaXXZ2oYmtkcVlOBxHdurglEdt1fVUVVTS1p6Bs3NTrM7Te40ezBoV1Nz8Big2aG5yQE/bBvVwM4Ne4A9reaQm5V+aBz7sLi0FJWSwmwONDbj7iouR6HCISJdJjsznVED8hg1oPXxUKD9rjbWuG0lAAAP0klEQVQ4vIg0u4cTNDU7c99YRPGIscFQw5VHDDW8t5Z9dY2s37Gf9TvavlllZNbTDMyPBOPV52UxIC+LAflZFOdFGNDSlp/FgLwIxflZ5GWl96lCo8IhIj2OmZGR3voH9dD8DEo/MKDN2KraBraGwwxvDotLy7DDWyoPsL3yAHWNzWwO58ciKyONgXlZFOdnUZxez6czt/Kh8YPIi/TOj9je+VuJiLShX3Ym/YZmMn5o6+PYl5eXc/yJJ7NrXz279teze38dO/cFww3v3l8fttcd9ri2ofmwYzev/GERWRlpnDN2INMnDeGC44cwML+Ng0o9UNIKh5ndB3wUqHD3E8K2YuARYDSwAfiUu++xYB/vTuASoAb4grsvCmOuAL4Trvb77v5AsnIWEYHg1Orc4gxGFsc2ZHBNfSO79tWzc18dT8xdxorKDBa9u4cXVlfwwuoKzJZRNqo/0ycNZdrEoRwzoGcPRZzMPY7fAb8AHoxquwF43t1/aGY3hM+/BVwMjAun04G7gNPDQnMzUEZwBKzczGa6e+tHvUREUiC60DSPz+PW0lIqqmt5flUFz6zYxmvrdrFgwx4WbNjD959axYShBUybNJRpE4cwaVi/Hnd8JGmFw91fMbPRRzTPAD4UPn4AeImgcMwAHnR3B143syIzKwmXfc7ddwOY2XPARcAfk5W3iEhnGFyQzeWnHcPlpx1DdW0DL63ZwbMrt/Pi6gpWb6tm9bZqfvb8WoYX5TBt0hCmTRzKqaP7pzrtmHT1MY4h7r4VwN23mtngsH048F7UcpvCtrbaRUR6jILsTD528jA+dvIw6hqbmLd+F8+u3M5zK7ezee8B7n91A/e/uoH+uZmM6ZfGyLVvkh/JID+87Ut+JOPg84LwKv387Izwiv0MIhlde7NMC77kJ2nlwR7H7KhjHHvdvShq/h53729mTwE/cPe5YfvzwDeBqUDE3b8ftt8E1Lj7T1rZ1tXA1QAlJSWls2bNSjjvmpoacnMT74NUvOIVr/hYNLuzdncD8zfXMX9zLdv2JTaiZIZBTqYFN71Md/5r6kByMuO/mWVZWVm5u5e1u72EskzcdjMrCfc2SoCKsH0TMDJquRHAlrD9Q0e0v9Tait39HuAegLKyMo/1XkGtieU8csUrXvGK74z4U4HPEFybsq5iH8+/sZQhI0ZRHV6Nv6+ukX3hz6CtIWgL26vrGqlvbKa63qmuDwrP6aeWJvUuyF1dOGYCVwA/DH8+GdV+rZn9ieDgeGVYXJ4B/svMWjr+pgE3dnHOIiJJZ2aMG1JA1bBsSk8ZEVdsXWPTweLyxpvLkn7r/GSejvtHgr2FgWa2ieDsqB8Cj5rZVcC7wCfDxecQnIq7juB03C8CuPtuM7sNWBAud2vLgXIREQlEMtKJ5KczID/Czv6ZSd9eMs+quryNWRe0sqwD17SxnvuA+zoxNRER6QANBSYiInFR4RARkbiocIiISFxUOEREJC4qHCIiEhcVDhERiUtSbzmSKma2A9jYgVUMBHYqXvGKV3wfix/l7oPaXcrdNR0xAQsVr3jFK74vxscyqatKRETiosIhIiJxUeFo3T2KV7ziFd9H49vVKw+Oi4hI8miPQ0RE4qLCISIicVHh6OPMLM3Mzkp1HiLSc+gYRzdiZicAE4HsljZ3f7ALtjvP3c9M9naSycy+5u53ttfWRmwu8O/AMe7+L2Y2Dhjv7rOTlG63FH6BGE3UOD3Jfv2Z2VR3f8HM/rG1+e7+eBzrGszh7513OyHF9rbZofzNrPho872bDlzX1UPHdktmdhxwPTCKw980U2OMHwT8C+9/010ZRw43E4yYOJFgRMSLgblATG9cM8sGrgImcfibJ5YcnjWzfwIe9wS/SYQftj/g/YXv2BjjO5I/BEMRH1kkvtBKW2vuB8qBluK5CXgMiKlwmNnXwnVUA78FTgFucPdnY4z/KHAbh15/RjC+Wb8YYpcBbf7P3P2kGHP4PfABYDHQ1BJOjK+/cB0f4f3/v1vbCTsPeAH4GIf/HhY+b7dwmNnHgZ8Aw4AKgr/jqjCXWHM/A/g5cDyQBaQD+2P4H3Q0//JwOQuft6yjJb7N94+ZPerun2rlNdDy+onpf58IFY7AY8Cvgd9w6E0TjyeBvwN/SzAe4FLgZOBNd/+imQ0h+BCK1e+B1cB04FbgswRvnlhcB+QBjWZWSxwfXFHuJxge+A7gfILhf+2oEYdLKH8zuxz4DDDGzGZGzSoAdsW47Q+4+6fDdeHuB8wsntyvdPc7zWw6MIjgd78fiKlwAD8F/hFYlkDh/mj4s2UEzd+HPz9LMAxzrMqAiR344vBrIJfgf/9bgtfzG+3FufvN4cN/Bf6Jw798xZrLbcAZwN/c/RQzOx9oawTStvwCuIzgs6AM+Dwwtr2gjubv7mNaHod7H+OIKrzt+Fr4836Cv/V7McZ1mApHoNHd7+pAfK67f6uDOdS6e7OZNZpZP4JvTjF9Ww+NdfdPmtkMd3/AzP4APBNLoLsXJPCiPVKOuz9vZubuG4HvmdnfCYpJLBLN/zVgK8H9eX4S1V4NLI1x2/VmlkP4RjezDwB1McbCoQJ5CXC/uy+Js/C8ByxP5EM7/FtjZme7+9lRs24ws1cJinAslgNDCf6WiTjL3U8ys6XufouZ/YQY9hai/BXYCywCasO2WP8eDe6+Kzxel+buL5rZj+LYdrAx93Vmlu7uTcD9ZvZaHOEdyR8z+xJBIRhBsNd3BsFr+31DbUfl2/K/KgDuBnYDfwL+7O7b48g9bn26cET1L84ys68ATxD1gRFH/+JsM7vE3eckmIcBS82siGCvpxzYRwzf2KI0hD/3hsdKthF8+4ll+3G/aFtRa2ZpwFozuxbYDAyOIz6h/MMPzo0c6mZKxM3A08AIM3sYOJugmytW5Wb2LDAGuNHMCoDmOOK/Ccwxs5c5/PX3v3GsI8/MznH3uRAUEoK9yFgNBFaa2RtH5PDxGONbPixrzGwYwYfYmKMsf6QR7n5RHMtH22tm+cArwMNmVsGh11OsaswsC1hsZj8mKKDx/P06kj8E779Tgdfd/XwzmwDcEkugu98C3GJmJwGfBl42s03ufmEH8jmqPl04eH//4vUc/i0h1m/8XwP+08zqCF6wcXX1uLub2WR33wv82syeBvq5e6zfmAHuMbP+wHeAmUA+cFMc+Sf0oo3ydYKuin8j6Do4n2B3P1YJ5W9mc939HDOrpvV+3lj+B38j+Lb9VeCPwI3h81hdBUwGMgm6OQYCv4sj/naCLwrZBP3ribgKuM/MCgn+DpUEXWax+l6C220xK/zi898E37qd4EtQrF4zsxPdfVkC215C0C33DYIuukKC1088vkFwlum14eORxNdV3JH8IehxqDUzzCzi7qvNbHyc66gg+MK1i/i+tMWtTxeOlv5FM/sU8LS7V5nZTcAUgg+/WNfTGV09r5vZqe6+wN03JBD/ew71sT4Qtg2JMbYzXrQe5jCK4AMUgg+OWA/QJZS/u58T/iyII9cj/YpgD6HA3WeHBewvBMU0Flfy/j22eQQHW2NR7O7T4kv5fZYDPyY4wN2foNvkY8CbsQS7+8sd3P5qoMnd/2JmEwneQ39tLyjqwG4G8EUze5tgjyeeA7znu3szwf/wgXC98XzpguC1ekX4wX9LeLzr68T+PzwH+IKZvZNA/gCbwsL7V+A5M9sDbIkl0Mz+lWBPYxDwZ+Bf3H1ljNtNSJ8uHFG+4+6Pmtk5wIcJ+srvAk6PJbiTunrOB75sZhuB/cT/wnuS4FtmOfH1z0MHXrRRHibYY1tGfN00LTqSf0ed7u5TzOxNAHffE3ZbxKqje2x/M7NpsZ6F1YYnOdTHvinWoE7aYwO4yd0fS+A99NF25rcp/MD8CvCBIwpFAfBqnKu7FPizmX2WoAh8DoinmF8c5/YO4+6fCB9+z8xeJNhrejrG8FHA1919cUdyiIsn+b7tPWEiOJMJgtNJPxPdFmP8MoI9jcXh8wnAI3HmMKq1KY745Z30tzgP+DiQFWfc3A5ut1PyT3Db8wlOv1wUPh8U5/9/QfhzMRBpeRxHfDVBsT0AVIXPq3rK3y/cfofeQwlus5BgD/WPR7xvihNc33HASoKTMnJS+ffs7pP2OAKbzexu4ELgR2YWIb6r6jvc1ePh2TEd0NE+1pY8Eu2yuNnMfgs8z+EHV2M9s6ZT8k/QzwhOjBhsZrcTfPv8ThzxHdpj887p6kzl3w86/h6Km7tXEuylxnvq7UH2/msgigm+RMw3MzyJ10L0ZLpynINXDl9EcB79WjMrAU702C/geoLgQOTXganAHiDT3S9JVs5R247uIx4HJNJH3Bl5PESwp7WCQ11V7jFewGdmKwnOm0+0j7hDwu6lC8LtPu/usV4Dc+R6ziPsZnD3+hhjWu3qdPeYuzq7wd+vQ++hVDGzUUeb3wlf6HolFY5OlsgHRwe31y1e+Ga2zN1P7EB8q79HX3jjhsW/5RjJ5JZjJO7+6TjW0Wf/ftL11FXVyTrQ1ZPo9rrLB8PrZjbREzyboxv9HqnQHbo6RWKmwiGd5Rzgig6cjtiXdcZZbSJdRl1V0inUVdI5urqrUyQRKhwiIhIXDeQkIiJxUeEQEZG4qHCItMPMvm1mK8xsqZktNrOYbkWT4LZeMrOyZK1fpDPorCqRozCzMwnupzTF3evMbCCJ38FWpFfQHofI0ZUAO929DsDdd7r7FjP7rpktMLPlZnaPWTBwU7jHcIeZvWJmq8zsVDN73MzWmtn3w2VGm9lqM3sg3Iv5c3jl9WHMbJqZzTOzRWb2mAVjTmBmPzSzlWHs/3Th30IEUOEQac+zwEgze8vMfhWeLgvwC3c/1d1PAHI4/C6v9e7+QYLhiJ8kGNb1BILbbg8IlxkP3BNe51JFcJfXg8I9m+8AF7r7FGAhcF14T6tPAJPC2O8n4XcWOSoVDpGjcPd9QClwNbADeMTMvgCcb2bzw9uFTAUmRYW1jH2+DFjh7lvDPZa3CQYIAnjP3Vtu/f0QwQWU0c4AJgKvmtli4AqCO79WEYy291sz+0fiG1dcpFPoGIdIOzwYg/ol4KWwUHyZYICqMnd/z8y+x+F3tW25O3Azh48t0syh99yRF1Ad+dyA59z9fXd+NbPTCG7IeBnBiHVT4/yVRDpEexwiR2Fm481sXFTTZGBN+HhneNzh0gRWfUx44B2C24LPPWL+68DZZjY2zCPXzI4Lt1fowfj2Xw/zEelS2uMQObp84OfhvaQagXUE3VZ7CbqiNgALEljvKoJ7e90NrCUYLe8gd98Rdon9MRzbAoJjHtXAk2aWTbBX8o0Eti3SIbrliEgXM7PRwOzwwLpIj6OuKhERiYv2OEREJC7a4xARkbiocIiISFxUOEREJC4qHCIiEhcVDhERiYsKh4iIxOX/A2ydWMCxgPOIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x25a69afb940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fd_fnbigrams=FreqDist(firstname_bigrams)\n",
    "fd_fnbigrams.plot(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exploring surname bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "surname_bigrams = get_ngrams(df['Surname'][surnames_mask], 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ha    2992\n",
       "ar    2548\n",
       "an    2276\n",
       "in    2175\n",
       "ra    2156\n",
       "sh    1753\n",
       "gh    1655\n",
       "si    1613\n",
       "ng    1566\n",
       "ma    1454\n",
       "al    1381\n",
       "at    1163\n",
       "pa    1098\n",
       "ai    1067\n",
       "wa    1034\n",
       "as    1033\n",
       "sa     945\n",
       "na     863\n",
       "ri     856\n",
       "ka     851\n",
       "ja     850\n",
       "da     834\n",
       "ch     830\n",
       "ta     794\n",
       "ah     769\n",
       "nd     679\n",
       "va     573\n",
       "er     572\n",
       "de     564\n",
       "ga     550\n",
       "      ... \n",
       "ft       1\n",
       "jb       1\n",
       "jl       1\n",
       "bg       1\n",
       "kc       1\n",
       "uy       1\n",
       "uq       1\n",
       "jv       1\n",
       "vh       1\n",
       "wb       1\n",
       "kn       1\n",
       "io       1\n",
       "db       1\n",
       "wt       1\n",
       "tm       1\n",
       "ps       1\n",
       "vl       1\n",
       "bj       1\n",
       "kj       1\n",
       "fz       1\n",
       "fs       1\n",
       "ec       1\n",
       "fu       1\n",
       "nz       1\n",
       "mw       1\n",
       "ex       1\n",
       "vg       1\n",
       "pc       1\n",
       "nl       1\n",
       "vt       1\n",
       "Length: 454, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(surname_bigrams).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEQCAYAAACnaJNPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPW5x/HPk51Awr6EfV9lkUQFEQW17tW6Vq/XrbZ0sa1V61Xb29tNW6916e5S1+pV3LACUhFFVBSBoOzIIiD7DmFJQrbn/nFOMCCQmSSTScj3/Xqd18z85jznPJNMzpPz+53F3B0REZFIJcQ7ARERqV9UOEREJCoqHCIiEhUVDhERiYoKh4iIREWFQ0REoqLCISIiUVHhEBGRqKhwiIhIVFQ4REQkKknxTiAWWrVq5V27dq1yfEFBAY0aNVK84hWv+AYVP2fOnG3u3rrSGd39mJuys7O9OnJzcxWveMUrvsHFA7kewTZWXVUiIhIVFQ4REYmKCoeIiEQlZoXDzNLMbJaZzTOzRWb267C9m5nNNLPlZvaimaWE7anh6xXh+10rLOuusH2pmZ0dq5xFRKRysdzj2A+c7u6DgSHAOWY2DPhf4CF37wXsBG4M578R2OnuPYGHwvkws/7AlcAA4Bzg72aWGMO8RUTkKGJWOMJB+r3hy+RwcuB04JWw/RngG+Hzi8LXhO+fYWYWto919/3uvgpYAZwYq7xFROToYjrGYWaJZjYX2AJMAT4Hdrl7STjLOqBD+LwDsBYgfD8PaFmx/TAxNWrtjnzun7yU2RsKY7F4EZFjgnkt3HPczJoBrwH/AzwVdkdhZp2ASe4+0MwWAWe7+7rwvc8J9ix+A8xw9+fC9ifCmFcPWccYYAxAVlZW9oQJE6LOc/Ln+Tz2yW76tUzk7tMrPwfmSPLz80lPT1e84hWv+HoVn5OTM8fdcyqdMZKTPWpiAn4J3A5sA5LCtuHA5PD5ZGB4+DwpnM+Au4C7KiznwHxHmqp6AmBeQZH3+e9J3uWOib5q694qLcO9fp8ApHjFK77hxhPvEwDNrHW4p4GZNQLOBJYA7wKXhbNdB7wePh8fviZ8f2r4QcYDV4ZHXXUDegGzYpFzZloy5w9sD8BLuWsrmVtEpGGK5RhHFvCumc0HZgNT3H0icAdwq5mtIBjDeCKc/wmgZdh+K3AngLsvAl4CFgNvAje5e2mskr7yxE4AvDxnHSWlZbFajYhIvRWzixy6+3zg+MO0r+QwR0W5eyFw+RGWdQ9wT03neDg5XZrTPiORDXv28+7SrXytf9vaWK2ISL2hM8cPYWac2S0YWHpx9po4ZyMiUveocBzGqC5pJCUYUz/bwqY8HZorIlKRCsdhNE1L5Gv921Lm8Oon6+KdjohInaLCcQTfPCEYJH9x9lrKymJ/rouISH2hwnEEI3u1pkOzRqzZkc/HK7fHOx0RkTpDheMIEhOMy3M6AjB2ts7pEBEpp8JxFJfndMIM3ly4iZ37iuKdjohInaDCcRQdmjXi1F6tKSot419z18c7HRGROkGFoxJXhoPkY2etLb9WlohIg6bCUYkz+rWlZeMUlm7ew7x1efFOR0Qk7lQ4KpGSlMCl2cEguc4kFxFR4YjIFTlBd9X4uRvYt7+kkrlFRI5tKhwR6NmmCSd0bc6+olLemL8x3umIiMSVCkeEvnlCZwDGqrtKRBo4FY4InTewHRmpSXyyZhfLNu+JdzoiInGjwhGh9JQkLhwS3B1w7CydSS4iDZcKRxSuDLurxn26jv0lMbsJoYhInabCEYXjOmTSPyuTXfnFvLVoc7zTERGJCxWOKJjZgXuSv6gLH4pIA6XCEaWLBncgNSmB6Su2sXZHfrzTERGpdSocUWqansx5A7MAeClXex0i0vCocFRB+d0BX85dR0lpWZyzERGpXSocVXBStxZ0a9WYTbsLeX/51ninIyJSq1Q4qsDMDux16JwOEWloVDiq6JKhHUhKMN75bAtb9hTGOx0RkVqjwlFFbTLSOKNfG0rLnFfn6O6AItJwqHBUQ/mZ5C/OXqO7A4pIg6HCUQ2n9m5Nu8w0Vm/PZ+aqHfFOR0SkVqhwVENignFFTvndATVILiINgwpHNV2e0wkzmLRgI3n5xfFOR0Qk5lQ4qqlTi3RO6dmK/SVlvD5Pg+QicuxT4agB5ed0vDBrrQbJReSYp8JRA77Wvy3N05NZsnE3C9fvjnc6IiIxpcJRA1KTErlkaDBIrnuSi8ixToWjhpR3V42fu4HCEl34UESOXSocNaR32wyGdm7Gnv0lzFi3P97piIjEjApHDSo/k/zVJXspLNY9yUXk2BSzwmFmnczsXTNbYmaLzOzmsP1XZrbezOaG03kVYu4ysxVmttTMzq7Qfk7YtsLM7oxVztX1jeM70KtNEzbuLeUvU5fHOx0RkZiI5R5HCXCbu/cDhgE3mVn/8L2H3H1IOE0CCN+7EhgAnAP83cwSzSwR+BtwLtAfuKrCcuqUlKQE7r10IAY8+t5KlmzUEVYicuyJWeFw943u/kn4fA+wBOhwlJCLgLHuvt/dVwErgBPDaYW7r3T3ImBsOG+dlN2lBWf3SKekzLlz3AJKy3Reh4gcW2pljMPMugLHAzPDph+a2Xwze9LMmodtHYCKF3xaF7Ydqb3OunpgE9plpjFv7S6e+Wh1vNMREalRFusznc2sCfAecI+7jzOztsA2wIHfAlnu/i0z+xsww92fC+OeACYRFLez3f3bYfs1wInu/qND1jMGGAOQlZWVPWHChCrnnJ+fT3p6erXiF+1K4N4Pd5GWaDx0divaNE6s1fUrXvGKV3y0cnJy5rh7TqUzunvMJiAZmAzceoT3uwILw+d3AXdVeG8yMDycJldoP2i+w03Z2dleHbm5uTUS//3ncr3LHRP9+idnellZWa2vX/GKV7ziowHkegTb9lgeVWXAE8ASd3+wQntWhdkuBhaGz8cDV5pZqpl1A3oBs4DZQC8z62ZmKQQD6ONjlXdN+tWFA8hMS+LdpVuZMH9jvNMREakRsRzjGAFcA5x+yKG395nZAjObD4wGbgFw90XAS8Bi4E3gJncvdfcS4IcEeyBLgJfCeeu8Nhlp/Oy8fgD8evwidu4rinNGIiLVlxSrBbv7dMAO89ako8TcA9xzmPZJR4ury755Qide+3Q9M1ft4J5JS7j/8sHxTklEpFp05niMmRm/v2QgKUkJvDJnHdOXb4t3SiIi1aLCUQu6t27CzWf0AuBnry2goEiXIxGR+kuFo5aMObU7fdtlsGZHPn98Z1m80xERqTIVjlqSnJjAvZcOwgwe/2AVC9fnxTslEZEqUeGoRUM6NeP6k7tSWubcNW4BJaW6b4eI1D8qHLXsp2f1oUOzRixYn8dTH66OdzoiIlFT4ahljVOTuPsbxwHw4JRlrN2RH+eMRESio8IRB6P7tuHCwe0pKC7lZ68tKL+UiohIvaDCESf/8/X+NEtP5oPl2/jX3PXxTkdEJGIqHHHSqkkqPw8vR/KbCYvZvlf3KReR+kGFI44uy+7IiJ4t2ZlfzN1vLIl3OiIiEVHhiCMz43cXDyQtOYHXPl3Pe8u2xjslEZFKqXDEWZeWjfnJmb0B+PlrC8gvKolzRiIiR6fCUQd8+5RuDGifybqdBTz4li5HIiJ1mwpHHZCUmMC9lwwiweDJD1exbLvu2yEidZcKRx0xsGNTbjylG2UO/zNtBw+8tVTdViJSJ6lw1CG3ndWHS4Z2oLgM/jJ1BWc88B6vz12vEwRFpE5R4ahD0pITefCKIdwzugUDOzRlY14hN4+dy+WPzNDVdEWkzlDhqIP6tkrh9ZtGcN+lg2jVJIXcL3by9b9O585X57NNJwqKSJypcNRRCQnGFSd0YupPR/Gdkd1INGPs7LWMvn8aj3+wkmJdkl1E4kSFo47LTEvm5+f3582fnMppvVuzp7CEu99Ywjl/fF8nDIpIXKhw1BM92zTh6RtO4InrcujaMp3Pt+7juidn8e1nZrN62754pyciDYgKRz1iZpzRry2TbzmVO8/tS+OURN5esoWzHnqfe//9GXv36/BdEYk9FY56KDUpke+d1oN3bx/FZdkdKSot45H3Puf0+6fx6px1lJbp8F0RiZ2keCcgVdcmI437Lx/M1Sd15lcTFjNv7S5ue3keCUCbKe/QrmkaWU3TKjw2Ch4z02ibmUZKkv5vEJHoqXAcA47v3JzXvn8yr326nj9PXc4X2/PZtLuQTbsLmbv2yHGtmqQeVFiywsLSbL+O2BKRI1PhOEYkJBiXZnfk0uyOfDw7l449+7Mpr5CNeYVszCtgY17hgdeb8grZsqeQbXv3s23vfhYccnLhca1TGHVynD6IiNR5KhzHoOQEo2PzdDo2Tz/iPCWlZWzdu/+QglLAC7PWsnBrEYs25DGgfdNazFpE6gsVjgYqKTEh7JpqdFB7aVlwhd6nP1zNHy4fHKfsRKQu0+ioHOTa4V0w4PV5G3QfdBE5LBUOOUjXVo0ZmpVKUUkZY2cfZWRdRBosFQ75ivN7BWMjz874QtfEEpGvUOGQrxjUJoWebZqwaXchby7cFO90RKSOibpwmFlzMxsUi2SkbjAzrj+5KwBPf7Q6rrmISN0TUeEws2lmlmlmLYB5wFNm9mBsU5N4umRoBzLSkpjzxU7mr9sV73REpA6JdI+jqbvvBi4BnnL3bODM2KUl8ZaeksSVJ3QCtNchIgeLtHAkmVkWcAUwMYb5SB1y7fCumMHEeRvZukeH5opIINLC8WtgMrDC3WebWXdg+dECzKyTmb1rZkvMbJGZ3Ry2tzCzKWa2PHxsHrabmf3ZzFaY2XwzG1phWdeF8y83s+uq9lElWp1apHNmv7YUlZbx/Mw18U5HROqISAvHRncf5O4/AHD3lUBlYxwlwG3u3g8YBtxkZv2BO4F33L0X8E74GuBcoFc4jQEehqDQAL8ETgJOBH5ZXmwk9m4IB8mfm/kFRSU6NFdEIi8cf4mw7QB33+jun4TP9wBLgA7ARcAz4WzPAN8In18E/NMDHwPNwu6xs4Ep7r7D3XcCU4BzIsxbqml4j5b0aZvB1j37+ffCjfFOR0TqgKNeq8rMhgMnA63N7NYKb2UCiZGuxMy6AscDM4G27r4RguJiZm3C2ToAFU9VXhe2HaldaoGZcf2Irtw1bgFPfbiai4boRy/S0Jn7ke8WZ2anAaOA7wGPVHhrDzDB3Y86zhEuownwHnCPu48zs13u3qzC+zvdvbmZvQH83t2nh+3vAP8FnA6kuvvdYfsvgHx3f+CQ9Ywh6OIiKysre8KECZV++CPJz88nPf3IV5ZtaPH7S5wxE7ewt9j5/ekt6N0ypVbXr3jFK7524nNycua4e06lM7p7pRPQJZL5DhOXTDCofmuFtqVAVvg8C1gaPn8UuOrQ+YCrgEcrtB803+Gm7Oxsr47c3FzFH+J3kxZ7lzsm+o9f+CQu61e84hUf+3gg1yPYtkc6xpFqZo+Z2VtmNrV8OlqAmRnwBLDE3SsOpI8Hyo+Mug54vUL7teHRVcOAPA+6tCYDZ4VnrDcHzgrbpBZdM6wLCQZvzN/I5t2F8U5HROIo0vtxvEzQVfU4UBphzAjgGmCBmc0N234G3Au8ZGY3AmuAy8P3JgHnASuAfOAGAHffYWa/BWaH8/3G3XdEmIPUkI7N0zl7QDv+vXAT/zdzDbd+rXe8UxKROIm0cJS4+8PRLNiDsQo7wttnHGZ+B246wrKeBJ6MZv1S864/uSv/XriJ52d+wU2je5CaFPHxESJyDIm0q2qCmf3AzLLCE/hahOdXSANyYrcW9MvKZNveIibO06G5Ig1VpIXjOuB24CNgTjjlxiopqZvM7MAJgU9/tLr8YAURaWAiKhzu3u0wU/dYJyd1z4VD2tM8PZkF6/P4ZM3OeKcjInEQ0RiHmV17uHZ3/2fNpiN1XVpyIv9xUmf+9u7nPPXharK7qMdSpKGJtKvqhArTSOBXwIUxyknquP8c1oXEBOPfCzexMa8g3umISC2LtKvqRxWm7xBcPuTopw/LMSuraSPOOa4dpWXOcx9/Ee90RKSWVfWe4/kEV7GVBqp8kPz5mWsoLI701B4RORZEOsYxASg/hCYR6Ae8FKukpO7L7tKc4zpksnD9bsbP28AVOZ3inZKI1JJITwC8v8LzEuALd18Xg3yknggOze3GbS/P4+kPV3N5dkeCq8yIyLEu0jGO94DPgAygOVAUy6SkfrhgcBatmqSweONuZq3SVWBEGoqICoeZXQHMIriu1BXATDO7LJaJSd2XmpTIf5zYGQhOCBSRhiHSwfGfAye4+3Xufi3BLVx/Ebu0pL64elgXkhKMyYs2sX6XDs0VaQgiLRwJ7r6lwuvtUcTKMaxtZhrnDcyizOHZGTo0V6QhiHTj/6aZTTaz683seuANgsugi3DDiK4AvDBrDQVFOjRX5Fh31MJhZj3NbIS7305w571BwGBgBvBYLeQn9cDxnZszuFMz8gqK+dfc9fFOR0RirLI9jj8S3F8cdx/n7re6+y0Eext/jHVyUn8cuGruh7pqrsixrrLC0dXd5x/a6O65QNeYZCT10nkDs2idkcrSzXtYuFVHa4scyyorHGlHea9RTSYi9VtKUgJXnxQcmvvInN28u3RLJREiUl9VVjhmm9l3Dm0M7xc+JzYpSX117fCudG/dmE17S7nhqdlc++Qslm3eE++0RKSGVXbJkZ8Ar5nZ1XxZKHIIrox7cSwTk/qnReMU/n3zSH730oeMW1bA+8u2cs7yrVx1Ymdu/VpvWjZJjXeKIlIDjrrH4e6b3f1k4NfA6nD6tbsPd/dNsU9P6pvUpEQu7NOYaT8dxbXDu2Bm/N/MNYz6wzQefe9z9pfocF2R+i7Sa1W96+5/CaepsU5K6r+WTVL5zUXH8ebNIxnVpzV79pfw+39/xpkPvsekBRt15JVIPaazvyWmerXN4OkbTuSZb51I77ZNWLujgB/83ydc8egM5q3dFe/0RKQKVDikVpzWuzWTfjySey4+jpaNU5i9eicX/e1Dbn1xrm4/K1LPqHBIrUlKTODqk7rw7u2j+O5p3UlJTGDcp+sZff80HpyyjPyikninKCIRUOGQWpeZlsxd5/bjndtO4/yBWRQWl/Hnd5Yz6g/TeDl3LWUa/xCp0yK9A6BIjevUIp2/XT2UG1bv4LcTFzNvXR63vzKfzBRj+JJcTuzWkpO6taBfViaJCbq7oEhdocIhcZfTtQWv/WAE4+dt4IEpS1m7o4DJizYzedFmADJSk8ju2pwTu7XgpG4tGNihGSlJ2lkWiRcVDqkTEhKMbxzfgYuGtGfS+7PY17g9s1btYNaqHazZkc+0pVuZtnQrAKlJCQzt/GUhOb5zcxqlJMb5E4g0HCocUqeYGe2aJJGd3YkrcjoBsDGv4EARmb16B8s272XGyu3MWLkdgKQEY1DHppwQFpLk4rJ4fgSRY54Kh9R5WU0bcdGQDlw0pAMAO/YVMXv1jgPFZNGGPD5Zs4tP1uzi0fdWkmAw9NOPOKVXK0b2as3gjk1JSlTXlkhNUeGQeqdF4xTOHtCOswe0A2BPYTFzvtjJrFU7mLlqB5+u2UnuF8H0x7eXk5GWxPDuLRnZuzUje7aiS8t0zDTYLlJVKhxS72WkJTOqTxtG9WkDwPsfz6YwoxPTV2zjg+XbWLVtH28t3sxbi4PB9o7NGzGyV2tG9mrFyT1a0iw9JZ7pi9Q7KhxyzGmcnMCpA9pxVrhHsm5nPtOXB0Xkw8+3sW5nAS/MWsMLs9aQYDCwYzNG9mzFyF6tOL5z8zhnL1L3qXDIMa9j83SuPLEzV57YmdIyZ+H6vHBvZCtzvtjJvLW7mLd2F399dwXpKYmM7JTCff2LadooOd6pi9RJKhzSoCQmGIM7NWNwp2bcNLon+/aXMGvVDj5YHhSS5Vv2MvnzAj598D1++43jDoyjiMiXVDikQWucmsTovm0Y3TcYH1m6aQ8/fnYGS7fv57vPzuH8gVn86sIBtM7QTahEysXsGEUze9LMtpjZwgptvzKz9WY2N5zOq/DeXWa2wsyWmtnZFdrPCdtWmNmdscpXBKBPuwx+O7oFv/x6f9JTEnljwUbOfPA9Xs5dq3uIiIRieXD708A5h2l/yN2HhNMkADPrD1wJDAhj/m5miWaWCPwNOBfoD1wVzisSM4lm3DCiG2/dciqn9m5NXkExt78yn2ufnMXaHfnxTk8k7mJWONz9fWBHhLNfBIx19/3uvgpYAZwYTivcfaW7FwFjw3lFYq5j83SeueEEHrxiMM3Sk/lg+TbOeuh9npi+itIy7X1IwxWP02l/aGbzw66s8mMfOwBrK8yzLmw7UrtIrTAzLhnakbdvPY0LBmVRUFzKbycu5tKHP2Lppj3xTk8kLiyW/bZm1hWY6O7Hha/bAtsAB34LZLn7t8zsb8AMd38unO8JYBJBYTvb3b8dtl8DnOjuPzrMusYAYwCysrKyJ0yYUOW88/PzSU9PV7ziv2L2hkIe+2Q3OwrKSDK4pF9jLunbhOREiyi+uutXvOJjGZ+TkzPH3XMqndHdYzYBXYGFlb0H3AXcVeG9ycDwcJpcof2g+Y40ZWdne3Xk5uYqXvFHlFdQ5D8bN9+73DHRu9wx0c98YJrP+WJHra1f8YqPVTyQ6xFs22u1q8rMsiq8vBgoP+JqPHClmaWaWTegFzALmA30MrNuZpZCMIA+vjZzFjlUZloy91w8kLFjhtGtVWOWb9nLpQ9/xK8nLGLfft3+Vo59sTwc9wVgBtDHzNaZ2Y3AfWa2wMzmA6OBWwDcfRHwErAYeBO4yd1L3b0E+CHBHsgS4KVwXpG4G9a9Jf++eSTfH9WDBDOe+nA1Zz30PjPWFVJSqku7y7ErZicAuvtVh2l+4ijz3wPcc5j2SQTjHSJ1TlpyInec05fzB2Zxx6vzWbRhN/fPKOD5xe9y9bAuXHlCJ1o20cmDcmzRTQpEasBxHZry+k0j+M1FA8hqksiGvEL+MHkpw38/lVtfnMvctbvinaJIjdElR0RqSFJiAtcO70q/5G0UZHbhnzNW885nWxj36XrGfbqewR2bcu3wrpw/KIu0ZN3qVuovFQ6RGpZgxqm9W3Nq79as3ZHPcx9/wYu5a5m3Lo/bXp7HPZOW8M0TOnH1SZ3p2Lzqh12KxIu6qkRiqFOLdO46rx8f33UG9102iOM6ZLJjXxEPT/ucU+97l+/8M5fpy7fpOlhSr2iPQ6QWpCUnckVOJy7P7sina3fxz49W88aCjUxZvJkpizfTo3Vjrh3elUuG6sIIUvdpj0OkFpkZQzs3549XHs9Hd57BbV/rTbvMND7fuo9fjl/EsN+9w7+W7ot3miJHpcIhEietM1L50Rm9mH7HaB6+eijDurdgX1Epz87fw+tz18c7PZEjUuEQibOkxATOHZjF2DHD+c1FAwDCc0Ly4pyZyOGpcIjUIdcM68LpXRtRWFzGd5+dw859RfFOSeQrVDhE6hAz4ztDMxnUsSnrdhbw47Gf6t4fUueocIjUMSmJxiP/mU3Lxil8sHwbf5i8NN4piRxEhUOkDmrfrBF//Y+hJCYYj7z3ORPnb4h3SiIHqHCI1FHDe7Tk5+f1A+C/XpmvOw5KnaHCIVKH3TCiKxcf34H8olLGPJtLXn5xvFMSUeEQqcvMjN9dPJD+WZl8sT2fm1/UYLnEnwqHSB3XKCWRR6/Jpll6MtOWbuWPby+Ld0rSwKlwiNQDnVqk89erhpJg8JepK3hz4aZ4pyQNmAqHSD1xSq9W3HluXwBue2kuK7ZosFziQ4VDpB75zsjuXDAoi31FpYx5dg67CzVYLrVPhUOkHjEz7rtsEH3bZbBy6z5ufXEeZRosl1qmwiFSz6SnJPHoNdlkpiXx9pLN/GXqininJA2MCodIPdSlZWP+fNXxmMFDby/jnSWb452SNCAqHCL11Kg+bfjpWX0A+MnYuazcujfOGUlDocIhUo/9YFQPzhnQjj37S/jus3PYu78k3ilJA6DCIVKPmRn3XzGYnm2asHzLXn760jzcNVgusaXCIVLPNUlN4rFrsslITeLNRZt4/NM9bNhVEO+05BimwiFyDOjeugl/vHIIAG9+ns+I/53KNU/M5PW56ykoKo1zdnKsSYp3AiJSM87o15axY4bx50lzyd1YxAfLt/HB8m00SU3igkFZXJbdkewuzTGzeKcq9ZwKh8gxZFj3liQPb0aPfgOZMH8jr8xZx7y1uxg7ey1jZ6+lW6vGXDq0AxcP7UiHZo3ina7UUyocIsegZukpXDOsC9cM68KKLXt4Zc56xn2yjlXb9nH/W8t4YMoyRvRoxWXZHTl7QDsapSTGO2WpR1Q4RI5xPdtkcOe5ffnpWb2ZvmIbr8xZx1uLNzN9xTamr/hqV5ZIZVQ4RBqIpMQERvVpw6g+bcjLL2bC/A28Mmcdcyt0ZXVtmc7AlrCadfRpl0HPNk1IS9beiBxMhUOkAWqansx/DuvCfw7rwoote3n1k3WM+2Qdq7fns3o7TFg2D4DEBKN7q8b0zcqkb7uMYMrKpH3TNA2yN2AqHCINXM82TbjjnL789Kw+zPh8O298vIjdCRks2bSb1dv2sXzLXpZv2cuEeV/GZKQlhYUkk75ZwWOfdhk0SdUmpSHQb1lEgGDv4pRerWi0uwnZ2UMBKCwuZfnmvSzZtJvPNu5h6ebdLNm4hx37ipi9eiezV+88aBmdWjQiK62Mk3cuY0D7pgxon0mW9k6OOSocInJEacmJDOzYlIEdmx5oc3e27t3P0k17+GzjngNFZcWWvazdUcBaYNaG5Qfmb56eTP/2mQxo35T+WZkMaJ9Jt1aNSUrU+cf1lQqHiETFzGiTkUabjDRG9mp9oL24tIxV2/Yx6aN55Ke2ZNGGPBZt2M3O/GI+XLGdD1dsPzBvWnICfdoFRaS8mPRtl6nDguuJmBUOM3sSuADY4u7HhW0tgBeBrsBq4Ap332nBfuyfgPOAfOB6d/8kjLkO+O9wsXe7+zOxyllEqi45MYHebTPY07kR2dn9gGDvZGNeIYs27Gbxht0Hisn6XQXMW7uLeWt3HYhPsODSKekU0XTuTMwMC9vN7MBj0GaYBY+Ej+XzZpTupc9xJRo3CFjHAAAQ/0lEQVRviaFY/mSfBv4K/LNC253AO+5+r5ndGb6+AzgX6BVOJwEPAyeFheaXQA7gwBwzG+/uB3esikidZGa0b9aI9s0a8bX+bQ+078ovYvHG8mISPK7YupcVW8J7imzZVq31Tvh8KmNO7c51w7vSWAWkxsXsJ+ru75tZ10OaLwJGhc+fAaYRFI6LgH96cD3oj82smZllhfNOcfcdAGY2BTgHeCFWeYtI7DVLT+HkHq04uUerA22FxaUs27yH2fMW07NXL8rcwaHMHS9/JNiLCV6D48Fj2FZUUsZT7y1hybZi7ntzKY9/sIrvndada4Z1VTdYDartUtzW3TcCuPtGM2sTtncA1laYb13YdqR2ETnGpCUnMqhjM4o3p5Ldu3XlAUfQ3TZT0LQLD05ZxqdrdvG7SZ/x2Pur+P6oHlx9Umed0FgDLJY3fQn3OCZWGOPY5e7NKry/092bm9kbwO/dfXrY/g7wX8DpQKq73x22/wLId/cHDrOuMcAYgKysrOwJEyZUOe/8/HzS09MVr3jF1+N4d2fu5iLGLtzLip3FADRPS+CSvo05s3s6KYmHP0S4ruQfj/icnJw57p5T6YzBLl5sJoJB8IUVXi8FssLnWcDS8PmjwFWHzgdcBTxaof2g+Y40ZWdne3Xk5uYqXvGKP0biy8rK/O3Fm/y8P73vXe6Y6F3umOgn3fO2/3PGai8sLon5+utTPJDrEWzba/tA6vHAdeHz64DXK7Rfa4FhQJ4HXVqTgbPMrLmZNQfOCttERCJiZpzRry0Tf3QKj16TTd92GWzaXcgv/rWQ0+9/jxdmraG4tCzeadYrMSscZvYCMAPoY2brzOxG4F7ga2a2HPha+BpgErASWAH8A/gBgAeD4r8FZofTb8I2EZGomBlnD2jHpB+P5OGrh9K7bRPW7yrgrnELGH3/NF6avVYFJEKxPKrqqiO8dcZh5nXgpiMs50ngyRpMTUQasIQE49yBWZw9oB1vLNjIH99exudb9/Ffr87nb9NWcEH3JAYNKSNZZ7YfkX4yItIgJSQYXx/cnrduOY0/XTmEbq0a88X2fP42ezen3fcuT324SvdrPwIVDhFp0BITjIuGdGDKLafywOWD6ZiRyIa8Qn49YTEj/ncqf526nLyC4ninWafolEoREYIbXV2a3ZHOvontaR15eNoK5q3L4/63lvHIeyu5elhnbhzRjTaZafFONe60xyEiUkGCGecc145/3TSC//v2SYzo2ZK9+0t49L2VnHLfu/zstQWs2Z4f7zTjSnscIiKHYWaM6NmKET1bMW/tLv4+bQWTF23m+ZlrGDtrDRcMas/3R/WgX1ZmvFOtddrjEBGpxOBOzXj0mhzevvVULh3akQQzxs/bwLl/+oBvPT2b3NUN6ywB7XGIiESoZ5sMHrhiMLee1Zt/vL+SsbPXMPWzLUz9bAsndm3B90f3ICOGl3GqK1Q4RESi1KFZI3514QB+dHpPnv5oNc98tJpZq3cw66kdpCcbTae8Q6OURBolJ5Kekkha8pfPG4Wv08P3G6UkhY8JNEpOYt2m/ez/fBupSQmkJiWSkpRASmICqcnBY0rYnpxocbslrwqHiEgVtWySym1n9WHMqd15fuYaHp++iq179pOfV1i9BX8wM6LZgiISTEFxSaS0aD+TB5bG9DLyKhwiItWUkZbMd0/rwbdHduf9j3Pp1W8AhcWlFBSVUVBcSn5RSfC6uJT8olIKikopLH9eXBrOG7zesn0naY2bUFRSxv6SsoMei0rL2F9cSlFpGcWlHrSVlLHnkHwSE2K7J6LCISJSQxITjMzUBDo2r/pl0efMmUN2dnal85WVeVBISsrYX1J6oMDMnb+Q5CNcMr6mqHCIiNRDCQlGWkJieGOq5APtu5olx3zsQ4fjiohIVFQ4REQkKiocIiISFRUOERGJigqHiIhERYVDRESiosIhIiJRMT8GL8hlZluBL6qxiFbANsUrXvGKb2DxXdy9daVzubumQyYgV/GKV7ziG2J8JJO6qkREJCoqHCIiEhUVjsN7TPGKV7ziG2h8pY7JwXEREYkd7XGIiEhUVDhERCQqKhzVZGYJZnZyvPMQEaktGuOoAWY2w92H14E8Tga6UuEGXe7+zyjijwP6A2lVia9tZtbX3T8zs6GHe9/dP6ntnKJhZi2O9r6774hiWc+6+zWVtdU0M8t0991H+iyRfob69t2rKTX186ttugNgyMzSgBuBARz85f1WBOFvmdmlwDivQiU2s9bAd/jqRj+SdZcv41mgBzAXKC1fBBDRH5+Z/RIYRfDHOwk4F5geSbyZ9QZuB7ockv/pEa57BPCrCvEWhHv3SkJvBcYAD1Roq/jzj2j9YQ4LDokFyANygbvdfXsl8b2A3/PVjd/RPsOccJ3lt2srX7+Fzyv7/BUNOCSfJKDy+48eHHM+X/3+/6aSsOeBC/jysxxYHBF+hup89yos42bgKWAP8DhwPHCnu78VxTKq8vnLY6u6/XjezL5OcKb3ar78LkAU3wEzGwb8BegHpACJwD53z4wkPloqHF96FvgMOBv4DXA1sCTC2FuBxkCJmRXy5YYv0l/a68AHwNt8udGPVg7QvyqFK3QZMBj41N1vMLO2BH+AkXgZeAT4B1XL/wngFoKNT8Tx7j4mfPow8Gb4n9svgKHAb6PM4d/hup8PX18ZPu4Gnga+Xkn8U8AvgYeA0cANHLwROFz+3cqfh/9x9qLCRicSZnYX8DOgkZntrvBWMVEclmlmjwDpBLk/TvB9mFVZnLtfED52q+pnoHrfvXLfcvc/mdnZQGuCn/9TQESFo6qfv4IqbT/Kf35mNtfdD7vnHKG/EnxnXybYFlwL9KzG8o4u1qem15eJ4EsLMD98TAamRhHfAjgJOK18iiJ2bg3k/zKQVY342eHjHCCTYKO3KMLYOdXMfWY148t/Z6cA7wMXRbtM4MMjtQELIv0ZVJwX+CDCdX8bWADsBN4FCoB3osz/90DzQ76Dp1bhZ1j+2AR4K4r4Kn+G6nz3DpP/n4CLw+ef1uLnr+7246/ACdF85kPicyuuP3z+UVWXV9mkPY4vFYePu8L+1k0EXUeVMrNvAzcDHQm6ioYBHwFnRLjuiWZ2nrtPiirjg7UCFpvZLGB/eaO7X1hZoAV3tp9vZs0I9hrmAHup5D+uCv2yE8zsB8Brh6z7qP2zFcYm3jWzPwDjDomPdIyifC/lfOARd3/dzH4VYWy5JmZ2krvPDHM7iWDjAVASQXyhmSUAy83sh8B6oE2E674ZOAH42N1Hm1lf4NfRpc9KgqJZ8Ts4g8i76wrCx3wzaw9sB7odZf5DVeczzI72u3cYc8zsLYKc7zKzDKAsivjC8LH88+8gus9f5e1H6HTge2b2BbCPL3stBkUYn29mKcBcM7sP2EjQCxITKhxfeszMmgP/DYwn2Gj8IsLY6v7h3wz8zMz2E3wBo+3qgmCMoErc3c1siLvvAh4xszeBTHefX0nooX30t3NwP3dl/bMPHPK6vE++vH880o3eejN7FDgT+F8zSyX6Iwa/DTxpZuXFYg9wo5k1JvhvvjI/Iejq+DFBN9logu6CSBS6e6GZYWapHgz494ky/x9Tve/gxHDjfR/B7xWi6y6qzmfIAC4HpgGRfvcOdSMwhOA//RyCf6SejiJ+Qvj5/wB8QvD9+0cU8dXZfkAwrlMdtxB8538YPu9E9N19EVPh+NKzwKUE/yU8E7a1jTC2Wn/47p5Rjf7h8mW8V5W4Cj42sxPcfba7r45wnd0AzOwKqjDG4O6jw/jb+Oog8e6wmM2NIJUrgHOA+919l5llERSxaMwH7iX4/bcm6HI5191nAy9FEO8E36EuBBsvCDY8kfzHuC7caP0LmGJmO4ENUWVf/eJzP/B9YCTBnsoHBGNHkarOZ3iKoJvxLwT/bMw1s/fd/U9RrP9bfHWvf0a4zEh8BpS6+6tm1p/gO/yvKNZfne0H7l6d20BA8F27zt0XAL82s6sI/pmJ9PNHRYfjhsL/svM4ZIDW3Q/9r/hwsa8RDMb9hOC/5J1AsrufF+G6D9vV5e6VdnWZ2XR3P8XM9nCYo1oi3Wsxs8VAb4L7mES1q2xm8919kJmdAvyOYE/iZ+5+UoTrfp7gv8Tx4XrPB2YDfYGX3f2+SJZTHeHvfxfBf5tR/f7D+KUExWoBFbpIot0gmNlpQFOCQlwURVx1v4MvEexlPRc2XQU0c/croki/fFlRfwYzSyTYYxoNfA8ocPe+UaxzAV/ucQ0p3+Ny929GGF/d73CVtx81wcy6A68QDMqfAlwDfN3d82KywlgNntS3CVhYQ8s5DbgQSIkiZgHBnsbc8HVf4MVa/vxdDjdFGFs+MPh74D8qtkUYPxloUuF1E4Iui0bA4vrw+wem1+bvq5JcqvIdnBdJW4zyfQf4mOCItEuANlVYRvkA+1wgtfx5FPHV/Q7XyPajmj/H3sDi8O+pUSzXpa6qL31kZgM92NWrMq9al1FN9HFXi1dvV7m6YwydgYr/mRYTFK2CcNynNlT39/9LM3ucYCNYcYB/XI1kF4Uqfgc/NbNh7v4xHDg44MOazeyI5hOMbx1H8F/7LgtOqi04ethBqtvdV93vcI1sP6JlXz3/qAXBORwzzQyPfHA9uvWGlarBqvCDTyIYY1hJ8Icf7VEN1cmhWt0M8WZm6QRjDAvcfXk4xjDQIzz5KhwXuZjgfBYIzpkYT9Bd8Ji7Xx2DtA/NYTHBce+rqMLv38yeI9hTXMSXXVXuUZzEGU9mtgToA6wJmzoTnIdQRu39HTQh+Dv4KdDO3VOruJyqdJVV9ztcre9PVZlZl6O9X81/CI+8XhWO+Pzgj6Sqfdz1nZllE/TNGkG3T24tr/+w34NIf/9mtsDdB9ZsVrUnnn8H4eHLIwn2Or4gOKz4A3efGqt11rTqfn/qmwZfOERqgpn9A3jI3RfHO5f6xsxuJygWc9w9knNmJM5UOERqQNjV04Na7qoQiQcVDpEa0NC6KqRhU+EQEZGo6EZOIiISFRUOERGJigqHSCXM7OdmtsjM5pvZ3PDkuFita5qZ5cRq+SI1QWeOixyFmQ0nuMPdUHffb2atCO6wJtJgaY9D5OiygG3uvh/A3be5+wYz+x8zm21mC83sMTMzOLDH8JCZvW9mS8zsBDMbZ2bLzezucJ6uZvaZmT0T7sW8Ep65fBAzO8vMZpjZJ2b2cnhmNWZ2r5ktDmPvr8WfhQigwiFSmbeATma2zMz+Hp7ZD/BXdz/B3Y8juBjjBRViitz9VILb6b4O3ERwHabrzaxlOE8fgsupDCK4Pe0PKq403LP5b+BMD24pmgvcasHl9y8GBoSxd8fgM4sclQqHyFG4+16CS2GMAbYCL5rZ9cBoM5sZXuvsdGBAhbDx4eMCglugbgz3WFYS3GAHYK27l19E8DmCy61UNAzoD3xoZnOB6wiuWLyb4G51j5vZJUB+jX1YkQhpjEOkEu5eSnB3umlhofguwQ2actx9rQW3qa14A67yq+OWVXhe/rr8b+7QE6gOfW3AFHe/6tB8zOxEgtsSX0lwx7dI75QoUiO0xyFyFGbWx8x6VWgaAiwNn28Lxx0uq8KiO4cD7xDcNGn6Ie9/DIwws55hHulm1jtcX1MP7k//kzAfkVqlPQ6Ro2sC/CW810MJsIKg22oXQVfUaoK7FUZrCXBdeA+I5Rxym1Z33xp2ib0Q3hsCgjGPPcDrZpZGsFdySxXWLVItuuSISC0zs67AxHBgXaTeUVeViIhERXscIiISFe1xiIhIVFQ4REQkKiocIiISFRUOERGJigqHiIhERYVDRESi8v+qV5JvLlU/4gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x25a6a2c8d68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fd_snbigrams=FreqDist(surname_bigrams)\n",
    "fd_snbigrams.plot(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exploring name trigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "firstname_trigrams = get_ngrams(df['Name'][firstnames_mask], 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "esh    940\n",
       "sha    791\n",
       "ish    691\n",
       "ash    648\n",
       "raj    627\n",
       "pra    554\n",
       "man    538\n",
       "san    510\n",
       "han    497\n",
       "dra    449\n",
       "ndr    442\n",
       "har    435\n",
       "ita    433\n",
       "end    415\n",
       "shi    409\n",
       "ana    409\n",
       "ant    400\n",
       "anj    373\n",
       "and    366\n",
       "jay    361\n",
       "ani    354\n",
       "eet    353\n",
       "bha    347\n",
       "ind    346\n",
       "dee    333\n",
       "eep    327\n",
       "nde    323\n",
       "vin    317\n",
       "ara    303\n",
       "ika    292\n",
       "      ... \n",
       "ual      1\n",
       "vsa      1\n",
       "ubd      1\n",
       "ehj      1\n",
       "riv      1\n",
       "ruh      1\n",
       "mza      1\n",
       "oof      1\n",
       "tbi      1\n",
       "too      1\n",
       "hja      1\n",
       "wdh      1\n",
       "aac      1\n",
       "ogo      1\n",
       "eis      1\n",
       "lep      1\n",
       "pjy      1\n",
       "llo      1\n",
       "rii      1\n",
       "phe      1\n",
       "bhh      1\n",
       "dic      1\n",
       "amc      1\n",
       "lfi      1\n",
       "ofa      1\n",
       "one      1\n",
       "axa      1\n",
       "nuu      1\n",
       "rqa      1\n",
       "kuk      1\n",
       "Length: 2987, dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(firstname_trigrams).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPXV+PHPyb6QPQHCGjYRQUASBBFRtFrbutel1ioulbba1tanrfan1tY+bW2fPk9bu7iBu3WtiuAubiCCJLIvCiQgOwmEBLKQ7fz+uDcwhJDcmclkEua8X695zcyde+49M5nMuff7vfd7RVUxxhhjWooKdwLGGGO6JisQxhhjWmUFwhhjTKusQBhjjGmVFQhjjDGtsgJhjDGmVVYgjDHGtMoKhDHGmFZZgTDGGNOqmHAnEIzs7GzNy8sLKLampobExMSA123xFh/O+K6Qg8V33/iioqIyVc1pd0ZV7ba3/Px8DVRhYWHAsRZv8eGO7wo5WHz3jQcK1cNvrDUxGWOMaZUVCGOMMa2yAmGMMaZVViCMMca0ygqEMcaYVlmBMMYY06qILRAHGu1KesYY05aILBA/f2EZ187aSXHp/nCnYowxXVZEFoiGJqWuEd5evTPcqRhjTJcVkQXinBN6AfD2qh1hzsQYY7quiCwQpw/PIS4KPvtyL7sqa8OdjjHGdEkRWSCS4mIY3SsesGYmY4w5mogsEAAT+lqBMMaYtkRsgSjok0CUwCcbyqisrQ93OsYY0+VEbIFIjY9ifF4m9Y3K+2t3hTsdY4zpciK2QAB8dWRvAN5eZc1MxhjTUkQXiHNGOoe7fvD5LmrrG8OcjTHGdC0RXSD6ZSQxsk8qVXWNLNhQFu50jDGmSwlpgRCRW0RkpYisEpGfuNMyReQdEVnn3me400VE7hOR9SKyXETGhTK3Zs3NTG+ttGYmY4zxFbICISKjgBuBk4ExwHkiMgy4HZirqsOAue5zgK8Bw9zbdOD+UOXmq7mZ6d01O2lssgH8jDGmWSj3IEYAC1W1WlUbgA+Bi4ELgcfdeR4HLnIfXwg84V5TeyGQLiK5IcwPgOG9UhiYlcTuqjqKNpWHenXGGNNthLJArASmiEiWiCQBXwf6A71UdTuAe9/Tnb8vsNknfos7LaRE5FAzk43NZIwxB4lq6JpVROQG4GZgP7AaqAGuU9V0n3nKVTVDRF4D/qCq893pc4FfqGpRi2VOx2mCIjc3N3/27NkB5VZdXU1SUhIAa8vquOP9PfRMjuZfX8tGRPyKD3b9Fm/x3TEHi+++8QUFBUWqWtDujKraKTfg98BNwOdArjstF/jcffwgcKXP/AfnO9otPz9fA1VYWHjwcWNjk+b/9h0deNscXbW1wu/4YNdv8RbfHXOw+O4bDxSqh9/tUB/F1NO9HwBcAjwDvApMc2eZBsxyH78KXOMezTQRqFC3KSrUoqKEs90hwK2ZyRhjHKE+D+I/IrIamA3crKrlwL3A2SKyDjjbfQ7wOlAMrAcextnb6DTNRzPZ4H3GGOOICeXCVfW0VqbtBs5qZbri9FeExaQhWfSIj2HN9ko276mmf2Zw7cPGGNPdRfSZ1L7iY6I5Y3gOYM1MxhgDViAOY4P3GWPMIVYgfJwxPIe46CgWb9pD2f4D4U7HGGPCygqEj5SEWCYNzUIV5q6xvQhjTGSzAtHCobOqrUAYYyKbFYgWvjKiFyIwf10Z+w80hDsdY4wJGysQLeSkxJM/IIO6xiY+/Lw03OkYY0zYWIFohQ3eZ4wxViBa1XxW9ftrd1HX0BTmbIwxJjysQLRiYFYyx/dOYd+BBj4p3h3udIwxJiysQBzFOdbMZIyJcFYgjuIcd3TXd1bvpMkuRWqMiUBWII5iZJ9U+qYnUrrvAEs27w13OsYY0+msQByFiBwaAtyamYwxEcgKRBt8D3fVEF6a1RhjuiIrEG0oGJhBZnIcG3dXs27X/nCnY4wxncoKRBtioqM46/iegDUzGWMijxWIdtjgfcaYSGUFoh2Th2WTFBfNiq0VbN1bE+50jDGm01iBaEdCbDSnH+dcivQda2YyxkQQKxAeWDOTMSYSWYHwYOrwnsRECZ9u3EN5VV240zHGmE5hBcKDtKRYThmSRWOTMnftrnCnY4wxncIKhEc2eJ8xJtJYgfDo7BHOsBvz1pVSU9cY5myMMSb0QlogROSnIrJKRFaKyDMikiAig0RkkYisE5HnRCTOnTfefb7efT0vlLn5q3daAmP7p1Nb38SHX9ilSI0xx76QFQgR6Qv8GChQ1VFANPAt4I/AX1R1GFAO3OCG3ACUq+pQ4C/ufF2KDd5njIkkoW5iigESRSQGSAK2A2cCL7qvPw5c5D6+0H2O+/pZIiIhzs8vzYe7zl27iwa7RoQx5hgXsgKhqluBPwNf4hSGCqAI2KuqDe5sW4C+7uO+wGY3tsGdPytU+QViSE4PhvbsQUVNPatL7XBXY8yxTUI1jLWIZAD/Aa4A9gIvuM/vdpuREJH+wOuqeqKIrAK+qqpb3Nc2ACer6u4Wy50OTAfIzc3Nnz17dkD5VVdXk5SU5Hfc0yv28dLaKs4eGMv3Tw68fgW6fou3+K6Sg8V33/iCgoIiVS1od0ZVDckNuAyY6fP8GuB+oAyIcaedArzlPn4LOMV9HOPOJ22tIz8/XwNVWFgYUNzSL8t14G1zdNyvX9empqZOX7/FW3xXycHiu288UKgefsdD2QfxJTBRRJLcvoSzgNXA+8Cl7jzTgFnu41fd57ivv+e+kS5ldL80eqcmsLumyS5Faow5poWyD2IRTmfzZ8AKd10PAbcBt4rIepw+hpluyEwgy51+K3B7qHILhohw/phcAB6ZXxLmbIwxJnRiQrlwVb0buLvF5GLg5FbmrcVpluryrp88iEfnl/D6iu1sLKsiLzs53CkZY0yHszOpA5CblsiUgYk0KTw0rzjc6RhjTEhYgQjQhcOTEYEXC7ewq7I23OkYY0yHswIRoH6pMZxzQi/qGpt45OON4U7HGGM6nBWIIHz/9CEAPL1wE5W19WHOxhhjOpYViCCcNCCDUwZnse9AA08t3BTudIwxpkNZgQjSTVOdvYhH5pdQW2/DgBtjjh1WIII0eWg2o/qmUra/jheKtoQ7HWOM6TBWIIIkIvzg9KEAPPTRBhoam8KckTHGdAwrEB3g3FG9yctKYvOeGl5bsT3c6RhjTIewAtEBoqOE77lHNN3/wQa64BBSxhjjNysQHeSScX3pmRLP2h37+MAuSWqMOQZYgegg8THR3DB5EODsRRhjTHdnBaIDfXvCAFITYvi0ZA9Fm/aEOx1jjAmKFYgOlJIQyzWn5AG2F2GM6f6sQHSwa0/NIz4minfX7OLzHfvCnY4xxgTMCkQHy+4RzxXj+wPw4Ie2F2GM6b6sQITAjacNJjpKmLVsG1vKq8OdjjHGBMQKRAj0z0zi/NG5NDYpM+bZZUmNMd2TFYgQ+f4Zzolzzy7+kt37D4Q5G2OM8Z8ViBA5vncqZx7fk9r6Jh5fsDHc6RhjjN+sQITQD9y9iMc/2cT+Aw1hzsYYY/xjBSKExudlMj4vg4qaep5Z9GW40zHGGL9YgQix5r2IGfOLOdBgFxQyxnQfViBCbOrwngzvlcLOygO8smRruNMxxhjPrECEmIgc3It48MNiGptsKHBjTPdgBaITnDc6l34ZiRSXVfH2qh3hTscYYzwJWYEQkeEistTnVikiPxGRTBF5R0TWufcZ7vwiIveJyHoRWS4i40KVW2eLiY5i+pTBANz/oV1QyBjTPYSsQKjq56o6VlXHAvlANfAycDswV1WHAXPd5wBfA4a5t+nA/aHKLRwuy+9PVnIcy7dUsGDD7nCnY4wx7eqsJqazgA2qugm4EHjcnf44cJH7+ELgCXUsBNJFJLeT8gu5xLhorjs1D7ChwI0x3YN0RnOHiDwCfKaq/xCRvaqa7vNauapmiMgc4F5Vne9OnwvcpqqFLZY1HWcPg9zc3PzZs2cHlFN1dTVJSUkBvqPA4qvqmvjea6XUNCi/OTWZUX1SOnX9Fn/sxHeFHCy++8YXFBQUqWpBuzOqakhvQBxQBvRyn+9t8Xq5e/8aMNln+lwgv61l5+fna6AKCwsDjg0m/vevrdaBt83RK+57Jyzrt/hjI74r5GDx3TceKFQPv99+NzGJSIaIjPYj5Gs4ew873ec7m5uO3Ptd7vQtQH+fuH7ANn/z6+qunzyIuOgoFm09QNGm8nCnY4wxR+WpQIjIByKSKiKZwDLgURH5P4/ruBJ4xuf5q8A09/E0YJbP9Gvco5kmAhWqut3jOrqNXqkJXHlyfxS4euYi5q0rDXdKxhjTKq97EGmqWglcAjyqqvnAV9oLEpEk4GzgJZ/J9wJni8g697V73emvA8XAeuBh4CaPuXU7d553AlMGJFBd18j1jy1mzvJjbkfJGHMMiPE6n9scdDlwh9eFq2o1kNVi2m6co5pazqvAzV6X3Z3FRkfxo5PTGNo/l0c+LuFHzyyhvLqeqycODHdqxhhzkNc9iN8AbwHrVXWxiAwG1oUurWNflAh3nTeCn391OKpw1ysr+eu7X9hJdMaYLsPrHsR2VT3YMa2qxX70QZijEBFunjqUzOQ47nh5BX99dx3lVXXcff5IoqIk3OkZYyKc1z2Iv3ucZgJw5ckD+NdV44iLjuLxTzZxy3NLqWtoCndaxpgI1+YehIicAkwCckTkVp+XUoHoUCYWac4dlctj18cy/YkiZi/bxt7qOh74Tj7J8V538owxpmO1twcRB/TAKSQpPrdK4NLQphZ5Jg3J5tnpE8lKjmPeujKumrGI8qq6cKdljIlQbW6equqHwIci8pg64yiZEBvVN40XfzCJq2cuYunmvVz24Cc8cf3J9ElPDHdqxpgI47UPIl5EHhKRt0XkveZbSDOLYIOyk/nPDyYxvFcK63ft59L7F7B+1/5wp2WMiTBeC8QLwBLgTuDnPjcTIr1SE3j+e6eQPzCDbRW1XPbAApZu3hvutIwxEcRrgWhQ1ftV9VNVLWq+hTQzQ1pSLE/dMIEzj+9JeXU93354oQ3NYYzpNF4LxGwRuUlEct0rwmW64zKZEEuMi+bBq/O55KS+NjSHMaZTeT2GsnlwPd9mJQUGd2w6pjWx0VH8+bIxZCTHMXO+MzTHDWNTyM8Pd2bGmGOZpwKhqoNCnYhpW1SUcOc3RpDVI44/vfk5M5bso2j3x9x42mC+OrI30XbmtTGmg3kqECJyTWvTVfWJjk3HtEVEuOmMoWT3iOeeWStY8uVebnr6M/pnJnLdpEFcPr4/PezEOmNMB/H6azLe53ECzmisnwFWIMLg8oL+9GncwYbGHB75uIRNu6u5Z85q/vLuF3x7wgCunZRHbpqdN2GMCY7XJqYf+T4XkTTgyZBkZDxJjIli2oQ8vjNxIO+s3snM+cUs3ljOgx8WM3NeCeeNzuW7pw1mVN+0cKdqjOmmAm2PqAaGdWQiJjDRUcK5o3pz7qjeLN28lxnzinlj5Q5eWbqNV5ZuY+LgTG48bTBTh/e0EWKNMX7x2gcxG+eoJXAG6RsBPB+qpExgxvZP5x/fHseW8moe+3gjzy7ezMLiPSws3sPgnGRumDyIS07qR2KcjbNojGmf1z2IP/s8bgA2qeqWEORjOkC/jCTuPO8EfvyVYTy/eDOPfryR4tIq7nh5JX9+63OunjiQMcmN4U7TGNPFee2D+FBEenGos9quJtcNpCbE8t3TBnPtpDxeX7mDGfOKWb6lgvveW09cNDyZu5sJg7PaX5AxJiJ5OpNaRC4HPgUuw7ku9SIRseG+u4mY6CguGNOHWTefygvfP4Upx+VQ1wi/+M9yaupsT8IY0zqvQ23cAYxX1Wmqeg1wMnBX6NIyoSAijM/LZMY1BQxIjWHT7mr+OveLcKdljOmivBaIKFXd5fN8tx+xpouJi4nipvGpiMCMeSWs3FoR7pSMMV2Q1x/5N0XkLRG5VkSuBV4DXg9dWibUhmXGcd2kQTQ2Kb94cTn1jXYNbGPM4dosECIyVEROVdWfAw8Co4ExwCfAQ52Qnwmhn331OPplJLJ6eyUz5pWEOx1jTBfT3h7EX4F9AKr6kqreqqo/xdl7+GuokzOhlRQXw+8vPhGAv777BSVlVWHOyBjTlbRXIPJUdXnLiapaCOS1t3ARSReRF0VkrYisEZFT3GtJvCMi69z7DHdeEZH7RGS9iCwXkXEBvSPjlynH5XDJuL4caGjily8tR1XbDzLGRIT2CkRCG695GQ3ub8Cbqno8TtPUGuB2YK6qDgPmus8BvoYzfMcwYDpwv4flmw5w1zdOICs5joXFe3h28eZwp2OM6SLaKxCLReTGlhNF5AagzUuOikgqMAWYCaCqdaq6F7gQeNyd7XHgIvfxhcAT6lgIpItIrud3YgKWkRzH3ReMBOD3r69hZ2VtmDMyxnQF0laTgnv29MtAHYcKQgEQB1ysqjvaiB2L05G9GmfvoQi4Bdiqquk+85WraoaIzAHuVdX57vS5wG1uc5bvcqfj7GGQm5ubP3v2bP/esau6upqkpKSAYo/FeFXlDx/vpWj7ASb0jecXkzI6df0W3/1ysPjuG19QUFCkqgXtzqiq7d6AqcCP3NuZHmMKcMZtmuA+/xvwW2Bvi/nK3fvXgMk+0+cC+W2tIz8/XwNVWFgYcOyxGr+1vFpH/upNHXjbHH1jxbZOX7/Fd68cLL77xgOF6uF33NN5EKr6vqr+3b2957FIbQG2qOoi9/mLwDhgZ3PTkXu/y2f+/j7x/YBtHtdlOkCf9ERuO3c4AHfNWkVFdX2YMzLGhFPIzoZWp/lps4gMdyedhdPc9CowzZ02DZjlPn4VuMY9mmkiUKGq20OVn2ndVRMGUjAwg9J9B/jDG2vCnY4xJoxCPVzGj4CnRWQ5MBb4PXAvcLaIrAPOdp+Dc25FMbAeeBi4KcS5mVZERQn3fvNE4qKjeHbxZhZsKAt3SsaYMAlpgVDVpapaoKqjVfUiVS1X1d2qepaqDnPv97jzqqrerKpDVPVEbdE5bTrP0J4p/PDMoQD88qUVNuKrMRHKBtwzrfr+6UMY3ivFRnw1JoJZgTCtiouJ4t5vnmgjvhoTwaxAmKM6aUCGjfhqTASzAmHa9F/nHEffdBvx1ZhIZAXCtCk5PobfX2IjvhoTiaxAmHadflwOl5xkI74aE2msQBhP7jrv0Iivz9mIr8ZEBCsQxpOM5Dh+df4JAPzu9TXsqbFzI4w51lmBMJ5dMKYPZx7fk321DcxYUhnudIwxIWYFwngmIvz2olEkx0WzaOsBLvrnx8ycX8KOCrt+hDHHIisQxi990xP5wzdHEx8tLN28l9/OWc0p987l8gc+4YlPNlK670C4UzTGdJCYcCdgup8LxvQhq3Yr5Yl9mb1sG+9/XsqnG/fw6cY9/PrVVZwyJIvzR/fh3FG9SU+KC3e6xpgAWYEwAUmIieK80X04b3Qf9tXW8+6ancxetp1560r5eP1uPl6/mztfWcnkYdmcP7oPZ4/sRWpCbLjTNsb4wQqECVpKQiwXn9SPi0/qR0V1PW+t2sHs5dtYsGE3H3xeygeflxL3UhSnD8/h/DF9OOv4nuFO2RjjgRUI06HSkmK5fHx/Lh/fn937D/DGyh3MWb6NRSV7eGf1Tt5ZvZOE2CjG9Yrj7r77GN47JdwpG2OOwgqECZmsHvF8Z+JAvjNxIDsra3l9xXbmLN9O0aZyFmyp5ev3zePaSXnc8pVh1vxkTBdkRzGZTtErNYHrTh3Ef34wifm3TeWrQxJpUmXm/BLO/POHvPTZFhvCw5guxgqE6XT9MpKYPi6N2T+czLgB6ZTtP8Ctzy/j8gc/YfU2OwHPmK7CCoQJm1F903jx+5P4n0tHk5Ucx+KN5Zz393n8+tVVVNTUhzs9YyKeFQgTVlFRwmUF/XnvZ2dw7aQ8AB5bsJEz//wBzxdupqnJmp2MCRcrEKZLSEuM5dcXjOS1H5/G+LwMdlfV8YsXl/PNBxbY5U6NCRMrEKZLGZGbyvPfO4W/XDGGnJR4lny5l/P/MZ87X1nB3uq6cKdnTESxAmG6HBHh4pP68d5/nc53Jw8iSoSnFn7J1D9/wDOffmnNTsZ0EisQpstKSYjlzvNO4I1bTmPi4EzKq+v55UsruPhfH7N+j3ViGxNqdqKc6fKO65XCMzdOZPby7fzutdUs21LBsi3w6KqPmHJcDlOG5VCQl0FCbHS4UzXmmBLSAiEiG4F9QCPQoKoFIpIJPAfkARuBy1W1XEQE+BvwdaAauFZVPwtlfqb7EBEucMdxuu+9dTw2v4S1O/axdsc+HvqomITYKCYOzmLKsBymHJfDkJxknK+UMSZQnbEHMVVVy3ye3w7MVdV7ReR29/ltwNeAYe5tAnC/e2/MQcnxMfzyayOYmlVFU2YeH64r5aMvylizvfLgwIDgXLdiynHZnDYsh1OHZpOWaEN5GOOvcDQxXQic4T5+HPgAp0BcCDyhzngLC0UkXURyVXV7GHI0XVxstJA/NJtJQ7P55ddg175a5n1RxkfrSpm3royte2t45tPNPPPpZqIExvZPd5qjjsthTL/0cKdvTLcgoRz/RkRKgHJAgQdV9SER2auq6T7zlKtqhojMAe5V1fnu9LnAbapa2GKZ04HpALm5ufmzZ88OKLfq6mqSkpICirX4rh3fpErJ3gaW7jjA0h0H+Hx3PY0+X/MescKonGimjc2gZ3Jg/Rbhfv9dIQeL777xBQUFRapa0O6MqhqyG9DHve8JLAOmAHtbzFPu3r8GTPaZPhfIb2v5+fn5GqjCwsKAYy2+e8VX1tTpWyu36x0vL9fT/vieDrxtjg68bY6eeu9c3VpeHfL1hyK+K+Rg8d03HihUD7/hIW1iUtVt7v0uEXkZOBnY2dx0JCK5wC539i1Af5/wfsC2UOZnIkNKQiznjOzNOSN7A1BSVsWNMz9mfXkNVz68kOemn0LvtIQwZ2lM1xOy8yBEJFlEUpofA+cAK4FXgWnubNOAWe7jV4FrxDERqFDrfzAhMCg7mbumZDCqbyqbdlfz7RkL2bWvNtxpGdPlhPJEuV7AfBFZBnwKvKaqbwL3AmeLyDrgbPc5wOtAMbAeeBi4KYS5mQjXIy6KJ6+fwPG9UygureKqhxdRtv9AuNMypksJWROTqhYDY1qZvhs4q5XpCtwcqnyMaSkjOY6nvzuBbz20kHW79vOdGYt45saJZCTHhTs1Y7oEG2rDRLSsHvE8feMEBucks3bHPr4zcxEV1TaMhzFgBcIYeqYk8MyNE8nLSmLVtkqueWQRlbVWJIyxAmEMzjWz/33jRPpnJrJsSwXXPvIp+w80hDstY8LKCoQxrj7pifz7uxPpm57IZ1/u5fpHF1NdZ0XCRC4rEMb46J+ZxL9vnEDv1AQ+3biHGx4rpKauMdxpGRMWViCMaWFgVjLPTJ9Iz5R4PinezfQnC6mttyJhIo8VCGNaMSg7mX/fOIHsHnHMW1fGD54q4kCDFQkTWaxAGHMUQ3um8PR3J5KZHMf7n5dy89NLqGtoCndaxnQaKxDGtGF47xSeumECaYmxvLtmJz9+Zgn1jVYkTGSwAmFMO07ok8pTN0wgJSGGN1ft4Nbnl9HYFLph8o3pKqxAGOPBif3SeOL6k+kRH8PsZdv4x+IKtlfU0GSFwhzDwnFFOWO6pZMGZPD49eO5euanfPRlLaf84T0SY6PJy05mcHYyg7KTGZzj3mf3IC3JLnNqujcrEMb4IX9gJk/ecDK/erGQHdXC7qo61myvZM32yiPmzUyOc4tFMoNymotIDwZmBXclOWM6ixUIY/yUPzCTe87IIj8/n4rqekp2V1FStp+S0io2lFVRUlpFSVkVe6rq2FNVR9Gm8sPiRSA7MYrjly46uOcxKKcHg7OT6ZOeSHSUhOmdGXM4KxDGBCEtKZaxSemM7Z9+2HRVZWflAYrL9lPiUzSKy6r4ck81pdVNlK4rY966ssPi4mKiyMtKcopGdg8G5xxqvspMjkPEiofpPFYgjAkBEaF3WgK90xKYNCT7sNfqG5t4Y95iknvlHSwaxaVOIdlZeYAvdu7ni537gZ2HxaUmxDAopwdDspMZkJVERVkV6xq/JDEumqS4GBJjo93Hzs13uu2VmEBYgTCmk8VGR9E3JYb8Eb2OeG3/gQY2ukXD2etwCkdxaRWVtQ0s27yXZZv3HgpYusLTOuNjotzCEUNCbBSpibEMT2mg95Aa+qYndtRbM8cYKxDGdCE94mMY1TeNUX3TDpuuqpTtr3Oaq8r28+Weako2b6dHeibVdY3U1DVSXddIdX0jNXUNh02rqW/kQEMTBxqaKPe5GNIS4LnV73HasBwuL+jH2Sf0Ij4mupPfsenKrEAY0w2ICDkp8eSkxHPyoEwAioqqyM8/4qq+R1BVauubqG4uHPWNbCmvZubclSzeVsdHX5Ty0RelpCfFctHYvlxW0I+RfdLaXa459lmBMOYYJyIkun0SWe6043qlkFa1hSEjTmTW0m08t3gzq7dX8tiCjTy2YCMj+6Ryxfj+XDimr53PEcGsQBgTwdKT4pg2KY9pk/JYubWCFwo388rSbazaVsmvZq3iv19bw1dH9ubygn6cOiSbKOvsjihWIIwxAAf7Pn759RG8s3onzxduZv76MmYv28bsZdvom57Ipfn9uDS/H/0z7WS/SGAFwhhzmITYaM4f04fzx/RhS3k1/ynaygtFm9lSXsPf5q7jb3PXccrgLHrF1rKmfhP9M5Pon5FIn/REEmKtk/tYYgXCGHNU/TKSuOUrw/jRmUNZWLyb5ws388bKHXxSvBuAVz5fedj8vVLj6Z+RRL+MRLdwJNEvM5H+GUnkpiUQE23jg3YnViCMMe2KihImDc1m0tBsflNTzwef72LhivU0JqazeU8Nm8ur2V5Ry87KA+ysPEBhi+FFAKKjhNy0BPplJNI3PYnKvRVkb/J2HscR+QjE1lZRl76bE/qkkpZoHemhEPICISLRQCGwVVXPE5FBwLNAJvAZcLWq1olIPPAEkA/sBq5Q1Y2hzs8Y45+0xFguHNuXfo07DjvMtqGxie0VtWwur2ZLeQ1b9lSzubyGzXuc5zv31TrTy2uAPU70ReVSAAAUiUlEQVRQ8ZdB5fLYsoUA9M9MZFSfNEb2SWWke98zNSGoZZvO2YO4BVgDpLrP/wj8RVWfFZEHgBuA+937clUdKiLfcue7ohPyM8Z0gJjoKKdZ6Sgd2AcaGtnqFojtFTUUl2xiwMABAa2rrqGJBatK2FUXx5od+5y9mD01vLFyx8F5snvEuwXDKRqj+qbSPyPJjsTyQ0gLhIj0A74B/A64VZyRxs4Evu3O8jjwa5wCcaH7GOBF4B8iIqpqV2Qx5hgQHxPN4JweDM7pAUBRVCn5+QMDXt7ohD3k5+fT0NjEhtIqVm2rYNW2SlZurWD19krK9h/gwy9K+fCL0oMxKfExjOiTyqg+aYxKric/6Hd1bJNQ/v6KyIvAH4AU4GfAtcBCVR3qvt4feENVR4nISuBcVd3ivrYBmKCqZS2WOR2YDpCbm5s/e/bsgHKrrq4mKSnwQ/Us3uLDGd8VcujK8arKzqpGSvY2ULK3npJy57689tD1xAU4Z0gi3xqZQmq8/53nXfn9t6egoKBIVQvanVFVQ3IDzgP+5T4+A5gD5ADrfebpD6xwH68C+vm8tgHIamsd+fn5GqjCwsKAYy3e4sMd3xVy6I7xuypr9f21O/WOl5froNvn6MDb5uiJd7+pM+cVa11DY8jX31XigUL18DseyiamU4ELROTrQAJOH8RfgXQRiVHVBqAfsM2df4tbMLaISAyQxsGeLGOMCV5OSjxnDO/JGcN7kp9axUslwrx1ZdwzZzVPL9rEneedwNThPcOdZpcRsoOSVfWXqtpPVfOAbwHvqepVwPvApe5s04BZ7uNX3ee4r7/nVjpjjOlwA9JieeL6k5k5rYBB2clsKK3iukcXc+2jn7J+1/5wp9clhOOsldtwOqzXA1nATHf6TCDLnX4rcHsYcjPGRBAR4awRvXjrJ1O44+sjSImP4YPPSzn3rx9xz+zVVNTUt7+QY1innCinqh8AH7iPi4GTW5mnFrisM/IxxhhfcTFR3DhlMBeP68v/vv05zy7ezCMfl/DK0q3cevZxXHnygIi8Kp+d926MMa7sHvH84ZLRzP7hZE4elMmeqjrufGUl37hvHgvWl7W/gGOMFQhjjGlhVN80nps+kX9dNY5+GYms3bGPb89YxPeeLOTL3dXhTq/T2FhMxhjTChHh6yfmcubxPZk5v4R/vr+et1bt5P21pVw/eRBD4+oZUddAUtyx+zN67L4zY4zpAAmx0dw8dSiX5vfjj2+u5aXPtvLAhxsA+Nk7b5GblsCg7OSDtyE5PRiUnUy/jMRuP3qtFQhjjPGgV2oC/3f5WK6eOJAZ80pYtnEXO6udAQq3V9SyYMPuw+aPiRIGZCUxODuZwW7RGJSdzODsZHJS4sP0LvxjBcIYY/xw0oAM/nlVBkVFRYwZexLb9tayoWw/JaVVlJQdum3dW0NxaRXFpVWwZtdhy0iOiyYjQRi0dBG9UxPonZZAr9SEg497pyWQmRQX9oEFrUAYY0yAYqKjGJCVxICsJKYOP/y1mrpGNu4+VDA2lO6npMwpGBU19VTVwZbKox8ZFRst9ExJIDctgV5pbvHwKSDlNY0hfndWIIwxJiQS46IZkZvKiNzUI14rr6rjvYWfkdl3MDsqnSaqnRW17KisZWelc7+3up6te2vYurem1eUX5MZzzuTQvgcrEMYY08kykuPIS48l//ijj/tUU9d4sFjsdIvIjopDBWRgcujP8rYCYYwxXVBiXDR52cnkZSe3+npRUVHIc+jex2AZY4wJGSsQxhhjWmUFwhhjTKusQBhjjGmVFQhjjDGtsgJhjDGmVVYgjDHGtMoKhDHGmFaJqoY7h4CJSCmwKcDwbCCYS0RZvMWHM74r5GDx3Td+oKrmtDuXqkbkDSi0eIvvrvFdIQeL797xXm7WxGSMMaZVViCMMca0KpILxEMWb/HdOL4r5GDx3Tu+Xd26k9oYY0zoRPIehDHGmDZYgTDGGNMqKxDGGNMNiMgPRSSjM9dpV5SLACJyvKquFZFxrbyswB5VDfSEQ39z6QsMxOe7p6ofdca6w0lEooG3VPUrQcTfq6o/79jMIoeIDFLVkvamdWG9gcUi8hnwCM73KaSdyBHVSS0ipwK/5tAPlACqqoM9xg8D/gCcACQ0T/cjPge4Ecjj8B/I60OZv4g8pKrTReT9o8ySBSxT1avbWU4CcAMwksPfv9f8/whcAawGGg+F6wUe44P9+10C/BHo6cY2xx95VfnW43OA2zjy73+mx/hXgatVtcLL/K3EvwecFcyPgohMBP4OjADigGigqq3PQET24WxItCrUn5+I/EJV/yQif28tD1X9scf1f6aq41pMK1LV/HbivqOqT4nIra28rMAe4FVVLT9KfId8fu6yBDgHuA4oAJ4HZqrqBq/L8Eek7UHMBH4KFHHoB8ofjwJ3A38BpuL8kcSP+FnAPODdANcfUP6qOt29n3q0eUTkbQ+LehJYC3wVuAe4CljjNQ/gImC4qh7wI8ZXsH+/PwHnq6o/Oft6GngO+AbwfWAaUOpHfC2wQkTeAaqaJ3r9gQOWALNE5IUW8S/5kcM/gG8BL+D8wFwDDG0rQFVTAETkHmAHzvdAcP7+KX6sO9DPr/nvVejHug4SkeNxNmrS3I2EZqn4FKo2NF8U+mjvdRDwA2Biay924OeHqqqI7HCX0wBkAC+KyDuq+gt/luVFpO1BLFLVCUHEF6lqvoisUNUT3WnzVPU0j/FLVXVsEOsPKn93GaM4cgvuCY+xS1T1JBFZrqqjRSQWZzfX6xb0G8Blqro/wNyD/ft9rKqnBhHf/Pdfrqqj3WkfqurpHuOntTZdVR/3GP9o6+He9uDcZRSqakGL97BAVSd5iD3i8/fnbxLs5xcoEbkQZ+PkAuBVn5f2Ac+q6oIOWMc9qvqrduYJ9vP7MU5RLQNmAK+oar2IRAHrVHVIgOkfVUTsQfi0vb8vIv8DvAQc3IpV1c88Lqq2+Y8hIj8EtuI0V3g1R0S+rqqv+xHTYfmLyN3AGTgF4nXga8B8wFOBAOrd+71uodmB01zmVTWwVETmcnj+Xregg/37FYrIc8ArLeK9boE3v//tIvINYBvQz2Os50LQRvx1wcS7qkUkDufv8CdgO4e2kNvTKCJXAc/iNJlciX97ckF9fiJyHPAzjmyibXMDRVVn4ex5naKqn/iRb8v1H7WJuL3i4Ar288sGLmnZX6iqTSJynh/L8Swi9iDaaHsHZwvM6xbweJzd3XTgt0Aa8CdVXdhOXHMbpOD8Mx7A+Wfx1AbegfmvAMYAS1R1jIj0Amao6vke478L/Ac4EXgM6AHcpaoPeowPdgu6tc/Bn/cf1Ba4+084D+iP046fCvxGVV9tJ24FbbdBj/a4/qD6gNxlDAR24vQ//BTnO/xPL23YIpIH/A04Fef9fAz8RFU3elx3QJ+fT/wy4AFaNDGqapHH+GD7ABe4+bdc/388xucRxOfns5yeHP73/9KfeL/WFQkFwjhEZLGqjheRIpw+lH3ASlUd6SE2CrhUVZ8PdZ5dkXsU0Y9V9S8BxA50H97s3j/p3l8FVKvqPR6X8wJOH9C38ekDUtVb/MjlFlX9W3vTuiIvHcrtxAf7Ax9UE3GwROR84P+APsAunIM11nj5/w14nZFUIETkFpyO5n3Aw8A44HZV9dJB27yL+3OOPEzT6xbsqcBSVa0Ske+46/+r1y2AYPJ3j36YAfwXTiflfwH73Xw8NV2IyEeqOsXLvEeJD+ooMHcZ3+DILWivP7DBHoX1flsd/R7ij+gD8adfJNg+IHcZrR3Js0RVT/IQG+wWeEDxIpLpPvwxzg/jyxzeRLjH4/qD7QP8b2CBv03EPvHBfn7LgDOBd93vwVTgyuaDUEIhIvogfFyvqn8Tka/i9B1ch/OD66lA4Bz58QDOj3MgR9HcD4wRkTHAL3COynkS8NpJF3D+7tEPY1V1L/CAiLwJpKrqcj/yf0dEfoZzJIrvUTSe/kEJ8igwEXkASHJjZwCXAp96jSf4o7AWiMg/OPL9e+0DSRaRyao6Hw5uMHht/4cg+oBE5EqcPY9B4hxu2ywF2O1x/cEehRdofBGHmmjB2Ujz3bL1uoERUB+gj1uA/ycifjUR+wj286tX1d0iEiUiUar6vjiHjoeOhviCE13pBix37+8DLnYfL/EjvijI9X/m3v8KuMF3mp/5/y3A/P8JjA8i/xKguOXN388PWOEzbV4A77/5vgfwth/xS1rExwLv+RH/vnt7z72972d8PrAM2Oh+lkuBk/yI/y7OYY1T3M9+F/A9j7EDcQ5Q+ARng6T5Ng6I8biMpYF+dzoo/nKcjRqAu3D2JMb5Eb8PaAJqgEr3eaWfOWQCE3w/w058/++63/l/AM+4vwMLgllme7dI24MoEpG3cLY4bheRFJwvTJt8dnFni8hNBLiLC+wTkV8C3wGmuO3asX7m/zbOcde/9Jq/j6nA90RkE84WcPMWkKdOUpymoZuAyThbcPNw9qi8CvYosFr3vlpE+uCcoDTIj/iAtsDl0AlSczh8Sxba6HxuxUqcczGG4PzQ7wXOxzm/wYsngW/i5Nzcsd/LS6A6R75sAk7xnu4Rgt0CDzb+TlV9XkQmA2cD/4uzV+7pMFFVTXH/l4fh7fyHw7gHadyCc+TVUpzzHhYAZ3lcRLDv/wKc/4FbcH5DUoHfBLgsTyKtDyIKuBPIUNWfisgAnGuzzmsnroQ2fhjU+5m8vXF28xer6jx3/Weo9/MQooCxOEUlHuewt76q+neP8QNbm64eh9kQkedxtryediddCaSr6uUe41seBZaKcxTYIo/xd+Ec/XIWzt6QAg+rt0MMAz4Kyz08GGA4MB6nqUBwftw/UtXvelz/mzhF4TMO7yT9Xz/iKziyk7XdeBGZr6qT5cizej03k7ixSUAdATSxuPF+H8XnE9/cB/MHnL3Qf3vtP3HjW/2BV1VPP/Du0WjjgYWqOlacE/B+o6pXeIwP6P0f5W/X/FvUhLOh9D+q+i8vefgj0grE/Tgf6JmqOkKcga/eVtXxHuMvB95U1Ur3x2oc8Fv13gYdlKN8wT9RPzopg1z/MlUd0960NuILgDtwmjua95w878GIyGU4n/++QD5/EYnn0Ba47/q9dnK/DXxTVfe5z1OAF1T1XI/xK1V1lJd5QxEfLHcD5SpgkKre427g5Hot8O4yjtiCV9UPPcbOwdnr/ApOc10N8Kkf379gf+CbjwJcCkxQ1QP+dnwH8/7bWGYWTqEbHsxyWhNpo7lOUNWbcZsq1Bk7Jc6P+Dvd4tC8i/sYzi5um0SkuVNyn4hU+tz2iUilH+u/BecLvkmdo2lOwr+hHoK1RJyxfAAQkQk4x3J79TROR/U3gfPcm6dzMFx3ucXBr8/fxyzgQpwhCva7t6o2Iw43AGfruVkd/p0ouEBETvRj/o6OD9Y/cTZKrnSf78NpD/fE3cD5EHgTZ0ytN3H647y6HHgLOFedgy0ycTqsvapV1Vo3l3hVXYuzV+jVFhFJxznR8h0RmYVzsp8nHfD+W6Wqu3H6lzpcpPVB1Lvt/goHDzvzpw2/ebf+G8ADqjpLRH7dXpCqTnbv/Rp3pRW1qlorIge/4CLS4VsNbZgAXCMizYflDgDWuFtmXvYEStXjSVFHEdDn76Of1639o3gS+FREXsb5Dl3Mob4ALyYD17pNlgfw2Ackh060iwGuE5Fif+I70ARVHSciS3BWXC7OWdleNW/gLFTVqc1b8F6DVbUa5yz65ufbcc4E96rlD3w5fvzAq+rF7sNfi3PSZhrOj7xXQb3/dnLz53PwLNIKxH04Hcw9ReR3OIdJ3ulH/FYReRBnF/ePbpNFZ+6FBfUF7wDB/LgC3C0iM4CWQ214Heoi2M9/gYicqKor/Ig5SFV/J854Us1jb12nql47mMEZ2iQQIRlGIQDBbmCFdQOnA37gfZcVSLNQuDfw/BZRfRAAbtU+C2fra676MbKniCTh/EiuUNV1IpILnKgeT7TrSCJyOu4XXFXr2pu/KxCRp4DjgVUc+mFR9X6iUECff4st8GE4h4iGYwu8WxNnHKErcPp+HsfdwFLVFzzGv4xz7stPcE74KgdiVfXrocm4a+mO7z/iCoQJH/EZBbeT19vq0VvNvB7FZYLbwGqxnG63gdORusv7twJhOo2IPAz8RVVXhzsXY0z7rECYTiMia3BOEvOrk9YYEx5WIEynCfZEPWNM57ICYYwxplWRdqKcMcYYj6xAGGOMaZUVCGNcInKHiKwSkeUistQdSiRU6/rAHZvKmC4r0s6kNqZVInIKzhnL49xB2LLxb5wuY445tgdhjCMXKFPVAwCqWqaq20TkVyKyWERWishDIiJwcA/gLyLykYisEZHxIvKSiKwT59KUiEieiKwVkcfdvZIX3bPBDyMi54jIJyLymYi8ICI93On3ishqN/bPnfhZGANYgTCm2dtAfxH5QkT+5Z7pCvAPVR3vDrOdyOHjItWpc43uB3BGir0ZGIUzIF+WO89w4CH3XI9KnAsuHeTuqdwJfEWda0UXAre6w0JfDIx0Y/87BO/ZmDZZgTAGUNX9ONcYmI4zhPpzInItMFVEFrnjOZ0JjPQJax6ZdgWwSlW3u3sgxUB/97XNqto8JPpTOCO6+pqIc6W+j8W5zsA0nOtlVOIMSz9DRC4BqjvszRrjkfVBGONS1UbgA+ADtyB8DxgNFKjqZndocd9LVTaPSNvk87j5efP/VssTjVo+F+AdVb2Sli+InIwz7tG3gB/iFChjOo3tQRgDiMhwERnmM2ks8Ln7uMztF7g0gEUPcDvAwbnQzvwWry8EThWRoW4eSSJynLu+NHWuX/wTNx9jOpXtQRjj6AH83b3eRgOwHqe5aS9OE9JGYHEAy10DTHOvY7GOFlfAU9VStynrGff6FuD0SewDZolIAs5exk8DWLcxQbGhNowJERHJA+aE8zrSxgTDmpiMMca0yvYgjDHGtMr2IIwxxrTKCoQxxphWWYEwxhjTKisQxhhjWmUFwhhjTKusQBhjjGnV/wf8pj2DRHnf+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x25a6a762a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fd_fntrigrams=FreqDist(firstname_trigrams)\n",
    "fd_fntrigrams.plot(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exploring surname trigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "surname_trigrams = get_ngrams(df['Surname'][surnames_mask], 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sin    1533\n",
       "ing    1461\n",
       "ngh    1443\n",
       "sha     623\n",
       "har     595\n",
       "cha     586\n",
       "han     585\n",
       "jai     495\n",
       "and     494\n",
       "ain     475\n",
       "pat     449\n",
       "wal     429\n",
       "rma     406\n",
       "mar     378\n",
       "ari     373\n",
       "uma     317\n",
       "das     315\n",
       "kar     310\n",
       "bha     307\n",
       "kum     306\n",
       "dha     306\n",
       "ath     281\n",
       "ish     276\n",
       "gar     275\n",
       "awa     269\n",
       "hat     262\n",
       "ate     260\n",
       "ava     255\n",
       "pra     252\n",
       "man     242\n",
       "       ... \n",
       "eol       1\n",
       "jme       1\n",
       "msu       1\n",
       "ubu       1\n",
       "ipi       1\n",
       "edw       1\n",
       "nzo       1\n",
       "koy       1\n",
       "khm       1\n",
       "bus       1\n",
       "job       1\n",
       "gge       1\n",
       "tts       1\n",
       "gpu       1\n",
       "stv       1\n",
       "hdr       1\n",
       "pav       1\n",
       "hbh       1\n",
       "wel       1\n",
       "bro       1\n",
       "ntw       1\n",
       "rrm       1\n",
       "dob       1\n",
       "kkk       1\n",
       "gui       1\n",
       "hye       1\n",
       "uar       1\n",
       "tif       1\n",
       "hke       1\n",
       "vha       1\n",
       "Length: 2366, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(surname_trigrams).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8XHW9//HXZ7I2bZqkK+lCC3RhKRRI2GURkE0EVGQRL4twuT70KupPRa96cYEr7iAqi4ogIgi4QBHZKaVQaBMK3UsXWrqFNl2SNmmSJvn8/jgn6TRkmSXJTJL38/GYR2bOnM+cb5KZ+ZzveszdERERiVUk1QUQEZG+RYlDRETiosQhIiJxUeIQEZG4KHGIiEhclDhERCQuShwiIhIXJQ4REYmLEoeIiMQlM9UF6AkjRozwiRMnJhy/e/duBg0apHjFK17xAyq+vLy80t1Hdrmju/e7W0lJiSejrKxM8YpXvOIHXDxQ5jF8x6qpSkRE4qLEISIicVHiEBGRuChxiIhIXJQ4REQkLkocIiISFyWONrbuqk91EURE0poSR5QV7+/k5J+8xF8W7WRPU3OqiyMikpaUOKLMW7Oduj1N/G1pDZfcPYd122pTXSQRkbSjxBHl08ftz1/+83iGD4ow/70dnHf7Kzzx9sZUF0tEJK0ocbRx/IHD+flZIzj7sNHsrG/kSw/N52uPvk1NfWOqiyYikhaUONqRnx3hrs+UcMvHp5GTGeGx8vWcf8dsFm2oSnXRRERSTomjA2bGFcdNYMYXP8TB++XzbmUNH//tq/xu1mqamz3VxRMRSRklji5MGZ3PP79wEleeMIE9Tc4tTy3l6vvmsWWnhu2KyMCkxBGD3KwMfnDhNH53ZSlFeVnMemcL594+i5nLN6e6aCIivU6JIw4fOXQ0/77hFE44cDiVuxq4+o/zuPnJJdQ3NqW6aCIivUaJI077FeTy5+uO4+tnTyUjYvx+9rt84revsXrLrlQXTUSkVyhxJCAjYnzhw5N49HMnMH7YIBZvrOb8O2bzSNk6gotoiYj0X0ocSTh6/yL+9aWTuWD6GGobmvjGYwv44kPzWV/dSJNGXolIP5WZ6gL0dUNzs7j9siM5ZcpI/vfxRTy5YBNPLoBvvPA0U/fL5+D98jmkeCgH7zeUQ4rzKczLTnWRRUSSosTRDcyMi0vGUTKhiJ8+s4y5KzdTubuZBeurWLB+30mDxQW5e5NJ8VAO2S+fA0YMJjNDlT8R6RuUOLrRASMG89srSigvL2fSIUewrKKapZuqWVaxk6UVO1leUc2mqjo2VdXx0vItrXHZmRGmjB4S1kqGckBEK/OKSPpS4ughBXlZHHfgcI47cHjrtqZmZ+3WGpZV7GTZpmqWbNrJsopq1m/fzaIN1SzaUA3ASeNzOf3EVJVcRKRzShy9KCNiHDhyCAeOHMJ5hxe3bq+u28M7FTt5ZUUlt7+wgnXVWlBRRNKXGtbTwNDcLEonDuOqEycCUFmjCYUikr6UONJIUV4Wg7IyqG10qnbvSXVxRETa1WOJw8zuNbPNZraonee+ZmZuZiPCx2ZmvzKzlWa2wMyOjtr3KjNbEd6u6qnypgMzY2zRIAA2bN+d4tKIiLSvJ2sc9wHntN1oZuOBjwDvRW0+F5gc3q4H7gz3HQbcBBwHHAvcZGZFPVjmlBtbGCaOHUocIpKeeixxuPssYFs7T/0S+AYQPbX6QuBPHngdKDSzYuBs4Dl33+bu24HnaCcZ9Sd7axy63rmIpKde7eMwswuADe7+dpunxgLroh6vD7d1tL3fUo1DRNKd9eSifGY2EXjS3aeZWR7wEnCWu1eZ2Rqg1N0rzexfwI/cfXYY9wJBreR0IMfdbw63fxeodfeft3Os6wmauSguLi6ZMWNGwuWura0lLy8vJfGvvLeb296o4oRxOXzthMRa5VJZfsUrXvF9N760tLTc3Uu73NHde+wGTAQWhfcPBzYDa8JbI0E/x37A3cDlUXHLgWLgcuDuqO377NfRraSkxJNRVlaWsvh57271CTc+6Rfc8UpKjq94xSt+4MYDZR7Dd3uvNVW5+0J3H+XuE919IkGz09HuXgE8AVwZjq46Hqhy903AM8BZZlYUdoqfFW7rt1r7ONRUJSJpqieH4z4EzAGmmtl6M7u2k92fAlYDK4HfAZ8HcPdtwA+BeeHtB+G2fmtUfi4ZBpW7GtjdoImAIpJ+emzJEXe/vIvnJ0bdd+ALHex3L3BvtxYujWVEjBF5Gbxf08SGHbuZNGpIqoskIrIPzRxPQyPzMgA1V4lIelLiSEMjB4eJQ7PHRSQNKXGkoZF5wb9lww5NAhSR9KPEkYZam6pU4xCRNKTEkYZam6rUxyEiaUiJIw2pxiEi6UyJIw0Nz8vADCqq69jTpOuPi0h6UeJIQ1kRY1R+Ds0OFVV1qS6OiMg+lDjSlFbJFZF0pcSRpsYWBatbqp9DRNKNEkeaUo1DRNKVEkea0rXHRSRdKXGkqXGqcYhImlLiSFO6LoeIpCsljjQV3cfR3Nxzl/cVEYmXEkeaGpyTSWFeFg2NzVTW1Ke6OCIirZQ40lhrrUMd5CKSRpQ40piG5IpIOlLiSGMakisi6UiJI42pxiEi6UiJI42N07IjIpKGlDjS2DjN5RCRNKTEkcaiR1W5ay6HiKSHHkscZnavmW02s0VR235qZsvMbIGZ/cPMCqOe+5aZrTSz5WZ2dtT2c8JtK83smz1V3nRUmJdFXnYGO+sbqd7dmOriiIgAPVvjuA84p82254Bp7n4E8A7wLQAzOxS4DDgsjPmtmWWYWQbwG+Bc4FDg8nDfAcHMWmsd63fUprg0IiKBHksc7j4L2NZm27Pu3nLq/DowLrx/IfCwu9e7+7vASuDY8LbS3Ve7ewPwcLjvgKEhuSKSblLZx/FZ4N/h/bHAuqjn1ofbOto+YGhIroikG+vJTlczmwg86e7T2mz/NlAKfMLd3cx+A8xx9z+Hz/8BeIogsZ3t7teF2/8DONbdv9jOsa4HrgcoLi4umTFjRsLlrq2tJS8vLy3i/75sFw8u3MXHpuRx9fShvX58xSte8QMnvrS0tNzdS7vc0d177AZMBBa12XYVMAfIi9r2LeBbUY+fAU4Ib890tF9Ht5KSEk9GWVlZ2sT/c/56n3Djk/65B2J/zXQqv+IVr/i+Ew+UeQzf7b3aVGVm5wA3Ahe4e3Rv7xPAZWaWY2YHAJOBucA8YLKZHWBm2QQd6E/0ZplTTXM5RCTdZPbUC5vZQ8BpwAgzWw/cRFBjyAGeMzOA1939c+6+2MweAZYAjcAX3L0pfJ3/JqiBZAD3uvvinipzOhpbqNnjIpJeeixxuPvl7Wz+Qyf73wLc0s72pwj6OwakUfk5ZGUYW2sa2N3QxKDsjFQXSUQGOM0cT3ORiFFcoOYqEUkfShx9gIbkikg6UeLoAzQJUETSiRJHH7C3xqFlR0Qk9ZQ4+gDVOEQknShx9AHj1MchImlEiaMPUI1DRNKJEkcfUFwwCDOoqK5jT1NzqosjIgOcEkcfkJ0ZYXR+Ls0OFVV1qS6OiAxwShx9xFitWSUiaUKJo4+Ivv64iEgqKXH0EapxiEi6UOLoI1TjEJF0ocTRR6jGISLpQomjj9AkQBFJF0ocfUR0jaO5ueeuEy8i0hUljj4iLzuTorwsGhqbqdxVn+riiMgApsTRh7TUOtaruUpEUkiJow/RyCoRSQdKHH3I2MI8QB3kIpJaShx9iFbJFZF0oMTRh+ja4yKSDpQ4+pBxqnGISBroscRhZvea2WYzWxS1bZiZPWdmK8KfReF2M7NfmdlKM1tgZkdHxVwV7r/CzK7qqfL2BdE1DnfN5RCR1OjJGsd9wDlttn0TeMHdJwMvhI8BzgUmh7frgTshSDTATcBxwLHATS3JZiAqzMsiLzuDXfWNVO9uTHVxRGSAijtxmFmRmR3R1X7uPgvY1mbzhcD94f37gYuitv/JA68DhWZWDJwNPOfu29x9O/AcH0xGA4aZtdY61u+oTXFpRGSgiilxmNlMMxsa1gDeBv5oZr9I4Hij3X0TQPhzVLh9LLAuar/14baOtg9YGlklIqlmsbSVm9l8dz/KzK4Dxrv7TWa2wN07rXmY2UTgSXefFj7e4e6FUc9vd/ciM/sX8CN3nx1ufwH4BnA6kOPuN4fbvwvUuvvP2znW9QTNXBQXF5fMmDEjhl+/fbW1teTl5aVl/N3lVTy7ejefPTKfj04e3OvHV7ziFd9/40tLS8vdvbTLHd29yxuwECgGngWOCbctiCFuIrAo6vFyoDi8XwwsD+/fDVzedj/gcuDuqO377NfRraSkxJNRVlaWtvG/eWmFT7jxSf/hjMUpOb7iFa/4/hsPlHkMOSHWPo7vA88AK919npkdCKyIMTbaE0DLyKirgMejtl8Zjq46HqjyoCnrGeCssF+lCDgr3DZgjSvS7HERSa3MGPfb5FHNUu6+uqs+DjN7CDgNGGFm6wlGR90KPGJm1wLvAZ8Kd38KOA9YCdQC14TH2WZmPwTmhfv9wN3bdrgPKJoEKCKpFmviuAM4OoZtrdz98g6eOqOdfR34Qgevcy9wb2zF7P80CVBEUq3TxGFmJwAnAiPN7KtRTw0FMnqyYNK+kUNyyM6IsLWmgd0NTQzK1r9BRHpXV30c2cAQggSTH3WrBi7u2aJJeyIRo7gwF1BzlYikRqc1Dnd/GXjZzO5z97W9VCbpwtjCQazdWsuGHbuZNGpIqosjIgNMrH0cOWZ2D8Hw2tYYdz+9JwolndMFnUQklWJNHI8CdwG/B5p6rjgSi9bZ41p2RERSINbE0ejud/ZoSSRmqnGISCrFOgFwhpl93syKw6XRh4XrVkkK7K1xKHGISO+LtcbRMtv761HbHDiwe4sjsRjXcu1x1ThEJAViShzufkBPF0Rit19BLmZQUV3HnqZmsjJ0IUcR6T0xJQ4zu7K97e7+p+4tjsQiOzPC6PxcKqrrqKiqY/ywxFfSFBGJV6xNVcdE3c8lWDbkTUCJI0XGFg2iorqO9dt3K3GISK+Ktanqi9GPzawAeKBHSiQxGVs4iPK129VBLiK9LtHG8VqC64NLiuhKgCKSKrH2ccwgGEUFweKGhwCP9FShpGt7l1fXJEAR6V2x9nH8LOp+I7DW3df3QHkkRprLISKpElNTVbjY4TKClXGLgIaeLJR0bZxmj4tIisSUOMzsEmAuwRX7LgHeMDMtq55CLTWOjTvqaG72LvYWEek+sTZVfRs4xt03A5jZSOB54LGeKph0Li87k2GDs9lW00DlrnpGDc1NdZFEZICIdVRVpCVphLbGESs9pKWDfL36OUSkF8X65f+0mT1jZleb2dXAv4Cneq5YEgutkisiqdDVNccnAaPd/etm9gngQ4ABc4AHe6F80gmNrBKRVOiqxnEbsBPA3f/u7l91968Q1DZu6+nCSedU4xCRVOgqcUx09wVtN7p7GcFlZCWFVOMQkVToKnF0NlRnUHcWROKnGoeIpEJXiWOemf1n241mdi1QnuhBzewrZrbYzBaZ2UNmlmtmB5jZG2a2wsz+ambZ4b454eOV4fMTEz1ufzMuqsbhrrkcItI7ukocXwauMbOZZvbz8PYycB1wQyIHNLOxwJeAUnefRrD21WXAj4FfuvtkYDtwbRhyLbDd3ScBvwz3E6BgUBaDszPYVd9I9e7GVBdHRAaIThOHu7/v7icC3wfWhLfvu/sJ7l6RxHEzgUFmlgnkAZuA09k7ofB+4KLw/oXhY8LnzzAzS+LY/YaZtfZzrNdihyLSSywVTRxmdgNwC7AbeJag9vJ6WKvAzMYD/3b3aWa2CDinZVFFM1sFHOfulW1e83rgeoDi4uKSGTNmJFy+2tpa8vISvzhSb8bf8sp23qyo58YTCzl2bG6vH1/xild8/4kvLS0td/fSLnd09169ESyS+CIwEsgC/gn8B7Ayap/xwMLw/mJgXNRzq4DhnR2jpKTEk1FWVtZn4r/9jwU+4cYn/d7Zq1NyfMUrXvH9Jx4o8xi+x1OxbMiZwLvuvsXd9wB/B04ECsOmK4BxwMbw/nqCREL4fAGwrXeLnL7GFgZnFhpZJSK9JRWJ4z3geDPLC/sqzgCWAC8BLSvuXgU8Ht5/InxM+PyLYWYUNJdDRHpfrycOd3+DoJP7TWBhWIZ7gBuBr5rZSmA48Icw5A/A8HD7V4Fv9naZ09neKwEqcYhI74h1WfVu5e43ATe12bwaOLadfesIrgMi7Rina4+LSC/T0uh93MghOWRnRNha08DuhqZUF0dEBgAljj4uEjGKC4NhuGquEpHeoMTRD7Re0Gm7JgGKSM9T4ugH1EEuIr1JiaMfGKsOchHpRUoc/cC4onASoGocItILlDj6AV2XQ0R6kxJHPzBOs8dFpBcpcfQD+xXkEjF4v7qOPU3NqS6OiPRzShz9QFZGhNFDc2l2qKiqS3VxRKSfU+LoJ/bO5VBzlYj0LCWOfkKr5IpIb1Hi6Cc0skpEeosSRz+xt8ahZUdEpGcpcfQTWnZERHqLEkc/oetyiEhvUeLoJ8aENY6NO+po1pV1RaQHKXH0E3nZmQwbnE1DUzNVdZoEKCI9R4mjH2np59hSqysBikjPUeLoR5Q4RKQ3KHH0Iy1DcrfUKHGISM9R4uhHWmoc7ytxiEgPykx1AaT7tAzJfXb1bs65bRanTh3JqVNGUjphGNmZOkcQke6RksRhZoXA74FpgAOfBZYDfwUmAmuAS9x9u5kZcDtwHlALXO3ub6ag2GnvpEkj+OgRxbywuIJlFTtZVrGTu19ezeDsDE44aASnTh3JaVNGMn5YXqqLKiJ9WKpqHLcDT7v7xWaWDeQB/wO84O63mtk3gW8CNwLnApPD23HAneFPaWNwTia/+fTRvD63jOZhE3n5nS28/M4WllXs5Pml7/P80vcBOHDEYE6ZMpJTp47k+AOGMyg7I8UlF5G+pNcTh5kNBU4BrgZw9wagwcwuBE4Ld7sfmEmQOC4E/uTuDrxuZoVmVuzum3q56H1GVoZRMmkEJ04awbfOO4RNVbuZFSaRV1ZUsrqyhtWVNdz32hpyMiMcd+BwTp0SNGsdNHJwqosvImnOvJdnGZvZkcA9wBJgOlAO3ABscPfCqP22u3uRmT0J3Orus8PtLwA3untZm9e9HrgeoLi4uGTGjBkJl7G2tpa8vMSbc9I5vqnZWbFtD/Mr6plfUc+q7Y37PD8yL8KYwRGGD86iMDdCQW6EwpzwZ24GhbkRhmQZQQti75df8YpXfM/Fl5aWlrt7aVf7paKpKhM4Gviiu79hZrcTNEt1pL1vqA9kO3e/hyAhUVpa6iUlJQkXsLy8nP4cfyxwRXi/clc9s1dU8vI7W5j1zha21DSwpbYZtjR2GJ+VYQwfnMOI/GxGDMlhxJAcRubnhPez2bB9DVOGjyMSgYgZETMyIoYZZLTeD35GbN99MiJG83vL0vrvp3jF9+f4WKQicawH1rv7G+HjxwgSx/stTVBmVgxsjtp/fFT8OGBjr5W2nxsxJIeLjhrLRUeNpbnZWVpRzSvlixg6ajyVu+r33nY2ULmrni0769lZ30hFdR0V1Z1cpva1so6f68JBRZk8NLWOUfm5Cb+GiPScXk8c7l5hZuvMbKq7LwfOIGi2WgJcBdwa/nw8DHkC+G8ze5igU7xK/Rs9IxIxDhtTQN2mXEpK9u9wv7o9TWFCaaByZ31rQqncVU9lTQObK7cxdGgBTe40OzQ3O03NTrO33IImM3cP9mmm9bktO+tZtX0PF985hweuPZYJw9XnIpJuUjWq6ovAg+GIqtXANQSTER8xs2uB94BPhfs+RTAUdyXBcNxrer+4Ei03K4NxRXmMK2q/HTWZqnLlrnou/c1MVm2r5ZN3vsZ91xzLtLEFyRRXRLpZShKHu78FtNcBc0Y7+zrwhR4vlKSFEUNy+P6pw7h7cTOvrKjk0rvncM+VpZw0aUSqiyYiIU0nlrQzKCvCH646hgumj6GmoYmr/ziXGW+rW0skXShxSFrKzoxw26VH8tmTDmBPk/Olh+dz36vvprpYIoISh6SxSMT47vmH8M1zD8YdvjdjCT99Zhm9PfdIRPalxCFpzcz43KkH8bNPTScjYvzmpVXc+LcFNDbpKociqaLEIX3CxSXj+N2VJeRmRXikbD2f+3M5uxu0fLxIKihxSJ9x+sGjefC64ynMy+L5pZv5zB/eYEdtQ6qLJTLgKHFIn1IyoYjHPncCYwpyKV+7nU/dNYeNO3anulgiA4oSh/Q5k0bl87fPn8iU0UNYsXkXn7zzNVa8vzPVxRIZMJQ4pE8qLhjEo/91IqUTithUVcfFd82hfO32VBdLZEBQ4pA+qyAviz9fdxxnHjKaqt17uOL3r/NCeLEqEek5uua49Gm5WRnc9Zmj+fY/FvHXsnVc/0A5x47JZvSK+cFy7eHS7a1LuVu4lHskvB+xcFl3wmXejYaqWnL2q2Ly6CHkZOrqiCJtKXFIn5eZEeHWTx7OyPwcfv3SSuasr4f1yS1RcmfZbLIyjEmj8jlszNDwVsAhxfnk52Z1U8lF+iYlDukXzIyvnT2V0w8ZxazyxUyYOJGmluXam8Ol3Fvve7ise7gt3N7UDI3Nzby5Yj2b6jJ4t7KGpZuqWbqpmsfK9x5rwvC81kRyaJhUdO0QGUiUOKRfOXr/InzLIEqOGpfwa5QP30VJSQk19Y0sq6hmycZqFoe35RU7Wbu1lrVba3lqYUVrzMj8nNaaSX5DHVMO26OaifRbShwiHRick0nJhGGUTBjWum1PUzMrN+8KE0kVizdWs3RjNVt21jNz+RZmLt8CwE/nPMcR4wo46aARnDhpOEfvX0RulvpLpH9Q4hCJQ1ZGhEOKh3JI8VAuLglqNe7Oum27WbyxioUbqnhh4Xus3N7I/Pd2MP+9Hfz6pZXkZEY4ZuIwTpw0nJMOGsG0sQVkRCzFv41IYpQ4RJJkZuw/PI/9h+dx7uHFnDGyhqnTpjP33a28unIrr66sZFnFTmavrGT2ykpgOUNzMzn+wOGceNBwTpo0gkmjhmCmRCJ9gxKHSA8YkpPJ6QeP5vSDRwPBJXHnrNrKa6sqeXXlVt7bVsuzS97n2SXBvJNR+TmceNBwTpw0gryaRtxdiUTSlhKHSC8YMSSHj00fw8emjwFg3bZaXltVyWurglrJ5p31/POtjfzzrWAY8bdeepbDxxYEt3HBz/2H5SmZSFpQ4hBJgfHD8rh02P5cesz+uDsrNu/i1ZVBIpm3egs76hp5bdVWXlu1tTWmYFAWh48tYNrYAo4Ik8m4okFKJtLrlDhEUszMmDI6nymj87nmpAMoLy9n3OTDWLA+6GxfuH4HCzdUU7mrPqqfJFCYl7W3ZhLWTnSFROlpShwiaWj00Fw+cmguHzk06CNxdyqq61jYkkw2VLFwfRVbaxp4ZUUlr6yISiY5EY5fWk7pxCJKJhRx2JgCsjO1LJ10HyUOkT7AzCguGERxwSDOOmw/IEgmm6rqWLC+ikUbqlgQ1k621+7h6cUVPL04mKCYkxlh+vhCSicUcczEYRy9fxEFeZqcKIlLWeIwswygDNjg7ueb2QHAw8Aw4E3gP9y9wcxygD8BJcBW4FJ3X5OiYoukDTNjTOEgxhQO4pxpe5PJky/PZfeQMZSv2U7Z2m2s2lLD3He3MffdbcAqAKaMHkLJhGGtyWT8MPWVSOxSWeO4AVgKDA0f/xj4pbs/bGZ3AdcCd4Y/t7v7JDO7LNzv0lQUWCTdmRlj8jMpKRnPJaXjAdhW08Cba7dTtnY7ZWu2sWBDFe+8v4t33t/FQ3PfA4IlU0onBE1bBfV7OKrZiWiConQgJYnDzMYBHwVuAb5qwanO6cCnw13uB75HkDguDO8DPAb82szM1QMoEpNhg7M589DRnBn2l9Q3NrFoQxVla4JkUr52O1t21vPvRRX8e1HQvPXj11/glCkjOG3qKE6eNIKiwdmp/BUkzaSqxnEb8A0gP3w8HNjh7o3h4/XA2PD+WGAdgLs3mllVuP/e3kARiVlOZkbrGlz/RdC8tbqyhvI125m3ZhsvLtlI5a56/v7mBv7+5gbMYPq4Qk6dMpJTp45k+rhCLZcywFlvn7ib2fnAee7+eTM7DfgacA0wx90nhfuMB55y98PNbDFwtruvD59bBRzr7lvbvO71wPUAxcXFJTNmzEi4jLW1teTl5Sle8QMyvqamhq2N2bxV0cCbFfUsrWygsXnv80OyjOn75XDkftkcNTqHokH7Lt6Y6vIrPvH40tLScncv7XJHd+/VG/AjghrFGqACqAUeJKhBZIb7nAA8E95/BjghvJ8Z7medHaOkpMSTUVZWpnjFKz60q26PP7+kwr/7z4V+8o9f9Ak3PrnP7ZzbZvmt/17qc1ZVekNjU9qVX/GxA8o8hu/xXm+qcvdvAd8CaKlxuPsVZvYocDHByKqrgMfDkCfCx3PC518Mf0ER6QWDczI545DRnHFI0EeyprKGl9/Zwszlm5mzemvrxa7unLmKITmZFA82Rs1/ncHZmQzJyWRweBuSk9F6f3B2JoNzMlqf37uflp7vC9JpHseNwMNmdjMwH/hDuP0PwANmthLYBlyWovKJCDBxxGAmjhjMVSdOpG5PE/PWbOPl5Vt4+Z0trNi8ixX1sGLb1q5fqKPXL8jkB/lbOGXKyG4stXSnlCYOd58JzAzvrwaObWefOuBTvVowEYlJblYGJ08eycmTR/IdYFPVbl564y3GTZxETX0ju+obqalvpKahKfhZ38iu+vB+Q/B8bX1TsF9DI7vqGllT1ciV987lw1NH8j/nHcLk0fldlkN6VzrVOESkjysuGMTU4dmUJFhbqNvTxC2PzOYf79Tx0vItzFpRyaeP3Z8vnzmZ4UNyurm0kigtYCMiaSM3K4OLDh7CzK+fxmeOD1YOfuD1tZz2s5ncM2sV9Y1NqS6ioMQhImloxJAcbr7ocJ7+8imcOmUkO+sa+b+nlnHmL17mqYWbtAJwiilxiEjamjI6n/s/eyz3XXMMU0YPYd223Xz+wTf51F1zeGvdjlQXb8BS4hCRtHfa1FE89aWTueXj0xg+OJuytdu56Dev8uWH57Nhx+5UF2/AUeIQkT4hMyPCFcfsLxRnAAASb0lEQVRN4KWvn8bnTj2I7IwI/3xrI6f/bCY/f3Y5NfWNXb+IdAslDhHpU4bmZvHNcw/mhf93KucfUUx9YzN3vLiS0342k7/Oe48m9X/0OA3HFZE+afywPH796aO55qRt/PDJpby1bgc3/m0hmQbZTzxNZsTIyoiQmWFkRiJkZ0bIjBiZGRGyMqz1+eh9sjKMmuod7L9uYTCjPXvfme15LbPdW2fFB7PhczIjA+p6JkocItKnlUwYxj8+fyJPvL2Rnz27nHXbdtPYkOSw3ffei2v3zIiRlx0kFWveQ8HsV8jOjJCTGSEnK4PsjAg5WeHjzAg5mRmt97NbHmdFyM6IsH7dbt6LrCdiRsSMjIgRMaLuG5FwW4YZFm7PiATXY1m5bQ9HNnuPrmCsxCEifZ6ZceGRY7lg+hhen1fO4dOPpLGpmT1NTmNzM41NTkNT8HNPUzONzd7h88tWrGLkmHF7Z73vM9M9egb83ucbmpqprmukui7oZ9mwszq5X2je20mFX3haMxmRnlv3S4lDRPoNMyM7wxiSk/hX2357NlJSMjGumIbGZmrDJVTK31rIQVMOpqGpmfo9zdQ3NlHf2ExDYzP1jcHj1vt79n1ct6eJzZWVFBYNo9mhudlpdqcp/NnsRN1v2R61n8OuXbuI9HCzmRKHiEiSsjMjZGdmU5iXzftDM5k2tiDh1yovL6ek5Kik4rMze3bck0ZViYhIXJQ4REQkLkocIiISFyUOERGJixKHiIjERYlDRETiosQhIiJxUeIQEZG4WH+8kpaZbQHWJvESI4BKxSte8YofYPET3L3rC8a7u25tbkCZ4hWveMUPxPhYbmqqEhGRuChxiIhIXJQ42neP4hWveMUP0Pgu9cvOcRER6TmqcYiISFyUOEREJC5KHCJpwMxyYtkmkg50BcA0YmZjgQlE/V/cfVYvHDcDuNXdv57k64wCclseu/t7yZati+PNdvcPmdlOILqzzoLD+9CePH43mwMcHcO2VmbW4XMA7v5mN5SrS+H75xl3PzOB2IPdfVlHv0s8v4OZFQGT2fc9GNfnJ3yN8e6+II6YCLDA3afFc6y+TIkDMLORwH8CE9n3S/uzcbzGDPb98gKoAsqAu929rov4HwOXAkuAppYiAF2+8c1sMvAj4FD2/dAcGEvZ3b3JzErMzDyB0RJmdgHwc2AMsJkg+S0FDovjNT4B/BgYRfDF3+WXv7t/KPyZH2+Zw2O2TThtXz+mxBPWDD7JB98/P4ghdj9gLDDIzI4i+L0BhgJ5XYT/vJPnHDi9q+NHleN44A7gECAbyABqYvkbhO+fWjMrcPeqWI8Z+ipwPe3/LjH/DmZ2HXADMA54CzieIPF2GW9mM4ELCP53bwFbzOxld/9qLMd292Yze9vM9k/0ZCnZz7CZnQ/8kL0nnj168qTEEXgceAV4nr1f2vFaDYwEHgofXwq8D0wBfgf8RxfxFwFT3b0+gWP/EbgJ+CXwYeAa9n4BxWo+8LiZPQrUtGx097/HEPtDgg/q8+5+lJl9GLg8zuP/BPiYuy+NM65VvDWeloRjZj8AKoAHCP5uVwDxJKPHCU4SyoF4/39nA1cTfOH9Imr7TuB/Ogt09w/HeazO/Bq4DHgUKAWuBCbFEV8HLDSz59j3/fOlzoLc/frwjP077v5q3KXe6wbgGOB1d/+wmR0MfD/G2AJ3rw6Tzx/d/SYzi7nGESoGFpvZXPb9/S+IMT7Zz/BtwCeAhYmc/MVLiSOQ5+43JvkaR7n7KVGPZ5jZLHc/xcwWxxC/Gsgi/i8egEHu/kJYY1gLfM/MXiF4I8ZqGLCVfc/QHIglcexx961mFjGziLu/FNag4vF+okmjG2o8Z7v7cVGP7zSzNwiSWSzGufs5sZY3mrvfD9xvZp90978l8hoAZjaND56t/inOsqw0swx3bwL+aGavxRH+r/AWt/CM/WfACYnEh+rcvc7MMLOcsPlraoyxmWZWDFwCfDvB48eapDqS7Gd4HbCoN5IGKHG0eNLMznP3p5J4jZHRVVUz259gsTGAho6CzOwOgi/oWuAtM3uBqOTR1RlbqC48a1thZv8NbCBo8omZu18Tz/5t7DCzIQTNag+a2WagMc7XKDOzvwL/ZN/fvzdqPE1mdgXwMMH/4nLiq3m+ZmaHu/vCOGL24e5/M7OPEiS76C//WJq7bgJOI0gcTwHnArOBeBJHrZllE7wHfwJsAgbHUf774zhWe541s08Cf0/wy2+9mRUSvH+eM7PtwMYYY38APAPMdvd5ZnYgsCKeg7v7y3GV9oOS/Qx/A3jKzF5m38/PLzoOSZwmANLa1j2Y4A++hwTaB83sPOAuYFUYfwDweWAm8J/uflsHcVd19rqxfCDN7BiCM+xCgi/RAuAn7v56HOXPBa7lg19cXfbzmNlggqaKlmaeAuBBd98ax/H/2M5mj/H4Ze5eamZvE9T8ms1srrsfG+OxJwK3AycRJI5XgS+7+5oY45cQNOu8S/Aeann/HBFLfPgadxH0aXwY+D1wMTDX3a+NIXYhMB2Y7+7TzWw08Ht3/1gcx59AUFvLAr5C8D/8rbuvjDE+2Tb6ls9gI3vfSwm10ZvZqQTlf9rdOzxp607J9BGF8Ul9hs3sWWAXsBBobtnu7snWhNo/nhJH9wk7SQ8meNMv66pDPJ2EfRvLgE8TnIFdASx19xtSWrAYmNnzBH1EtwLDCb4Aj3H3E3vp+BPa2x42OcT6Ggvc/Yion0MIzr7PiiF2nrsfY2blBIlnJ0GzRcyDE5JlZrPZ20b/McI2enePubnUzIbxwVFRMZ/Jt4yIYt8BCl2OykrmpCnqNcr4YB/RZHfvtJ+qu7ScPPXGsWCAN1V151DAUAl7R9YcYWYxtzOHZ40djcq6ubOzdzObAnydDw7ljXlUDTDJ3T9lZhe6+/1m9heC6nssZY97RFQ7r5HMh/cCgrPUG4DPEIxIivlMK9FRdWY21N2rCb6ok7U7/FlrZmMI+psOiDF2XthM8zuCDvpdwNx4Dt4No3KSaqPvYFTUa8AZMcb/kGCQwWr2nnHHOirrAYKTprOJOmmK5bjRkukj6obP8PNmdpa7PxtXoRM0oBMH7Q8FjP7yjmc44wPAQQRv+ujhtLG2M/87jPtL+Pgygg9vFXAfwVlcRx4laCb7HYmPCtsT/twRdrRWEHyRxiLpEVEk8OG1cB4Hwei1lv9by0iUm81sG/BTd/9tF8dOdFTdX4DzCb6sPzCPBIipmSb0ZPjl/5Pw9SBosopFPvApgmbRp4GhHsc8hFCyo3KSbaNPZlQUBB3bByXYNJXwSVOUpPqISP4z/AXgG2aWcHN7XLyHL/jRF24Eb7qh4f3vAv8Ajo7zNZYSNv0lWIZXO9pG8GHuLLa8G/4G1wFFwCkEZ22bgf9KtOwJHH9++HNB+DMLeDHJ1xwOLI9hv7e6ofzDgOOAU1tuccYPIjiR+QfBSLavALkxxp4O/C/wHEEf29+AG+I8/ktAJInf/xhgCEGN4Y9hGY6LI35ey/8CyIn3/xIeb1SCZZ8b/pwFTCMY1LI6ztcoIagpDyWoZf0C+GIc8Ul/hnvzNtBrHC2+4+6PmNmHgI8Q1EDuJPgiiNUiYD+CM41EDDGz49z9DQAzO47ggwgdjFAK24QhGPr7eYIvnegRFdviOP4D7J3E1tIhP7qzgLCJCpIbEdUimRpPuzwYInxaDLsmNaou2WaW0P0ETV6/Ch9fTlBbvaSrQHd/MRxNcwxBH8fnCJr8bo/j+MmOynGC99AEgqQPwdlzrAMEkhkVBUHH/HwzW8S+5Y9lHsU9Yf/Id4AnCD53343j2BD8rld5MLLu+2Z2OfBlgg7zDrX5DH+B4KQh7s+wmZ3S3nbvoZUn1DkOmNl8D4Zx/ojg7P4vLdvieI2XgCMJ2pbjfeO2jKq4l73JYidBm/8S4KPu/kg7Me8SfGCjJwq1/kM9xhEt4Ws9zd5JbK1VZXfvcHZy1EiotmUIQ+PqXLyO4KzxcIKmuSHAd9397lhfI1HJjqoL+6damlmObGlmcfdL4yjD2+4+vattHcS+EJZ/DkGT22x33xzrscPXSGpUjpktJ2ijbxsf8wCBqNeKe1SUBXOl7m7n+B12rptZezPDW97HHkfSJBzC+xhBE+uHCCb8fsy7mEnfzmd4ny/kWD/DFqxc0SIXOJagFhNPP2fMVOMIbDCzu4EzgR+Ho6PiXQDye0mWYQHBqKCJBDPQtwPnuvs84ANJA8DdDwAws0sIPmTVZvZdgvWNfhjn8eOexObh3A8zu5+gaWRH+LiIzpfDaE/cNZ7u4u757Y3oiUMyk89azDez4z0cfhnWOGOdSb2AoKlkGkHy32Fmc9x9d+dh+xjmMYzg6sQWd38iifhWnX3Zd6LS3X/V9W77aFkdYCpB4m8p/8eIYamfaO6+2swuI6gxrSOYVNrl3z/qMzyIYPj+hwiSxysEfR6xHn+fPlAzG0/sE1jjphoHYGZ5wDkEtY0VFswiPdx7aYRCWIangR3Am8R4xh8V2zKE80PA/xF8af+P7zsbuqvXuAe4wxOYxNZe7SyBGlvcNZ7u0lFTk7vHOqLnHwTDT79M0N+wHchy9/PiKMNSgi+wlmVS9ifoN2smxjkh4RDea4CvAfu5e8yr65rZrQR9Sgm9583sDILmtbYTWONprkyYmf0iPO4TbY4fy3DcZ4FPuvvO8HE+8GgsJ1LtjIYcRfA+rg+PH1NTnZk9AlQDD4abLgcK3b3LpsoOXs8I+gsPTyS+y9dX4kiOddMKrWa2yBNcXTOZpraoN34mwRn3auKcxGbBxLvT3H17+HgY8HI8b9pkfv9kdUdTU9RrJTT5zDqYC9KisyafcBTTyQS1jrUEZ8uvuPuLMR7b2JusE22u+zPBHKbFRA2Hjae5MhlhUzHsO7rOY2mqMbNlwHQP14kLWxzedveDY4hN+P/W5nUSbqoM921ZgQKC1pKjCDr4u1ojLyFqqkqSJ7lCa5Rklq1Ipqnt/ASO19bPCcr/GMGb9xLgljhfI+llO5LQHU1NQOJLTyTSFxBlEMEonnJ3j3epF9zdzewtd+90mfYuTO+ps9sYzWxnW6xnxQ8Ac8OaowMfZ29zaaeS/L9FS6apEoIThi3h/UaCxVaHdbx7clTjSBOWxLIVadLUdihBM40BL7j7khjjkq7xJKs7mpr6OjP7DXBf2KeWSPzvgF/G+n/vbmb2/6Ie5hKcEC2NtcZjwSTgk8OHs9x9fjcXsaPjtrz/s9jbVOkEo9OWxFoLN7M3gas9nL/TMqornubquMqtxJEeOqryduMZTVrqrqp+d0m0qamvC09cpgJrCJYFjytxh300B5HEel3dKax1P+HuZ6fi+LHqxqautqO6rgTO72pUV6KUOEQk6ROXdDvxCUf2zXX3yak4fipYsGxJy6iui+IcVRcX9XGISNJf8KmuGbcZ3ZRBMKS9yyXp+7p2RnUNI/j937BgrbweqfGpxiEifV6bGk8jwYXB4h4o0NekqqlXiUNEROIS7+xoEREZ4JQ4REQkLkocIl0ws2+b2WIzW2Bmb4WTs3rqWDPNrNeu5CaSCI2qEumEmZ1AMJnsaHevN7MRBNeUFhmwVOMQ6VwxwcqrLYvWVbr7RjP7XzObZ2aLzOyecL2nlhrDL81slpktNbNjzOzvZrbCzG4O95loZsvM7P6wFvNYOPt/H2Z2lpnNMbM3zezRcBFDzOxWM1sSxv6sF/8WIoASh0hXngXGm9k7ZvbbcGY5wK/d/ZhwSYhB7LvmV4O7n0KwLPbjBJf1nAZcbWbDw32mAveE4+yrCZbUbhXWbL4DnBmuIVUGfDVcQPLjwGFh7M098DuLdEqJQ6QT7r6LYNXZ6wkWkfurmV0NfNjM3ggnYJ1OcMW9Fi3XdVgILHb3TWGNZTUwPnxunbu3LGL3Z4JlIqIdDxwKvGpmbwFXEaxfVA3UAb+34AqMtd32y4rESH0cIl1w9yaC1VdnhonivwguiVrq7uvM7HvsewGolutBNEfdb3nc8plrO4Gq7WMDnnP3y9uWx8yOJbgs7WXAfxMkLpFeoxqHSCfMbKqZRa93dCSwPLxfGfY7XJzAS+8fdrxDcNGe2W2efx04ycwmheXIM7Mp4fEKPLg++pfD8oj0KtU4RDo3BLjDzAoJlrJYSdBstYOgKWoNkMhS5EuBq8LrqKwA7ox+0t23hE1iD4UrvULQ57ETeNzMcglqJV9J4NgiSdGSIyK9zMwmAk+m6oqHIslSU5WIiMRFNQ4REYmLahwiIhIXJQ4REYmLEoeIiMRFiUNEROKixCEiInFR4hARkbj8fzuQVr1BBdyMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x25a6ab853c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fd_sntrigrams=FreqDist(surname_trigrams)\n",
    "fd_sntrigrams.plot(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# getting labels (a number for each community)\n",
    "le = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = le.fit_transform(df['Community'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classifier:\n",
    "    count = 0\n",
    "    \n",
    "    def __init__(self, name, description=\"\"):\n",
    "        self.name = name\n",
    "        self.description = description\n",
    "        self.vectorizer = CountVectorizer()\n",
    "        self.metrics = {'accuracy': accuracy_score}\n",
    "        self.scores = {}\n",
    "        Classifier.count = Classifier.count + 1\n",
    "        \n",
    "    def fit(self, x, y, clf, vectorizer=None, metrics_dict=None):\n",
    "        if vectorizer is not None:\n",
    "            self.vectorizer = vectorizer\n",
    "        if metrics_dict is not None:\n",
    "            self.metrics.update(metrics_dict)\n",
    "            \n",
    "        x = self.vectorizer.fit_transform(x)\n",
    "        self.clf = clf.fit(x,y)\n",
    "        \n",
    "        self.scores['train'] = {}\n",
    "        for metric, metric_fn in self.metrics.items():\n",
    "            self.scores['train'][metric] = metric_fn(y, self.clf.predict(x))\n",
    "        \n",
    "        self.print_scores('train')\n",
    "        \n",
    "    def predict(self, x):\n",
    "        x = self.vectorizer.transform(x)\n",
    "        return self.clf.predict(x)\n",
    "    \n",
    "    def score(self, x, y, data_set='dev', metrics_dict=None):\n",
    "        if metrics_dict is not None:\n",
    "            self.metrics.update(metrics_dict)\n",
    "        \n",
    "        self.scores[data_set] = {}\n",
    "        for metric, metric_fn in self.metrics.items():\n",
    "            self.scores[data_set][metric] = metric_fn(y, self.predict(x))\n",
    "        \n",
    "        self.print_scores(data_set)\n",
    "        \n",
    "        \n",
    "    def print_scores(self, data_set):\n",
    "        if data_set in self.scores:\n",
    "            print('Scores on',data_set)\n",
    "            scores_dict = self.scores[data_set]\n",
    "            for metric, score in scores_dict.items():\n",
    "                print(metric, \"=\", score)\n",
    "        else:\n",
    "            print('No scores available')\n",
    "        print()\n",
    "        \n",
    "    def get_scores(self):\n",
    "        for data_set in self.scores.keys():\n",
    "            self.print_scores(data_set)\n",
    "        \n",
    "    def add_method(self, name, method):\n",
    "        self.__setattr__(name, method)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {'f1_all':lambda y,y_pred: f1_score(y, y_pred, average=None, labels=list(range(25))), \n",
    "                'f1_micro':lambda y,y_pred: f1_score(y, y_pred, average='micro'),\n",
    "                'f1_macro':lambda y,y_pred: f1_score(y, y_pred, average='macro'),\n",
    "                'f1_weighted':lambda y,y_pred: f1_score(y, y_pred, average='weighted')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(df['Name'][firstnames_mask].values, labels[firstnames_mask], random_state = RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "clfs = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.352315732564426\n",
      "f1_all = [0.         0.38505232 0.16724739 0.         0.29120246 0.\n",
      " 0.43300934 0.25443787 0.         0.17777778 0.61538462 0.19104478\n",
      " 0.23103448 0.13312203 0.         0.         0.21192053 0.\n",
      " 0.         0.36363636 0.36781609 0.27939698 0.37701472 0.\n",
      " 0.44319776]\n",
      "f1_micro = 0.352315732564426\n",
      "f1_macro = 0.20509564557814997\n",
      "f1_weighted = 0.3371866882865932\n",
      "\n",
      "Scores on dev\n",
      "accuracy = 0.3202702702702703\n",
      "f1_all = [0.         0.3539823  0.19047619 0.         0.25345622 0.\n",
      " 0.41371557 0.18965517 0.         0.         0.         0.1372549\n",
      " 0.21367521 0.01970443 0.         0.         0.2081448  0.\n",
      " 0.         0.34046512 0.20588235 0.28134557 0.34754797 0.\n",
      " 0.28444444]\n",
      "f1_micro = 0.3202702702702703\n",
      "f1_macro = 0.1495543589069367\n",
      "f1_weighted = 0.3043070224600764\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['nb_fn_2'] = Classifier('nb_fn_2', description=\"naive bayes on first name bigrams\")\n",
    "clfs['nb_fn_2'].fit(X_train, y_train, MultinomialNB(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams([x],2)), metrics_dict=metrics_dict)\n",
    "clfs['nb_fn_2'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.4903586231753469\n",
      "f1_all = [0.07692308 0.56492162 0.27210884 0.         0.44875776 0.\n",
      " 0.5514158  0.33426184 0.         0.48275862 0.         0.42180095\n",
      " 0.30912477 0.27836611 0.         0.         0.4        0.\n",
      " 0.         0.46061415 0.38461538 0.41478439 0.52781457 0.\n",
      " 0.62369338]\n",
      "f1_micro = 0.4903586231753469\n",
      "f1_macro = 0.27299838621439604\n",
      "f1_weighted = 0.47542299875979405\n",
      "\n",
      "Scores on dev\n",
      "accuracy = 0.3956756756756757\n",
      "f1_all = [0.         0.47140381 0.2195122  0.         0.33255269 0.\n",
      " 0.50597177 0.224      0.         0.75       0.         0.28571429\n",
      " 0.17560976 0.01941748 0.         0.         0.23893805 0.\n",
      " 0.         0.37181996 0.23333333 0.32911392 0.428      0.\n",
      " 0.38509317]\n",
      "f1_micro = 0.39567567567567574\n",
      "f1_macro = 0.21610784467630395\n",
      "f1_weighted = 0.3728263999057889\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['nb_fn_3'] = Classifier('nb_fn_3', description=\"naive bayes on first name trigrams\")\n",
    "clfs['nb_fn_3'].fit(X_train, y_train, MultinomialNB(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams([x],3)), metrics_dict=metrics_dict)\n",
    "clfs['nb_fn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.4517030095512705\n",
      "f1_all = [0.         0.503961   0.25165563 0.         0.40315908 0.\n",
      " 0.5279661  0.28871391 0.         0.47826087 0.66666667 0.41269841\n",
      " 0.34138973 0.22916667 0.         0.         0.36383208 0.\n",
      " 0.         0.43124597 0.4372093  0.37083708 0.49455041 0.\n",
      " 0.58187135]\n",
      "f1_micro = 0.4517030095512705\n",
      "f1_macro = 0.2826326770610507\n",
      "f1_weighted = 0.4418599427467069\n",
      "\n",
      "Scores on dev\n",
      "accuracy = 0.3854054054054054\n",
      "f1_all = [0.         0.44484305 0.21686747 0.         0.32685714 0.\n",
      " 0.50184049 0.21538462 0.         0.5        0.         0.28571429\n",
      " 0.2578125  0.00943396 0.         0.         0.24087591 0.\n",
      " 0.         0.37335835 0.275      0.34482759 0.42650104 0.\n",
      " 0.37837838]\n",
      "f1_micro = 0.3854054054054054\n",
      "f1_macro = 0.2085954251036278\n",
      "f1_weighted = 0.37112920133719385\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['nb_fn_23'] = Classifier('nb_fn_23', description=\"naive bayes on first name bigrams and trigrams\")\n",
    "clfs['nb_fn_23'].fit(X_train, y_train, MultinomialNB(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams([x],2)+get_ngrams([x],3)), metrics_dict=metrics_dict)\n",
    "clfs['nb_fn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(df['Surname'][surnames_mask].values, labels[surnames_mask], random_state = RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.5533989266547406\n",
      "f1_all = [0.23529412 0.60770235 0.25308642 0.         0.60234959 0.\n",
      " 0.48929081 0.52475248 0.         0.10526316 0.         0.50396825\n",
      " 0.39777469 0.4691358  0.         0.         0.63985702 0.\n",
      " 0.         0.74405328 0.11111111 0.44444444 0.48787447 0.\n",
      " 0.58187599]\n",
      "f1_micro = 0.5533989266547406\n",
      "f1_macro = 0.2879133592176292\n",
      "f1_weighted = 0.5495915353628488\n",
      "\n",
      "Scores on dev\n",
      "accuracy = 0.537429568017172\n",
      "f1_all = [0.44444444 0.59368836 0.12371134 0.         0.5987526  0.\n",
      " 0.45227606 0.3960396  0.         0.         0.         0.59016393\n",
      " 0.34513274 0.46046512 0.         0.         0.6446281  0.\n",
      " 0.         0.75874439 0.05405405 0.4        0.44676409 0.\n",
      " 0.53211009]\n",
      "f1_micro = 0.537429568017172\n",
      "f1_macro = 0.34204874702046767\n",
      "f1_weighted = 0.5316984020025789\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['nb_sn_2'] = Classifier('nb_sn_2', description=\"naive bayes on surname bigrams\")\n",
    "clfs['nb_sn_2'].fit(X_train, y_train, MultinomialNB(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams([x],2)), metrics_dict=metrics_dict)\n",
    "clfs['nb_sn_2'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.681216457960644\n",
      "f1_all = [0.48484848 0.76780581 0.40506329 0.         0.75703704 0.\n",
      " 0.56568627 0.6875     0.         0.2        0.         0.73221757\n",
      " 0.67718447 0.57164068 0.         0.15384615 0.76254826 0.\n",
      " 0.         0.8017673  0.37096774 0.62464722 0.63146151 0.\n",
      " 0.65934066]\n",
      "f1_micro = 0.681216457960644\n",
      "f1_macro = 0.3941424991934132\n",
      "f1_weighted = 0.6764507800644495\n",
      "\n",
      "Scores on dev\n",
      "accuracy = 0.6230211966729273\n",
      "f1_all = [0.57142857 0.72156863 0.17391304 0.         0.70031546 0.\n",
      " 0.51038576 0.60655738 0.         0.         0.         0.68674699\n",
      " 0.568      0.54054054 0.         0.         0.71559633 0.\n",
      " 0.         0.78777589 0.21052632 0.47904192 0.55755016 0.\n",
      " 0.49302326]\n",
      "f1_micro = 0.6230211966729273\n",
      "f1_macro = 0.4161485114885828\n",
      "f1_weighted = 0.6158898283169926\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['nb_sn_3'] = Classifier('nb_sn_3', description=\"naive bayes on surname trigrams\")\n",
    "clfs['nb_sn_3'].fit(X_train, y_train, MultinomialNB(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams([x],3)), metrics_dict=metrics_dict)\n",
    "clfs['nb_sn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.6546511627906977\n",
      "f1_all = [0.32258065 0.74061883 0.33121019 0.         0.73545966 0.\n",
      " 0.54531451 0.63963964 0.         0.10526316 0.         0.66666667\n",
      " 0.63625731 0.57377049 0.         0.         0.74538745 0.\n",
      " 0.         0.7938238  0.33333333 0.57216495 0.57500921 0.\n",
      " 0.6913229 ]\n",
      "f1_micro = 0.6546511627906977\n",
      "f1_macro = 0.3603129098952995\n",
      "f1_weighted = 0.6496997212300953\n",
      "\n",
      "Scores on dev\n",
      "accuracy = 0.6120203917359807\n",
      "f1_all = [0.75       0.69818913 0.16326531 0.         0.70350691 0.\n",
      " 0.49920761 0.55357143 0.         0.         0.         0.66666667\n",
      " 0.53435115 0.54811715 0.         0.         0.71264368 0.\n",
      " 0.         0.78824546 0.10810811 0.43799472 0.53403141 0.\n",
      " 0.57021277]\n",
      "f1_micro = 0.6120203917359807\n",
      "f1_macro = 0.4134055750867228\n",
      "f1_weighted = 0.6053861759473466\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['nb_sn_23'] = Classifier('nb_sn_23', description=\"naive bayes on surname bigrams and trigrams\")\n",
    "clfs['nb_sn_23'].fit(X_train, y_train, MultinomialNB(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams([x],2)+get_ngrams([x],3)), metrics_dict=metrics_dict)\n",
    "clfs['nb_sn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(df[['Name','Surname']][fullnames_mask].values, labels[fullnames_mask], random_state = RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.7432015953589558\n",
      "f1_all = [0.4137931  0.79614874 0.42524917 0.         0.78327833 0.\n",
      " 0.70293083 0.71195652 0.         0.61538462 0.22222222 0.78225806\n",
      " 0.70698925 0.65679926 1.         0.2        0.73969631 1.\n",
      " 0.         0.82123386 0.62585034 0.68706811 0.71754069 0.\n",
      " 0.77399381]\n",
      "f1_micro = 0.7432015953589558\n",
      "f1_macro = 0.535295728967625\n",
      "f1_weighted = 0.7393184063807705\n",
      "\n",
      "Scores on dev\n",
      "accuracy = 0.6623164763458401\n",
      "f1_all = [0.         0.75804968 0.27368421 0.         0.69621622 0.\n",
      " 0.65876153 0.57142857 0.         0.         0.         0.66206897\n",
      " 0.59649123 0.51873199 0.         0.         0.63492063 0.\n",
      " 0.         0.77744807 0.23529412 0.53333333 0.60262009 0.\n",
      " 0.69918699]\n",
      "f1_micro = 0.6623164763458401\n",
      "f1_macro = 0.39134455347117025\n",
      "f1_weighted = 0.655566355677859\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['nb_fsn_3'] = Classifier('nb_fsn_3', description=\"naive bayes on first name and surname trigrams\")\n",
    "clfs['nb_fsn_3'].fit(X_train, y_train, MultinomialNB(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['nb_fsn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.6999637418419145\n",
      "f1_all = [0.35714286 0.75017349 0.35294118 0.         0.74697025 0.\n",
      " 0.6471464  0.64788732 0.         0.28571429 0.22222222 0.74851485\n",
      " 0.66666667 0.6038688  0.         0.2        0.70348837 0.\n",
      " 0.         0.79584176 0.57534247 0.64632984 0.67924528 0.\n",
      " 0.71468144]\n",
      "f1_micro = 0.6999637418419145\n",
      "f1_macro = 0.4137670992768108\n",
      "f1_weighted = 0.6956513431974882\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.6326808047852094\n",
      "f1_all = [0.         0.70805044 0.27956989 0.         0.66738197 0.\n",
      " 0.6147248  0.5620915  0.         0.22222222 0.         0.66225166\n",
      " 0.55319149 0.52083333 0.         0.         0.60941828 0.\n",
      " 0.         0.75648703 0.23529412 0.5398773  0.58092176 0.\n",
      " 0.68100358]\n",
      "f1_micro = 0.6326808047852094\n",
      "f1_macro = 0.39015806567590683\n",
      "f1_weighted = 0.6280114240072823\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['nb_fsn_23'] = Classifier('nb_fsn_23', description=\"naive bayes on first name and surname bigrams trigrams\")\n",
    "clfs['nb_fsn_23'].fit(X_train, y_train, MultinomialNB(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,2) + get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['nb_fsn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hence Trigrams for both first and last names are most meaningful for Naive Bayes classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "assamese\n",
      "['aab' 'onj' 'onk' 'onn' 'ono' 'onp' 'ons' 'ont' 'onu' 'onv']\n",
      "bengali\n",
      "['lhe' 'mmi' 'mle' 'mku' 'mkr' 'mke' 'mka' 'mji' 'mjh' 'mje']\n",
      "english\n",
      "['aab' 'ohi' 'ohk' 'ohl' 'ohm' 'oho' 'ohr' 'oht' 'ohu' 'oia']\n",
      "garhwali\n",
      "['aab' 'ope' 'oph' 'opi' 'opl' 'opn' 'opo' 'opr' 'ora' 'ord']\n",
      "gujarati\n",
      "['aab' 'nie' 'nif' 'nig' 'nij' 'nip' 'niw' 'niy' 'niz' 'njh']\n",
      "haryanavi\n",
      "['aab' 'ope' 'oph' 'opi' 'opl' 'opn' 'opo' 'opr' 'ora' 'ord']\n",
      "hindi\n",
      "['aab' 'nib' 'nic' 'nid' 'nie' 'nif' 'nij' 'nin' 'nip' 'niz']\n",
      "kannada\n",
      "['aab' 'oen' 'oer' 'ofa' 'ofe' 'off' 'ofi' 'ogc' 'ogh' 'ogi']\n",
      "kashmiri\n",
      "['aab' 'opl' 'opn' 'opo' 'opr' 'ora' 'ord' 'ore' 'org' 'ori']\n",
      "konkani\n",
      "['aab' 'ony' 'oob' 'ood' 'oof' 'ooh' 'ooj' 'ool' 'oom' 'oon']\n",
      "kumaoni\n",
      "['aab' 'oop' 'oor' 'opa' 'ope' 'oph' 'opi' 'opl' 'opn' 'opo']\n",
      "malayalam\n",
      "['aab' 'oct' 'odb' 'odd' 'odh' 'odi' 'odj' 'odr' 'odw' 'ody']\n",
      "marathi\n",
      "['aab' 'odj' 'odr' 'odw' 'ody' 'oeb' 'oel' 'oem' 'oen' 'oer']\n",
      "marwari\n",
      "['aab' 'nra' 'nri' 'nsd' 'nse' 'nsi' 'nso' 'nsr' 'nss' 'nsu']\n",
      "mizo\n",
      "['aab' 'opi' 'opl' 'opn' 'opo' 'opr' 'ora' 'ord' 'ore' 'org']\n",
      "nepali\n",
      "['aab' 'oom' 'oon' 'oop' 'oor' 'opa' 'ope' 'oph' 'opi' 'opl']\n",
      "oriya\n",
      "['aab' 'nse' 'nsg' 'nsh' 'nsi' 'nso' 'nsr' 'nss' 'nsu' 'nsy']\n",
      "pashto\n",
      "['aab' 'opi' 'opl' 'opn' 'opo' 'opr' 'ord' 'ore' 'org' 'ori']\n",
      "persian\n",
      "['aab' 'opi' 'opl' 'opn' 'opo' 'opr' 'ora' 'ord' 'ore' 'org']\n",
      "punjabi\n",
      "['aab' 'nas' 'nat' 'naw' 'naz' 'nbu' 'ncy' 'naq' 'ndg' 'ndl']\n",
      "sindhi\n",
      "['aab' 'oka' 'okb' 'okc' 'oke' 'okh' 'okk' 'oks' 'oku' 'ola']\n",
      "tamil\n",
      "['aab' 'ntk' 'ntn' 'ntr' 'ntu' 'ntw' 'nub' 'nue' 'nug' 'nte']\n",
      "telugu\n",
      "['aab' 'ncy' 'ndg' 'ndk' 'ndn' 'nds' 'ndv' 'nca' 'ndy' 'neh']\n",
      "tulu\n",
      "['aab' 'oph' 'opi' 'opl' 'opn' 'opo' 'opr' 'ora' 'ord' 'ore']\n",
      "urdu\n",
      "['aab' 'odj' 'odw' 'ody' 'oel' 'oem' 'oen' 'oer' 'ofa' 'ofe']\n"
     ]
    }
   ],
   "source": [
    "# 10 most impt features for each class\n",
    "for i, class_idx in enumerate(clfs['nb_fsn_3'].clf.classes_):\n",
    "    print(le.classes_[class_idx])\n",
    "    class_prob_sorted = clfs['nb_fsn_3'].clf.feature_log_prob_[i, :].argsort()\n",
    "    print(np.take(clfs['nb_fsn_3'].vectorizer.get_feature_names(), class_prob_sorted[:10]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train and dev f1 scores for each community as well as the total number of samples available for the community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "clfs['nb_fsn_3'].scores_df = pd.DataFrame({'f1_train':clfs['nb_fsn_3'].scores['train']['f1_all'], 'f1_dev':clfs['nb_fsn_3'].scores['dev']['f1_all']}, index=le.classes_).join(pd.DataFrame(df['Community'].value_counts().rename('value_counts')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1_dev</th>\n",
       "      <th>f1_train</th>\n",
       "      <th>value_counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mizo</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pashto</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>persian</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kashmiri</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tulu</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>garhwali</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>haryanavi</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kumaoni</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nepali</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>konkani</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>assamese</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sindhi</th>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.625850</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>english</th>\n",
       "      <td>0.273684</td>\n",
       "      <td>0.425249</td>\n",
       "      <td>302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>marwari</th>\n",
       "      <td>0.518732</td>\n",
       "      <td>0.656799</td>\n",
       "      <td>674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tamil</th>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.687068</td>\n",
       "      <td>661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kannada</th>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.711957</td>\n",
       "      <td>358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>marathi</th>\n",
       "      <td>0.596491</td>\n",
       "      <td>0.706989</td>\n",
       "      <td>545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>telugu</th>\n",
       "      <td>0.602620</td>\n",
       "      <td>0.717541</td>\n",
       "      <td>1770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oriya</th>\n",
       "      <td>0.634921</td>\n",
       "      <td>0.739696</td>\n",
       "      <td>672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hindi</th>\n",
       "      <td>0.658762</td>\n",
       "      <td>0.702931</td>\n",
       "      <td>2935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>malayalam</th>\n",
       "      <td>0.662069</td>\n",
       "      <td>0.782258</td>\n",
       "      <td>364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gujarati</th>\n",
       "      <td>0.696216</td>\n",
       "      <td>0.783278</td>\n",
       "      <td>1760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>urdu</th>\n",
       "      <td>0.699187</td>\n",
       "      <td>0.773994</td>\n",
       "      <td>446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bengali</th>\n",
       "      <td>0.758050</td>\n",
       "      <td>0.796149</td>\n",
       "      <td>2073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>punjabi</th>\n",
       "      <td>0.777448</td>\n",
       "      <td>0.821234</td>\n",
       "      <td>2208</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             f1_dev  f1_train  value_counts\n",
       "mizo       0.000000  1.000000             1\n",
       "pashto     0.000000  1.000000             1\n",
       "persian    0.000000  0.000000             1\n",
       "kashmiri   0.000000  0.000000             2\n",
       "tulu       0.000000  0.000000             4\n",
       "garhwali   0.000000  0.000000             6\n",
       "haryanavi  0.000000  0.000000             6\n",
       "kumaoni    0.000000  0.222222            10\n",
       "nepali     0.000000  0.200000            13\n",
       "konkani    0.000000  0.615385            26\n",
       "assamese   0.000000  0.413793            30\n",
       "sindhi     0.235294  0.625850           131\n",
       "english    0.273684  0.425249           302\n",
       "marwari    0.518732  0.656799           674\n",
       "tamil      0.533333  0.687068           661\n",
       "kannada    0.571429  0.711957           358\n",
       "marathi    0.596491  0.706989           545\n",
       "telugu     0.602620  0.717541          1770\n",
       "oriya      0.634921  0.739696           672\n",
       "hindi      0.658762  0.702931          2935\n",
       "malayalam  0.662069  0.782258           364\n",
       "gujarati   0.696216  0.783278          1760\n",
       "urdu       0.699187  0.773994           446\n",
       "bengali    0.758050  0.796149          2073\n",
       "punjabi    0.777448  0.821234          2208"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clfs['nb_fsn_3'].scores_df.sort_values(by=['f1_dev','value_counts'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NB classifier on first name and last name trigrams does relatively well on Punjabi and Bengali names.\n",
    "\n",
    "Overall, the greater number of available examples for a class, the better the performance (dev)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.9887599709934736\n",
      "f1_all = [1.         0.98930481 0.97706422 1.         0.99237805 1.\n",
      " 0.98466468 0.99539171 1.         0.94444444 1.         0.99447514\n",
      " 0.99393939 0.98274112 1.         1.         0.99391481 1.\n",
      " 1.         0.98962348 0.99459459 0.98850575 0.99003831 1.\n",
      " 0.98157454]\n",
      "f1_micro = 0.9887599709934736\n",
      "f1_macro = 0.9917062018504751\n",
      "f1_weighted = 0.9887548927659521\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.7074497009244154\n",
      "f1_all = [0.22222222 0.80408859 0.42519685 0.5        0.72508215 0.\n",
      " 0.71192661 0.61935484 0.         0.2        0.66666667 0.7133758\n",
      " 0.62100457 0.47940075 0.         0.         0.74461538 0.\n",
      " 0.         0.79669763 0.56       0.64285714 0.6482593  0.\n",
      " 0.63589744]\n",
      "f1_micro = 0.7074497009244154\n",
      "f1_macro = 0.5103164724409006\n",
      "f1_weighted = 0.7011945971881904\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['rf_fsn_3'] = Classifier('rf_fsn_3', description=\"random forest on first name and surname trigrams\")\n",
    "clfs['rf_fsn_3'].fit(X_train, y_train, RandomForestClassifier(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['rf_fsn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest overfits on data.\n",
    "\n",
    "Trying Search over hyperparameters to reduce variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameters currently in use:\n",
      "\n",
      "{'bootstrap': True,\n",
      " 'class_weight': None,\n",
      " 'criterion': 'gini',\n",
      " 'max_depth': None,\n",
      " 'max_features': 'auto',\n",
      " 'max_leaf_nodes': None,\n",
      " 'min_impurity_decrease': 0.0,\n",
      " 'min_impurity_split': None,\n",
      " 'min_samples_leaf': 1,\n",
      " 'min_samples_split': 2,\n",
      " 'min_weight_fraction_leaf': 0.0,\n",
      " 'n_estimators': 10,\n",
      " 'n_jobs': 1,\n",
      " 'oob_score': False,\n",
      " 'random_state': None,\n",
      " 'verbose': 0,\n",
      " 'warm_start': False}\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "# Look at parameters used by our current forest\n",
    "print('Parameters currently in use:\\n')\n",
    "pprint(clfs['rf_fsn_3'].clf.get_params())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_grid = \\\n",
    "{'bootstrap': [True, False],\n",
    " 'max_depth': [None]+list(range(5,50,5)),\n",
    " 'max_features': ['auto', 'sqrt'],\n",
    " 'min_samples_leaf': list(range(1,12)),\n",
    " 'min_samples_split': list(range(2,21)),\n",
    " 'n_estimators': list(range(1,25))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 4 folds for each of 100 candidates, totalling 400 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=4.\n",
      "  % (min_groups, self.n_splits)), Warning)\n",
      "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed:   10.6s\n",
      "[Parallel(n_jobs=-1)]: Done 146 tasks      | elapsed:   21.7s\n",
      "[Parallel(n_jobs=-1)]: Done 349 tasks      | elapsed:   40.0s\n",
      "[Parallel(n_jobs=-1)]: Done 400 out of 400 | elapsed:   45.7s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=4, error_score='raise',\n",
       "          estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "          fit_params=None, iid=True, n_iter=100, n_jobs=-1,\n",
       "          param_distributions={'bootstrap': [True, False], 'max_depth': [None, 5, 10, 15, 20, 25, 30, 35, 40, 45], 'max_features': ['auto', 'sqrt'], 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], 'n_estimators': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]},\n",
       "          pre_dispatch='2*n_jobs', random_state=0, refit=True,\n",
       "          return_train_score='warn', scoring=None, verbose=2)"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_v = clfs['rf_fsn_3'].vectorizer.transform(X_train)\n",
    "rf = RandomForestClassifier()\n",
    "\n",
    "rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 4, verbose=2, random_state=RANDOM_STATE, n_jobs = -1)\n",
    "\n",
    "rf_random.fit(X_train_v, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': False,\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'min_samples_leaf': 2,\n",
       " 'min_samples_split': 19,\n",
       " 'n_estimators': 18}"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_random.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.78535170413343\n",
      "f1_all = [0.51612903 0.83385777 0.43835616 0.         0.82191781 0.28571429\n",
      " 0.75522322 0.71794872 0.         0.56       0.4        0.8111332\n",
      " 0.74725275 0.59767442 0.         0.61538462 0.78586724 0.\n",
      " 0.         0.86292655 0.6122449  0.74543502 0.79108635 0.\n",
      " 0.77922078]\n",
      "f1_micro = 0.7853517041334299\n",
      "f1_macro = 0.5070949126375863\n",
      "f1_weighted = 0.7795447721161721\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.6859706362153344\n",
      "f1_all = [0.         0.77796902 0.35164835 0.         0.72829763 0.\n",
      " 0.67625    0.58823529 0.         0.22222222 0.         0.67114094\n",
      " 0.63013699 0.49438202 0.         0.         0.69902913 0.\n",
      " 0.         0.791381   0.30508475 0.56357388 0.64059197 0.\n",
      " 0.61139896]\n",
      "f1_micro = 0.6859706362153344\n",
      "f1_macro = 0.41673057865823826\n",
      "f1_weighted = 0.6768852564648038\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['rf_fsn_3_randsearch'] = Classifier('rf_fsn_3_randsearch', description=\"random forest on first name and surname trigrams, min_samples_split=10\")\n",
    "clfs['rf_fsn_3_randsearch'].fit(X_train, y_train, RandomForestClassifier().set_params(**rf_random.best_params_), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['rf_fsn_3_randsearch'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lower dev scores.. ??"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trying a grid search on parameter ranges that seem promising"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_grid = \\\n",
    "{'min_samples_split': list(range(2,7))+list(range(15,21)),\n",
    " 'n_estimators': list(range(8,12))+list(range(18,21))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min_samples_split': [2, 3, 4, 5, 6, 15, 16, 17, 18, 19, 20],\n",
       " 'n_estimators': [8, 9, 10, 11, 18, 19, 20]}"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 4 folds for each of 77 candidates, totalling 308 fits\n",
      "[CV] min_samples_split=2, n_estimators=8 .............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=4.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .............. min_samples_split=2, n_estimators=8, total=   3.1s\n",
      "[CV] min_samples_split=2, n_estimators=8 .............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    3.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .............. min_samples_split=2, n_estimators=8, total=   2.9s\n",
      "[CV] min_samples_split=2, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=2, n_estimators=8, total=   2.7s\n",
      "[CV] min_samples_split=2, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=2, n_estimators=8, total=   2.8s\n",
      "[CV] min_samples_split=2, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=2, n_estimators=9, total=   2.7s\n",
      "[CV] min_samples_split=2, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=2, n_estimators=9, total=   3.2s\n",
      "[CV] min_samples_split=2, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=2, n_estimators=9, total=   3.9s\n",
      "[CV] min_samples_split=2, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=2, n_estimators=9, total=   3.6s\n",
      "[CV] min_samples_split=2, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=10, total=   4.0s\n",
      "[CV] min_samples_split=2, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=10, total=   4.2s\n",
      "[CV] min_samples_split=2, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=10, total=   3.8s\n",
      "[CV] min_samples_split=2, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=10, total=   4.2s\n",
      "[CV] min_samples_split=2, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=11, total=   3.6s\n",
      "[CV] min_samples_split=2, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=11, total=   3.3s\n",
      "[CV] min_samples_split=2, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=11, total=   3.4s\n",
      "[CV] min_samples_split=2, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=11, total=   3.3s\n",
      "[CV] min_samples_split=2, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=18, total=   6.2s\n",
      "[CV] min_samples_split=2, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=18, total=   6.2s\n",
      "[CV] min_samples_split=2, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=18, total=   5.6s\n",
      "[CV] min_samples_split=2, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=18, total=   5.6s\n",
      "[CV] min_samples_split=2, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=19, total=   5.8s\n",
      "[CV] min_samples_split=2, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=19, total=   5.8s\n",
      "[CV] min_samples_split=2, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=19, total=   6.0s\n",
      "[CV] min_samples_split=2, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=19, total=   5.8s\n",
      "[CV] min_samples_split=2, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=20, total=   6.0s\n",
      "[CV] min_samples_split=2, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=20, total=   6.0s\n",
      "[CV] min_samples_split=2, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=20, total=   6.3s\n",
      "[CV] min_samples_split=2, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=2, n_estimators=20, total=   6.5s\n",
      "[CV] min_samples_split=3, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=3, n_estimators=8, total=   2.0s\n",
      "[CV] min_samples_split=3, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=3, n_estimators=8, total=   2.0s\n",
      "[CV] min_samples_split=3, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=3, n_estimators=8, total=   2.1s\n",
      "[CV] min_samples_split=3, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=3, n_estimators=8, total=   2.5s\n",
      "[CV] min_samples_split=3, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=3, n_estimators=9, total=   2.2s\n",
      "[CV] min_samples_split=3, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=3, n_estimators=9, total=   2.2s\n",
      "[CV] min_samples_split=3, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=3, n_estimators=9, total=   2.2s\n",
      "[CV] min_samples_split=3, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=3, n_estimators=9, total=   2.2s\n",
      "[CV] min_samples_split=3, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=10, total=   2.4s\n",
      "[CV] min_samples_split=3, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=10, total=   2.5s\n",
      "[CV] min_samples_split=3, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=10, total=   2.5s\n",
      "[CV] min_samples_split=3, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=10, total=   2.6s\n",
      "[CV] min_samples_split=3, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=11, total=   3.0s\n",
      "[CV] min_samples_split=3, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=11, total=   2.7s\n",
      "[CV] min_samples_split=3, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=11, total=   3.0s\n",
      "[CV] min_samples_split=3, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=11, total=   2.9s\n",
      "[CV] min_samples_split=3, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=18, total=   4.5s\n",
      "[CV] min_samples_split=3, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=18, total=   4.5s\n",
      "[CV] min_samples_split=3, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=18, total=   4.6s\n",
      "[CV] min_samples_split=3, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=18, total=   4.7s\n",
      "[CV] min_samples_split=3, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=19, total=   4.8s\n",
      "[CV] min_samples_split=3, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=19, total=   4.8s\n",
      "[CV] min_samples_split=3, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=19, total=   4.8s\n",
      "[CV] min_samples_split=3, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=19, total=   5.1s\n",
      "[CV] min_samples_split=3, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=20, total=   5.3s\n",
      "[CV] min_samples_split=3, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=20, total=   5.0s\n",
      "[CV] min_samples_split=3, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=20, total=   5.1s\n",
      "[CV] min_samples_split=3, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=3, n_estimators=20, total=   5.1s\n",
      "[CV] min_samples_split=4, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=4, n_estimators=8, total=   1.7s\n",
      "[CV] min_samples_split=4, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=4, n_estimators=8, total=   1.7s\n",
      "[CV] min_samples_split=4, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=4, n_estimators=8, total=   1.7s\n",
      "[CV] min_samples_split=4, n_estimators=8 .............................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .............. min_samples_split=4, n_estimators=8, total=   1.7s\n",
      "[CV] min_samples_split=4, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=4, n_estimators=9, total=   2.0s\n",
      "[CV] min_samples_split=4, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=4, n_estimators=9, total=   2.2s\n",
      "[CV] min_samples_split=4, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=4, n_estimators=9, total=   2.0s\n",
      "[CV] min_samples_split=4, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=4, n_estimators=9, total=   2.0s\n",
      "[CV] min_samples_split=4, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=10, total=   2.1s\n",
      "[CV] min_samples_split=4, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=10, total=   2.2s\n",
      "[CV] min_samples_split=4, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=10, total=   2.3s\n",
      "[CV] min_samples_split=4, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=10, total=   2.3s\n",
      "[CV] min_samples_split=4, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=11, total=   2.5s\n",
      "[CV] min_samples_split=4, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=11, total=   2.4s\n",
      "[CV] min_samples_split=4, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=11, total=   2.5s\n",
      "[CV] min_samples_split=4, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=11, total=   2.5s\n",
      "[CV] min_samples_split=4, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=18, total=   4.0s\n",
      "[CV] min_samples_split=4, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=18, total=   4.0s\n",
      "[CV] min_samples_split=4, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=18, total=   4.0s\n",
      "[CV] min_samples_split=4, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=18, total=   4.0s\n",
      "[CV] min_samples_split=4, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=19, total=   4.2s\n",
      "[CV] min_samples_split=4, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=19, total=   4.2s\n",
      "[CV] min_samples_split=4, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=19, total=   4.3s\n",
      "[CV] min_samples_split=4, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=19, total=   4.2s\n",
      "[CV] min_samples_split=4, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=20, total=   4.4s\n",
      "[CV] min_samples_split=4, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=20, total=   4.4s\n",
      "[CV] min_samples_split=4, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=20, total=   4.8s\n",
      "[CV] min_samples_split=4, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=4, n_estimators=20, total=   5.4s\n",
      "[CV] min_samples_split=5, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=5, n_estimators=8, total=   1.6s\n",
      "[CV] min_samples_split=5, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=5, n_estimators=8, total=   1.6s\n",
      "[CV] min_samples_split=5, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=5, n_estimators=8, total=   1.6s\n",
      "[CV] min_samples_split=5, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=5, n_estimators=8, total=   1.6s\n",
      "[CV] min_samples_split=5, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=5, n_estimators=9, total=   1.7s\n",
      "[CV] min_samples_split=5, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=5, n_estimators=9, total=   1.8s\n",
      "[CV] min_samples_split=5, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=5, n_estimators=9, total=   1.8s\n",
      "[CV] min_samples_split=5, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=5, n_estimators=9, total=   1.8s\n",
      "[CV] min_samples_split=5, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=10, total=   2.0s\n",
      "[CV] min_samples_split=5, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=10, total=   2.0s\n",
      "[CV] min_samples_split=5, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=10, total=   2.0s\n",
      "[CV] min_samples_split=5, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=10, total=   2.0s\n",
      "[CV] min_samples_split=5, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=11, total=   2.1s\n",
      "[CV] min_samples_split=5, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=11, total=   2.2s\n",
      "[CV] min_samples_split=5, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=11, total=   2.2s\n",
      "[CV] min_samples_split=5, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=11, total=   2.2s\n",
      "[CV] min_samples_split=5, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=18, total=   3.8s\n",
      "[CV] min_samples_split=5, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=18, total=   3.8s\n",
      "[CV] min_samples_split=5, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=18, total=   3.7s\n",
      "[CV] min_samples_split=5, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=18, total=   3.6s\n",
      "[CV] min_samples_split=5, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=19, total=   3.9s\n",
      "[CV] min_samples_split=5, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=19, total=   3.8s\n",
      "[CV] min_samples_split=5, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=19, total=   3.9s\n",
      "[CV] min_samples_split=5, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=19, total=   4.1s\n",
      "[CV] min_samples_split=5, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=20, total=   4.0s\n",
      "[CV] min_samples_split=5, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=20, total=   4.0s\n",
      "[CV] min_samples_split=5, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=20, total=   4.2s\n",
      "[CV] min_samples_split=5, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=5, n_estimators=20, total=   4.1s\n",
      "[CV] min_samples_split=6, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=6, n_estimators=8, total=   1.4s\n",
      "[CV] min_samples_split=6, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=6, n_estimators=8, total=   1.5s\n",
      "[CV] min_samples_split=6, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=6, n_estimators=8, total=   1.5s\n",
      "[CV] min_samples_split=6, n_estimators=8 .............................\n",
      "[CV] .............. min_samples_split=6, n_estimators=8, total=   1.5s\n",
      "[CV] min_samples_split=6, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=6, n_estimators=9, total=   1.7s\n",
      "[CV] min_samples_split=6, n_estimators=9 .............................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .............. min_samples_split=6, n_estimators=9, total=   1.6s\n",
      "[CV] min_samples_split=6, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=6, n_estimators=9, total=   1.7s\n",
      "[CV] min_samples_split=6, n_estimators=9 .............................\n",
      "[CV] .............. min_samples_split=6, n_estimators=9, total=   1.7s\n",
      "[CV] min_samples_split=6, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=10, total=   1.8s\n",
      "[CV] min_samples_split=6, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=10, total=   1.9s\n",
      "[CV] min_samples_split=6, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=10, total=   1.9s\n",
      "[CV] min_samples_split=6, n_estimators=10 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=10, total=   1.9s\n",
      "[CV] min_samples_split=6, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=11, total=   2.0s\n",
      "[CV] min_samples_split=6, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=11, total=   2.1s\n",
      "[CV] min_samples_split=6, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=11, total=   2.1s\n",
      "[CV] min_samples_split=6, n_estimators=11 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=11, total=   2.0s\n",
      "[CV] min_samples_split=6, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=18, total=   3.4s\n",
      "[CV] min_samples_split=6, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=18, total=   3.4s\n",
      "[CV] min_samples_split=6, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=18, total=   3.4s\n",
      "[CV] min_samples_split=6, n_estimators=18 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=18, total=   3.4s\n",
      "[CV] min_samples_split=6, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=19, total=   3.5s\n",
      "[CV] min_samples_split=6, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=19, total=   3.6s\n",
      "[CV] min_samples_split=6, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=19, total=   3.7s\n",
      "[CV] min_samples_split=6, n_estimators=19 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=19, total=   3.6s\n",
      "[CV] min_samples_split=6, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=20, total=   3.8s\n",
      "[CV] min_samples_split=6, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=20, total=   3.8s\n",
      "[CV] min_samples_split=6, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=20, total=   4.2s\n",
      "[CV] min_samples_split=6, n_estimators=20 ............................\n",
      "[CV] ............. min_samples_split=6, n_estimators=20, total=   4.1s\n",
      "[CV] min_samples_split=15, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=15, n_estimators=8, total=   1.1s\n",
      "[CV] min_samples_split=15, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=15, n_estimators=8, total=   1.2s\n",
      "[CV] min_samples_split=15, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=15, n_estimators=8, total=   1.1s\n",
      "[CV] min_samples_split=15, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=15, n_estimators=8, total=   1.1s\n",
      "[CV] min_samples_split=15, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=15, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=15, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=15, n_estimators=9, total=   1.4s\n",
      "[CV] min_samples_split=15, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=15, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=15, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=15, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=15, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=10, total=   1.4s\n",
      "[CV] min_samples_split=15, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=15, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=10, total=   1.4s\n",
      "[CV] min_samples_split=15, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=10, total=   1.4s\n",
      "[CV] min_samples_split=15, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=11, total=   1.6s\n",
      "[CV] min_samples_split=15, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=11, total=   1.7s\n",
      "[CV] min_samples_split=15, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=11, total=   1.6s\n",
      "[CV] min_samples_split=15, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=11, total=   1.5s\n",
      "[CV] min_samples_split=15, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=18, total=   2.6s\n",
      "[CV] min_samples_split=15, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=18, total=   2.7s\n",
      "[CV] min_samples_split=15, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=18, total=   2.8s\n",
      "[CV] min_samples_split=15, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=18, total=   2.5s\n",
      "[CV] min_samples_split=15, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=19, total=   3.1s\n",
      "[CV] min_samples_split=15, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=19, total=   3.1s\n",
      "[CV] min_samples_split=15, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=19, total=   2.7s\n",
      "[CV] min_samples_split=15, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=19, total=   3.2s\n",
      "[CV] min_samples_split=15, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=20, total=   3.1s\n",
      "[CV] min_samples_split=15, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=20, total=   3.0s\n",
      "[CV] min_samples_split=15, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=20, total=   2.8s\n",
      "[CV] min_samples_split=15, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=15, n_estimators=20, total=   2.8s\n",
      "[CV] min_samples_split=16, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=16, n_estimators=8, total=   1.1s\n",
      "[CV] min_samples_split=16, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=16, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=16, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=16, n_estimators=8, total=   1.1s\n",
      "[CV] min_samples_split=16, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=16, n_estimators=8, total=   1.1s\n",
      "[CV] min_samples_split=16, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=16, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=16, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=16, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=16, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=16, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=16, n_estimators=9 ............................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............. min_samples_split=16, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=16, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=16, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=16, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=16, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=16, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=11, total=   1.5s\n",
      "[CV] min_samples_split=16, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=11, total=   1.5s\n",
      "[CV] min_samples_split=16, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=11, total=   1.5s\n",
      "[CV] min_samples_split=16, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=11, total=   1.5s\n",
      "[CV] min_samples_split=16, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=18, total=   2.4s\n",
      "[CV] min_samples_split=16, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=18, total=   2.4s\n",
      "[CV] min_samples_split=16, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=18, total=   2.4s\n",
      "[CV] min_samples_split=16, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=18, total=   2.4s\n",
      "[CV] min_samples_split=16, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=19, total=   2.6s\n",
      "[CV] min_samples_split=16, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=19, total=   2.5s\n",
      "[CV] min_samples_split=16, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=19, total=   2.6s\n",
      "[CV] min_samples_split=16, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=19, total=   2.6s\n",
      "[CV] min_samples_split=16, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=20, total=   2.7s\n",
      "[CV] min_samples_split=16, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=20, total=   2.8s\n",
      "[CV] min_samples_split=16, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=20, total=   2.9s\n",
      "[CV] min_samples_split=16, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=16, n_estimators=20, total=   2.8s\n",
      "[CV] min_samples_split=17, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=17, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=17, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=17, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=17, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=17, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=17, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=17, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=17, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=17, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=17, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=17, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=17, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=17, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=17, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=17, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=17, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=17, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=17, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=17, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=17, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=11, total=   1.5s\n",
      "[CV] min_samples_split=17, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=17, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=11, total=   1.5s\n",
      "[CV] min_samples_split=17, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=17, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=18, total=   2.4s\n",
      "[CV] min_samples_split=17, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=18, total=   2.4s\n",
      "[CV] min_samples_split=17, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=18, total=   2.4s\n",
      "[CV] min_samples_split=17, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=18, total=   2.4s\n",
      "[CV] min_samples_split=17, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=19, total=   2.5s\n",
      "[CV] min_samples_split=17, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=19, total=   2.5s\n",
      "[CV] min_samples_split=17, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=19, total=   2.6s\n",
      "[CV] min_samples_split=17, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=19, total=   2.6s\n",
      "[CV] min_samples_split=17, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=20, total=   2.7s\n",
      "[CV] min_samples_split=17, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=20, total=   2.7s\n",
      "[CV] min_samples_split=17, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=20, total=   2.7s\n",
      "[CV] min_samples_split=17, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=17, n_estimators=20, total=   2.7s\n",
      "[CV] min_samples_split=18, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=18, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=18, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=18, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=18, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=18, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=18, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=18, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=18, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=18, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=18, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=18, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=18, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=18, n_estimators=9, total=   1.2s\n",
      "[CV] min_samples_split=18, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=18, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=18, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=18, n_estimators=10 ...........................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............ min_samples_split=18, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=18, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=18, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=18, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=18, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=18, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=18, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=18, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=18, total=   2.5s\n",
      "[CV] min_samples_split=18, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=18, total=   3.4s\n",
      "[CV] min_samples_split=18, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=18, total=   2.9s\n",
      "[CV] min_samples_split=18, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=18, total=   2.9s\n",
      "[CV] min_samples_split=18, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=19, total=   2.5s\n",
      "[CV] min_samples_split=18, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=19, total=   2.5s\n",
      "[CV] min_samples_split=18, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=19, total=   2.5s\n",
      "[CV] min_samples_split=18, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=19, total=   2.5s\n",
      "[CV] min_samples_split=18, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=20, total=   2.6s\n",
      "[CV] min_samples_split=18, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=20, total=   2.6s\n",
      "[CV] min_samples_split=18, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=20, total=   3.0s\n",
      "[CV] min_samples_split=18, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=18, n_estimators=20, total=   2.9s\n",
      "[CV] min_samples_split=19, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=19, n_estimators=8, total=   1.4s\n",
      "[CV] min_samples_split=19, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=19, n_estimators=8, total=   1.3s\n",
      "[CV] min_samples_split=19, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=19, n_estimators=8, total=   1.1s\n",
      "[CV] min_samples_split=19, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=19, n_estimators=8, total=   1.2s\n",
      "[CV] min_samples_split=19, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=19, n_estimators=9, total=   1.3s\n",
      "[CV] min_samples_split=19, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=19, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=19, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=19, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=19, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=19, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=19, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=10, total=   1.2s\n",
      "[CV] min_samples_split=19, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=10, total=   1.2s\n",
      "[CV] min_samples_split=19, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=10, total=   1.2s\n",
      "[CV] min_samples_split=19, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=10, total=   1.2s\n",
      "[CV] min_samples_split=19, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=19, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=19, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=19, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=19, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=18, total=   2.3s\n",
      "[CV] min_samples_split=19, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=18, total=   2.3s\n",
      "[CV] min_samples_split=19, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=18, total=   2.3s\n",
      "[CV] min_samples_split=19, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=18, total=   2.3s\n",
      "[CV] min_samples_split=19, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=19, total=   2.4s\n",
      "[CV] min_samples_split=19, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=19, total=   2.4s\n",
      "[CV] min_samples_split=19, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=19, total=   2.4s\n",
      "[CV] min_samples_split=19, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=19, total=   2.5s\n",
      "[CV] min_samples_split=19, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=20, total=   2.6s\n",
      "[CV] min_samples_split=19, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=20, total=   2.5s\n",
      "[CV] min_samples_split=19, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=20, total=   2.6s\n",
      "[CV] min_samples_split=19, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=19, n_estimators=20, total=   2.6s\n",
      "[CV] min_samples_split=20, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=20, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=20, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=20, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=20, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=20, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=20, n_estimators=8 ............................\n",
      "[CV] ............. min_samples_split=20, n_estimators=8, total=   1.0s\n",
      "[CV] min_samples_split=20, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=20, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=20, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=20, n_estimators=9, total=   1.3s\n",
      "[CV] min_samples_split=20, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=20, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=20, n_estimators=9 ............................\n",
      "[CV] ............. min_samples_split=20, n_estimators=9, total=   1.1s\n",
      "[CV] min_samples_split=20, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=10, total=   1.2s\n",
      "[CV] min_samples_split=20, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=10, total=   1.2s\n",
      "[CV] min_samples_split=20, n_estimators=10 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=20, n_estimators=10 ...........................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ............ min_samples_split=20, n_estimators=10, total=   1.3s\n",
      "[CV] min_samples_split=20, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=11, total=   1.5s\n",
      "[CV] min_samples_split=20, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=20, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=11, total=   1.4s\n",
      "[CV] min_samples_split=20, n_estimators=11 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=11, total=   1.3s\n",
      "[CV] min_samples_split=20, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=18, total=   2.3s\n",
      "[CV] min_samples_split=20, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=18, total=   2.3s\n",
      "[CV] min_samples_split=20, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=18, total=   2.3s\n",
      "[CV] min_samples_split=20, n_estimators=18 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=18, total=   2.9s\n",
      "[CV] min_samples_split=20, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=19, total=   2.9s\n",
      "[CV] min_samples_split=20, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=19, total=   3.0s\n",
      "[CV] min_samples_split=20, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=19, total=   2.9s\n",
      "[CV] min_samples_split=20, n_estimators=19 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=19, total=   2.8s\n",
      "[CV] min_samples_split=20, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=20, total=   2.5s\n",
      "[CV] min_samples_split=20, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=20, total=   2.5s\n",
      "[CV] min_samples_split=20, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=20, total=   2.6s\n",
      "[CV] min_samples_split=20, n_estimators=20 ...........................\n",
      "[CV] ............ min_samples_split=20, n_estimators=20, total=   2.5s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 308 out of 308 | elapsed: 14.1min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=4, error_score='raise',\n",
       "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'min_samples_split': [2, 3, 4, 5, 6, 15, 16, 17, 18, 19, 20], 'n_estimators': [8, 9, 10, 11, 18, 19, 20]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=2)"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_v = clfs['rf_fsn_3'].vectorizer.transform(X_train)\n",
    "rf = RandomForestClassifier()\n",
    "\n",
    "rf_grid = GridSearchCV(estimator = rf, param_grid = params_grid, cv = 4, verbose=2)\n",
    "\n",
    "rf_grid.fit(X_train_v, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min_samples_split': 3, 'n_estimators': 19}"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.9888506163886874\n",
      "f1_all = [1.         0.98825897 0.95260664 0.8        0.9912381  1.\n",
      " 0.98593498 0.99310345 1.         1.         1.         0.99816514\n",
      " 0.99393939 0.98696088 1.         0.94117647 0.98696088 1.\n",
      " 1.         0.98612341 1.         0.99792531 0.99579992 0.8\n",
      " 0.98497496]\n",
      "f1_micro = 0.9888506163886874\n",
      "f1_macro = 0.9753267401507796\n",
      "f1_weighted = 0.9887995635747706\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.7327351821642197\n",
      "f1_all = [0.22222222 0.82764654 0.43564356 0.         0.76276958 0.\n",
      " 0.72578616 0.65277778 0.         0.22222222 0.66666667 0.74172185\n",
      " 0.67248908 0.55244755 0.         0.         0.75903614 0.\n",
      " 0.         0.81836327 0.43835616 0.63120567 0.69018743 0.\n",
      " 0.68932039]\n",
      "f1_micro = 0.7327351821642198\n",
      "f1_macro = 0.5004220145861188\n",
      "f1_weighted = 0.7267706695958368\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['rf_fsn_3_gridsearch'] = Classifier('rf_fsn_3_gridsearch', description=\"random forest on first name and surname trigrams, min_samples_split=10\")\n",
    "clfs['rf_fsn_3_gridsearch'].fit(X_train, y_train, RandomForestClassifier().set_params(**rf_grid.best_params_), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['rf_fsn_3_gridsearch'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.2577048585931835\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.32821275 0.         0.         0.         0.         0.\n",
      " 0.         0.         1.         0.         0.         1.\n",
      " 0.         0.61322789 0.         0.         0.00152555 0.\n",
      " 0.        ]\n",
      "f1_micro = 0.2577048585931835\n",
      "f1_macro = 0.11771864794057052\n",
      "f1_weighted = 0.15871556064839917\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.2544861337683524\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.33349633 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.62331288 0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.2544861337683524\n",
      "f1_macro = 0.04556234361685335\n",
      "f1_weighted = 0.14947281870442197\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['ab_fsn_3'] = Classifier('ab_fsn_3', description=\"Adaboost on first name and surname trigrams\")\n",
    "clfs['ab_fsn_3'].fit(X_train, y_train, AdaBoostClassifier(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['ab_fsn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.2577048585931835\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.32821275 0.         0.         0.         0.         0.\n",
      " 0.         0.         1.         0.         0.         1.\n",
      " 0.         0.61322789 0.         0.         0.00152555 0.\n",
      " 0.        ]\n",
      "f1_micro = 0.2577048585931835\n",
      "f1_macro = 0.11771864794057052\n",
      "f1_weighted = 0.15871556064839917\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.2544861337683524\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.33349633 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.62331288 0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.2544861337683524\n",
      "f1_macro = 0.04556234361685335\n",
      "f1_weighted = 0.14947281870442197\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['ab_fsn_23'] = Classifier('ab_fsn_23', description=\"Adaboost on first name and surname bigrams and trigrams\")\n",
    "clfs['ab_fsn_23'].fit(X_train, y_train, AdaBoostClassifier(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,2) + get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['ab_fsn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Default AdaBoost classifier does very poorly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.8733683828861494\n",
      "f1_all = [0.9047619  0.91140896 0.65306122 0.         0.88149533 0.66666667\n",
      " 0.83948897 0.87088608 0.         0.875      0.76923077 0.93690249\n",
      " 0.87859825 0.83214649 1.         0.8        0.87174349 1.\n",
      " 0.         0.89734058 0.86904762 0.86956522 0.87401869 0.8\n",
      " 0.89930556]\n",
      "f1_micro = 0.8733683828861494\n",
      "f1_macro = 0.756026730547816\n",
      "f1_weighted = 0.871797655027933\n",
      "\n",
      "Scores on dev\n",
      "accuracy = 0.7495921696574225\n",
      "f1_all = [0.         0.82477876 0.49019608 0.         0.76521739 0.\n",
      " 0.7457405  0.75471698 0.         0.22222222 0.         0.76129032\n",
      " 0.70781893 0.6557377  0.         0.         0.77014925 0.\n",
      " 0.         0.83561644 0.35483871 0.63043478 0.70627803 0.\n",
      " 0.68965517]\n",
      "f1_micro = 0.7495921696574225\n",
      "f1_macro = 0.47212815587710083\n",
      "f1_weighted = 0.7434768627829413\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['lr_fsn_3'] = Classifier('lr_fsn_3', description=\"logistic regression on first name and surname trigrams\")\n",
    "clfs['lr_fsn_3'].fit(X_train, y_train, LogisticRegression(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['lr_fsn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.8881435823060189\n",
      "f1_all = [0.93023256 0.92013201 0.71666667 0.         0.89422356 0.8\n",
      " 0.85394829 0.89826303 1.         0.97297297 0.93333333 0.94095238\n",
      " 0.90663391 0.85192698 1.         0.875      0.87887888 1.\n",
      " 1.         0.90684301 0.92655367 0.89077413 0.88696958 1.\n",
      " 0.92020374]\n",
      "f1_micro = 0.888143582306019\n",
      "f1_macro = 0.8761803475423018\n",
      "f1_weighted = 0.8871858795414143\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.7517672648178357\n",
      "f1_all = [0.44444444 0.82005372 0.48598131 0.         0.76431718 0.\n",
      " 0.75664894 0.7375     0.         0.2        0.         0.73202614\n",
      " 0.70682731 0.64686469 0.         0.         0.75801749 0.\n",
      " 0.         0.83984375 0.38095238 0.64827586 0.71668533 0.\n",
      " 0.70422535]\n",
      "f1_micro = 0.7517672648178357\n",
      "f1_macro = 0.49250780441145037\n",
      "f1_weighted = 0.74667975720378\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['lr_fsn_23'] = Classifier('lr_fsn_23', description=\"logistic regression on first name and surname bigrams and trigrams\")\n",
    "clfs['lr_fsn_23'].fit(X_train, y_train, LogisticRegression(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,2) + get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['lr_fsn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.25734227701232776\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.32786074 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.61449693 0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.25734227701232776\n",
      "f1_macro = 0.03769430689162556\n",
      "f1_weighted = 0.15847718021203616\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.2544861337683524\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.33325189 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.62561576 0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.2544861337683524\n",
      "f1_macro = 0.045660364620165025\n",
      "f1_weighted = 0.1497271751712573\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['svmr_fsn_3'] = Classifier('svmr_fsn_3', description=\"SVM with rbf kernel on first name and surname trigrams\")\n",
    "clfs['svmr_fsn_3'].fit(X_train, y_train, SVC(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['svmr_fsn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.25734227701232776\n",
      "f1_all = [0.         0.00268456 0.         0.         0.         0.\n",
      " 0.32771721 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.61300755 0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.25734227701232776\n",
      "f1_macro = 0.03773637285332313\n",
      "f1_weighted = 0.1585835895582161\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.2544861337683524\n",
      "f1_all = [0.         0.00368324 0.         0.         0.         0.\n",
      " 0.33300733 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.62407862 0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.2544861337683524\n",
      "f1_macro = 0.04575091429972625\n",
      "f1_weighted = 0.15001791516576066\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['svmr_fsn_23'] = Classifier('svmr_fsn_23', description=\"SVM with rbf kernel on first name and surname bigrams and trigrams\")\n",
    "clfs['svmr_fsn_23'].fit(X_train, y_train, SVC(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,2) + get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['svmr_fsn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVM with rbf kernel performs very poorly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.9509608411892676\n",
      "f1_all = [1.         0.96505824 0.92009685 0.         0.95860235 1.\n",
      " 0.92467652 0.97663551 1.         0.97297297 0.93333333 0.99267399\n",
      " 0.9684466  0.91935484 1.         1.         0.95085256 1.\n",
      " 1.         0.95481221 0.98378378 0.96963351 0.94785276 1.\n",
      " 0.97822446]\n",
      "f1_micro = 0.9509608411892677\n",
      "f1_macro = 0.9326804197654534\n",
      "f1_weighted = 0.9507361854701186\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.745241979336596\n",
      "f1_all = [0.54545455 0.82426405 0.5        0.         0.76129032 0.\n",
      " 0.74365482 0.73053892 0.         0.36363636 0.5        0.725\n",
      " 0.71255061 0.62046205 0.         0.         0.71779141 0.\n",
      " 0.         0.84188912 0.4057971  0.65277778 0.70716889 0.\n",
      " 0.70093458]\n",
      "f1_micro = 0.745241979336596\n",
      "f1_macro = 0.5263433600310139\n",
      "f1_weighted = 0.7417027205641732\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['svml_fsn_3'] = Classifier('svml_fsn_3', description=\"SVM with linear kernel on first name and surname trigrams\")\n",
    "clfs['svml_fsn_3'].fit(X_train, y_train, SVC(kernel='linear'), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['svml_fsn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.9635605511240029\n",
      "f1_all = [1.         0.976      0.94588235 0.8        0.96891585 1.\n",
      " 0.93721145 0.99307159 1.         0.97297297 1.         0.99449541\n",
      " 0.98660171 0.94343434 1.         1.         0.97467072 1.\n",
      " 1.         0.96082474 1.         0.98550725 0.96307692 1.\n",
      " 0.97993311]\n",
      "f1_micro = 0.9635605511240029\n",
      "f1_macro = 0.9753039366967589\n",
      "f1_weighted = 0.963497372116347\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.7468733007069059\n",
      "f1_all = [0.5        0.81906443 0.50746269 0.5        0.76567657 0.\n",
      " 0.75079669 0.72727273 0.         0.4        1.         0.72049689\n",
      " 0.70967742 0.6442953  0.         0.         0.71826625 0.\n",
      " 0.         0.82507583 0.375      0.68531469 0.70356704 0.\n",
      " 0.71889401]\n",
      "f1_micro = 0.7468733007069059\n",
      "f1_macro = 0.574802882477354\n",
      "f1_weighted = 0.7434744001299007\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['svml_fsn_23'] = Classifier('svlm_fsn_23', description=\"SVM with linear kernel on first name and surname bigrams and trigrams\")\n",
    "clfs['svml_fsn_23'].fit(X_train, y_train, SVC(kernel='linear'), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,2) + get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['svml_fsn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVM with linear kernel gives much higher scores for both train and dev than SVM with rbf kernel, however dev scores are similar to those given by Logistic Regression, which was much quicker to train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.19760696156635243\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.33000303 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.1976069615663524\n",
      "f1_macro = 0.013200121102028459\n",
      "f1_weighted = 0.06521089558199339\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.20174007612833061\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.33574661 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.2017400761283306\n",
      "f1_macro = 0.01598793363499246\n",
      "f1_weighted = 0.06773354592181961\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['svmp_fsn_3'] = Classifier('svmp_fsn_3', description=\"SVM with poly (degree=3) kernel on first name and surname trigrams\")\n",
    "clfs['svmp_fsn_3'].fit(X_train, y_train, SVC(kernel='poly'), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['svmp_fsn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.19760696156635243\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.33000303 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.1976069615663524\n",
      "f1_macro = 0.013200121102028459\n",
      "f1_weighted = 0.06521089558199339\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.20174007612833061\n",
      "f1_all = [0.         0.         0.         0.         0.         0.\n",
      " 0.33574661 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.        ]\n",
      "f1_micro = 0.2017400761283306\n",
      "f1_macro = 0.01598793363499246\n",
      "f1_weighted = 0.06773354592181961\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['svmp_fsn_23'] = Classifier('svlp_fsn_23', description=\"SVM with poly (degree=3) kernel on first name and surname bigrams and trigrams\")\n",
    "clfs['svmp_fsn_23'].fit(X_train, y_train, SVC(kernel='poly'), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,2) + get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['svmp_fsn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Default SVM with poly kernel gives very poor results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## k Nearest Neighbours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.7531725888324873\n",
      "f1_all = [0.57142857 0.81125227 0.59125964 0.         0.77423775 0.6\n",
      " 0.7361483  0.66285714 0.         0.64285714 0.4        0.71861472\n",
      " 0.71633238 0.56790123 0.         0.         0.66290868 0.\n",
      " 0.         0.82891125 0.64864865 0.77377049 0.73552686 0.\n",
      " 0.78373383]\n",
      "f1_micro = 0.7531725888324873\n",
      "f1_macro = 0.48905555662058264\n",
      "f1_weighted = 0.7485695781339124\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.6324089178901577\n",
      "f1_all = [0.22222222 0.75434243 0.37931034 0.         0.65440465 0.\n",
      " 0.63785047 0.55172414 0.         0.22222222 0.         0.625\n",
      " 0.48756219 0.38888889 0.         0.         0.46052632 0.\n",
      " 0.         0.72863248 0.20833333 0.5546875  0.57853403 0.\n",
      " 0.68269231]\n",
      "f1_micro = 0.6324089178901577\n",
      "f1_macro = 0.3874730246533145\n",
      "f1_weighted = 0.6204713865039307\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['knn_fsn_3'] = Classifier('knn_fsn_3', description=\"kNN on first name and surname trigrams\")\n",
    "clfs['knn_fsn_3'].fit(X_train, y_train, KNeighborsClassifier(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['knn_fsn_3'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on train\n",
      "accuracy = 0.751178390137781\n",
      "f1_all = [0.64864865 0.8202765  0.52763819 0.         0.77218225 0.66666667\n",
      " 0.72674176 0.69060773 0.         0.62068966 0.4        0.67973856\n",
      " 0.70070922 0.5861244  0.         0.         0.66366704 0.\n",
      " 0.         0.83173653 0.62585034 0.76339286 0.73767095 0.\n",
      " 0.79633028]\n",
      "f1_micro = 0.751178390137781\n",
      "f1_macro = 0.4903468633055779\n",
      "f1_weighted = 0.7468081774025634\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on dev\n",
      "accuracy = 0.6373028820010875\n",
      "f1_all = [0.2        0.76112026 0.31932773 0.         0.64313725 0.\n",
      " 0.63555818 0.56944444 0.         0.         0.         0.625\n",
      " 0.53921569 0.37450199 0.         0.         0.5083612  0.\n",
      " 0.         0.74143302 0.17777778 0.61176471 0.58441558 0.\n",
      " 0.65700483]\n",
      "f1_micro = 0.6373028820010875\n",
      "f1_macro = 0.37847917513784385\n",
      "f1_weighted = 0.6252560436081231\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clfs['knn_fsn_23'] = Classifier('knn_fsn_23', description=\"kNN on first name and surname bigrams and trigrams\")\n",
    "clfs['knn_fsn_23'].fit(X_train, y_train, KNeighborsClassifier(), vectorizer=CountVectorizer(analyzer=lambda x:get_ngrams(x,2) + get_ngrams(x,3)), metrics_dict=metrics_dict)\n",
    "clfs['knn_fsn_23'].score(X_dev, y_dev, 'dev', metrics_dict=metrics_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "kNN is unable to fit data very well (it has high bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trying out embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dense, Embedding, Input, GRU\n",
    "from keras.models import Model\n",
    "from keras.utils import to_categorical\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get label/id for a character\n",
    "def get_char_val(ch):\n",
    "    n = ord(ch) - ord('a')\n",
    "    if n>25 or n<0:\n",
    "        n = 26\n",
    "    return n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get np array of labels/ids corresponding to characters in a name\n",
    "def get_name_val(name):\n",
    "    return np.array([get_char_val(c) for c in name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert sequence of names to sequences of label arrays\n",
    "def get_names_vals(names):\n",
    "    return np.array([get_name_val(name) for name in names])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get character corresponding to label\n",
    "def get_val_char(val):\n",
    "    return chr(val + ord('a'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get name corresponding to sequence of labels\n",
    "def get_val_name(vals):\n",
    "    return np.array([get_val_char(v) for v in vals])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 5 #embedding vector dimensions for a character (smaller value chosen to avoid overfitting small dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_classes = df['Community'].unique().shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_vocab = 26"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN over characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Many to one sequence model\n",
    "\n",
    "With varying sequence lengths (stochastic / 1 name per batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Over last names only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_generator(X,y):\n",
    "    for x_i, y_i in zip(X,y):\n",
    "        yield (get_name_val(x_i).reshape(1,-1), to_categorical(y_i, num_classes=n_classes).reshape(1,-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_predict(names, model, argmax=True, x_processor=None):\n",
    "    if x_processor is None:\n",
    "        x_processor = lambda x: get_name_val(x).reshape(1,-1)\n",
    "        \n",
    "    predictions = []\n",
    "    for name in names:\n",
    "        result = model.predict( x_processor(name) )\n",
    "        if argmax:\n",
    "            result = np.argmax(result)\n",
    "        predictions += [result]\n",
    "    return np.array(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(df['Surname'][surnames_mask].values, labels[surnames_mask], random_state = RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = Input(batch_shape=(1,None,))\n",
    "x = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)(inp)\n",
    "x = GRU(64)(x)\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=inp, outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (1, None)                 0         \n",
      "_________________________________________________________________\n",
      "embedding_3 (Embedding)      (1, None, 5)              130       \n",
      "_________________________________________________________________\n",
      "gru_4 (GRU)                  (1, 64)                   13440     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (1, 25)                   1625      \n",
      "=================================================================\n",
      "Total params: 15,195\n",
      "Trainable params: 15,195\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11180/11180 [==============================] - 83s 7ms/step - loss: 2.0481 - acc: 0.3598 - val_loss: 1.7642 - val_acc: 0.4752\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.76423, saving model to model.hdf5\n",
      "Epoch 2/20\n",
      "11180/11180 [==============================] - 72s 6ms/step - loss: 1.6271 - acc: 0.5209 - val_loss: 1.4956 - val_acc: 0.5573\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.76423 to 1.49557, saving model to model.hdf5\n",
      "Epoch 3/20\n",
      "11180/11180 [==============================] - 70s 6ms/step - loss: 1.3940 - acc: 0.5886 - val_loss: 1.3635 - val_acc: 0.5991\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.49557 to 1.36348, saving model to model.hdf5\n",
      "Epoch 4/20\n",
      "11180/11180 [==============================] - 56s 5ms/step - loss: 1.2408 - acc: 0.6326 - val_loss: 1.2959 - val_acc: 0.6206\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.36348 to 1.29591, saving model to model.hdf5\n",
      "Epoch 5/20\n",
      "11180/11180 [==============================] - 79s 7ms/step - loss: 1.1420 - acc: 0.6574 - val_loss: 1.2592 - val_acc: 0.6407\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.29591 to 1.25921, saving model to model.hdf5\n",
      "Epoch 6/20\n",
      "11180/11180 [==============================] - 62s 6ms/step - loss: 1.0686 - acc: 0.6758 - val_loss: 1.2436 - val_acc: 0.6482\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.25921 to 1.24355, saving model to model.hdf5\n",
      "Epoch 7/20\n",
      "11180/11180 [==============================] - 57s 5ms/step - loss: 1.0125 - acc: 0.6887 - val_loss: 1.2411 - val_acc: 0.6405\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.24355 to 1.24110, saving model to model.hdf5\n",
      "Epoch 8/20\n",
      "11180/11180 [==============================] - 72s 6ms/step - loss: 0.9685 - acc: 0.7003 - val_loss: 1.2375 - val_acc: 0.6617\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.24110 to 1.23751, saving model to model.hdf5\n",
      "Epoch 9/20\n",
      "11180/11180 [==============================] - 57s 5ms/step - loss: 0.9324 - acc: 0.7074 - val_loss: 1.2473 - val_acc: 0.6507\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 1.23751\n",
      "Epoch 10/20\n",
      "11180/11180 [==============================] - 37s 3ms/step - loss: 0.9076 - acc: 0.7147 - val_loss: 1.2535 - val_acc: 0.6563\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 1.23751\n",
      "Epoch 11/20\n",
      "11180/11180 [==============================] - 37s 3ms/step - loss: 0.8805 - acc: 0.7202 - val_loss: 1.2538 - val_acc: 0.6523\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 1.23751\n",
      "Epoch 12/20\n",
      "11180/11180 [==============================] - 61s 5ms/step - loss: 0.8671 - acc: 0.7252 - val_loss: 1.2588 - val_acc: 0.6531\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 1.23751\n",
      "Epoch 00012: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x175b81e9278>"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_lastname.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator(X_train,y_train), validation_data=rnn_generator(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict['accuracy'] = accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train..\n",
      "f1_all = [0.71794872 0.83224401 0.34       0.         0.75893887 0.5\n",
      " 0.66012216 0.70588235 0.         0.10526316 0.         0.69487751\n",
      " 0.70113493 0.53056769 0.         0.6        0.81031308 0.\n",
      " 0.         0.80210935 0.16363636 0.65259117 0.66214058 0.\n",
      " 0.70700637]\n",
      "f1_micro = 0.7176207513416817\n",
      "f1_macro = 0.43779105203222535\n",
      "f1_weighted = 0.7092129072705889\n",
      "accuracy = 0.7176207513416816\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "yhat_train = rnn_predict(X_train, model)\n",
    "print('Train..')\n",
    "for metric, metric_fn in metrics_dict.items():\n",
    "    print(metric,\"=\", metric_fn(y_train, yhat_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dev..\n",
      "f1_all = [0.5        0.78793591 0.2        0.         0.72788354 0.\n",
      " 0.60892388 0.58333333 0.         0.         0.         0.68789809\n",
      " 0.53846154 0.45751634 0.         0.         0.76080692 0.\n",
      " 0.         0.78219396 0.11111111 0.53374233 0.56374269 0.\n",
      " 0.66046512]\n",
      "f1_micro = 0.661658170110008\n",
      "f1_macro = 0.42520073786658835\n",
      "f1_weighted = 0.6512782685015335\n",
      "accuracy = 0.661658170110008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "yhat_dev = rnn_predict(X_dev, model)\n",
    "print('Dev..')\n",
    "for metric, metric_fn in metrics_dict.items():\n",
    "    print(metric,\"=\", metric_fn(y_dev, yhat_dev))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exploring name representations (last GRU state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adjusted Random Index is a metric which gives the similarity between 2 different sets of clusters.\n",
    "\n",
    "Here we cluster name vector representations using KMeans and compare them with the actual clusters of communities to get an idea of how close name vectors from the same community are to each other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import adjusted_rand_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "gru_layer = Model(inputs=model.input, outputs=model.layers[2].output)\n",
    "name_vecs = rnn_predict(df['Surname'][surnames_mask].values, gru_layer, argmax=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "name_vecs = np.squeeze(name_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "km = KMeans(n_clusters=n_classes, random_state=RANDOM_STATE).fit(name_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_vecs_clusters = km.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1531286968230099"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#how well do name vector clusters correspond with community\n",
    "adjusted_rand_score(name_vecs_clusters, labels[surnames_mask])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "... not very well"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "May have better results with more data?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Over full names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Concatenate, BatchNormalization, Dropout, Bidirectional\n",
    "from keras import regularizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_generator2(X,y):\n",
    "    while True:\n",
    "        for (x_fn_i, x_sn_i), y_i in zip(X,y):\n",
    "            yield ({'fn':get_name_val(x_fn_i).reshape(1,-1), 'sn':get_name_val(x_sn_i).reshape(1,-1)}, to_categorical(y_i, num_classes=n_classes).reshape(1,-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(df[['Name','Surname']][fullnames_mask].values, labels[fullnames_mask], random_state = RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Considerations: Should first names and surnames use same or different RNNs? \n",
    "    \n",
    "There would be fewer parameters with the same RNN - beneficial for limited data, but first names and surnames seem to be fundamentally different sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Different GRUs for first and last names\n",
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "fn_inp = Input(batch_shape=(1,None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "fn_x = GRU(32)(fn_x)\n",
    "sn_inp = Input(batch_shape=(1,None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "sn_x = GRU(32)(sn_x)\n",
    "x = Concatenate()([fn_x, sn_x])\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "fn (InputLayer)                 (1, None)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sn (InputLayer)                 (1, None)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (1, None, 5)         130         fn[0][0]                         \n",
      "                                                                 sn[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "gru_1 (GRU)                     (1, 32)              3648        embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "gru_2 (GRU)                     (1, 32)              3648        embedding_1[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (1, 64)              0           gru_1[0][0]                      \n",
      "                                                                 gru_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (1, 25)              1625        concatenate_1[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 9,051\n",
      "Trainable params: 9,051\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 1.6725 - acc: 0.4751 - val_loss: 1.5626 - val_acc: 0.5101\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.56262, saving model to model_rnn_fullname.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 49s 4ms/step - loss: 1.4377 - acc: 0.5644 - val_loss: 1.3914 - val_acc: 0.5810\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.56262 to 1.39143, saving model to model_rnn_fullname.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 47s 4ms/step - loss: 1.2608 - acc: 0.6171 - val_loss: 1.2659 - val_acc: 0.6267\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.39143 to 1.26588, saving model to model_rnn_fullname.hdf5\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 48s 4ms/step - loss: 1.1240 - acc: 0.6601 - val_loss: 1.1824 - val_acc: 0.6558\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.26588 to 1.18239, saving model to model_rnn_fullname.hdf5\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 47s 4ms/step - loss: 1.0218 - acc: 0.6923 - val_loss: 1.1405 - val_acc: 0.6697\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.18239 to 1.14051, saving model to model_rnn_fullname.hdf5\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 49s 4ms/step - loss: 0.9377 - acc: 0.7161 - val_loss: 1.0956 - val_acc: 0.6887\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.14051 to 1.09555, saving model to model_rnn_fullname.hdf5\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 48s 4ms/step - loss: 0.8700 - acc: 0.7343 - val_loss: 1.0703 - val_acc: 0.6960\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.09555 to 1.07034, saving model to model_rnn_fullname.hdf5\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 48s 4ms/step - loss: 0.8146 - acc: 0.7495 - val_loss: 1.0592 - val_acc: 0.7001\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.07034 to 1.05923, saving model to model_rnn_fullname.hdf5\n",
      "Epoch 9/20\n",
      "11032/11032 [==============================] - 48s 4ms/step - loss: 0.7652 - acc: 0.7622 - val_loss: 1.0594 - val_acc: 0.7004\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 1.05923\n",
      "Epoch 10/20\n",
      "11032/11032 [==============================] - 47s 4ms/step - loss: 0.7193 - acc: 0.7760 - val_loss: 1.0729 - val_acc: 0.6933\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 1.05923\n",
      "Epoch 11/20\n",
      "11032/11032 [==============================] - 51s 5ms/step - loss: 0.6855 - acc: 0.7871 - val_loss: 1.0731 - val_acc: 0.6974\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 1.05923\n",
      "Epoch 12/20\n",
      "11032/11032 [==============================] - 49s 4ms/step - loss: 0.6509 - acc: 0.7961 - val_loss: 1.0708 - val_acc: 0.7031\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 1.05923\n",
      "Epoch 00012: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x175d06362b0>"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_fullname.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator2(X_train,y_train), validation_data=rnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Same GRU for first and last names\n",
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "gru = GRU(48)\n",
    "fn_inp = Input(batch_shape=(1,None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "fn_x = gru(fn_x)\n",
    "sn_inp = Input(batch_shape=(1,None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "sn_x = gru(sn_x)\n",
    "x = Concatenate()([fn_x, sn_x])\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "fn (InputLayer)                 (1, None)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sn (InputLayer)                 (1, None)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (1, None, 5)         130         fn[0][0]                         \n",
      "                                                                 sn[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "gru_15 (GRU)                    (1, 48)              7776        embedding_6[0][0]                \n",
      "                                                                 embedding_6[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (1, 96)              0           gru_15[0][0]                     \n",
      "                                                                 gru_15[1][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (1, 25)              2425        concatenate_5[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 10,331\n",
      "Trainable params: 10,331\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 83s 8ms/step - loss: 2.0723 - acc: 0.3282 - val_loss: 1.8742 - val_acc: 0.4035\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.87418, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 1.6695 - acc: 0.4815 - val_loss: 1.5098 - val_acc: 0.5313\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.87418 to 1.50981, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 71s 6ms/step - loss: 1.3613 - acc: 0.5862 - val_loss: 1.2978 - val_acc: 0.6150\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.50981 to 1.29777, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 54s 5ms/step - loss: 1.1541 - acc: 0.6458 - val_loss: 1.1778 - val_acc: 0.6539\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.29777 to 1.17777, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 37s 3ms/step - loss: 1.0088 - acc: 0.6911 - val_loss: 1.1054 - val_acc: 0.6686\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.17777 to 1.10538, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 36s 3ms/step - loss: 0.8934 - acc: 0.7234 - val_loss: 1.0673 - val_acc: 0.6824\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.10538 to 1.06727, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 33s 3ms/step - loss: 0.8061 - acc: 0.7487 - val_loss: 1.0566 - val_acc: 0.6854\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.06727 to 1.05661, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 42s 4ms/step - loss: 0.7409 - acc: 0.7690 - val_loss: 1.0277 - val_acc: 0.6936\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.05661 to 1.02771, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 9/20\n",
      "11032/11032 [==============================] - 40s 4ms/step - loss: 0.6854 - acc: 0.7863 - val_loss: 1.0249 - val_acc: 0.6966\n",
      "\n",
      "Epoch 00009: val_loss improved from 1.02771 to 1.02492, saving model to model_rnn_fullname1.hdf5\n",
      "Epoch 10/20\n",
      "11032/11032 [==============================] - 40s 4ms/step - loss: 0.6450 - acc: 0.8005 - val_loss: 1.0332 - val_acc: 0.7012\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 1.02492\n",
      "Epoch 11/20\n",
      "11032/11032 [==============================] - 36s 3ms/step - loss: 0.6129 - acc: 0.8086 - val_loss: 1.0367 - val_acc: 0.7036\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 1.02492\n",
      "Epoch 12/20\n",
      "11032/11032 [==============================] - 33s 3ms/step - loss: 0.5947 - acc: 0.8152 - val_loss: 1.0494 - val_acc: 0.7088\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 1.02492\n",
      "Epoch 13/20\n",
      "11032/11032 [==============================] - 35s 3ms/step - loss: 0.5613 - acc: 0.8248 - val_loss: 1.0573 - val_acc: 0.7096\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 1.02492\n",
      "Epoch 00013: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2885edffcf8>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_fullname1.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator2(X_train,y_train), validation_data=rnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similar performance with similar number of parameters.\n",
    "\n",
    "Difficult to say whether there is any benefit in sharing RNN layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fullname_processor(fullname):\n",
    "    return {'fn':get_name_val(fullname[0]).reshape(1,-1),'sn':get_name_val(fullname[1]).reshape(1,-1)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_train2 = rnn_predict(X_train, model, x_processor=fullname_processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_dev2 = rnn_predict(X_dev, model, x_processor=fullname_processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train..\n",
      "f1_all = [0.68421053 0.85723794 0.595      0.         0.82251407 0.76923077\n",
      " 0.79509871 0.72139303 0.         0.70967742 0.4        0.77480916\n",
      " 0.76869965 0.72095333 1.         0.75       0.84020101 0.\n",
      " 0.         0.86227192 0.65116279 0.76129032 0.81085332 0.8\n",
      " 0.78913738]\n",
      "f1_micro = 0.807741116751269\n",
      "f1_macro = 0.6353496540877878\n",
      "f1_weighted = 0.8065696191677791\n",
      "accuracy = 0.807741116751269\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print('Train..')\n",
    "for metric, metric_fn in metrics_dict.items():\n",
    "    print(metric,\"=\", metric_fn(y_train, yhat_train2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dev..\n",
      "f1_all = [0.         0.78888889 0.3442623  0.         0.73060345 0.\n",
      " 0.72821847 0.57861635 0.         0.4        0.         0.5748503\n",
      " 0.5754386  0.53459119 0.         0.4        0.74404762 0.\n",
      " 0.         0.78024194 0.56       0.57857143 0.65336658 0.\n",
      " 0.63070539]\n",
      "f1_micro = 0.6965742251223491\n",
      "f1_macro = 0.45725726198493116\n",
      "f1_weighted = 0.693128811297419\n",
      "accuracy = 0.6965742251223491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print('Dev..')\n",
    "for metric, metric_fn in metrics_dict.items():\n",
    "    print(metric,\"=\", metric_fn(y_dev, yhat_dev2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploring name representations (last GRU state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "gru_fn = Model(inputs=model.input, outputs=model.layers[3].get_output_at(0))\n",
    "gru_sn = Model(inputs=model.input, outputs=model.layers[3].get_output_at(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.merge.Concatenate at 0x2885f049208>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gru_vecs1 = np.squeeze(rnn_predict(df[['Name','Surname']][fullnames_mask].values, gru_fn, argmax=False, x_processor=fullname_processor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "gru_vecs2 = np.squeeze(rnn_predict(df[['Name','Surname']][fullnames_mask].values, gru_sn, argmax=False, x_processor=fullname_processor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_vecs2 = np.hstack([gru_vecs1, gru_vecs2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "km2 = KMeans(n_clusters=n_classes, random_state=RANDOM_STATE).fit(name_vecs2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_vecs2_clusters = km2.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14305019969241764"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#how well do name vector clusters correspond with community\n",
    "adjusted_rand_score(name_vecs2_clusters, labels[fullnames_mask])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running more experiments with RNNs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deeper network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "\n",
    "fn_inp = Input(batch_shape=(1,None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "fn_x = GRU(64, return_sequences=True)(fn_x)\n",
    "fn_x = Dropout(0.2)(fn_x)\n",
    "fn_x = GRU(64)(fn_x)\n",
    "\n",
    "sn_inp = Input(batch_shape=(1,None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "sn_x = GRU(64, return_sequences=True)(sn_x)\n",
    "sn_x = Dropout(0.2)(sn_x)\n",
    "sn_x = GRU(64)(sn_x)\n",
    "\n",
    "x = Concatenate()([fn_x, sn_x])\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 75s 7ms/step - loss: 2.0249 - acc: 0.3466 - val_loss: 1.7811 - val_acc: 0.4454\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.78106, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 67s 6ms/step - loss: 1.5322 - acc: 0.5341 - val_loss: 1.3689 - val_acc: 0.5813\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.78106 to 1.36889, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 60s 5ms/step - loss: 1.2296 - acc: 0.6294 - val_loss: 1.2009 - val_acc: 0.6378\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.36889 to 1.20095, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 61s 6ms/step - loss: 1.0463 - acc: 0.6755 - val_loss: 1.1135 - val_acc: 0.6713\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.20095 to 1.11347, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.9062 - acc: 0.7128 - val_loss: 1.0676 - val_acc: 0.6933\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.11347 to 1.06757, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 60s 5ms/step - loss: 0.7963 - acc: 0.7456 - val_loss: 1.0472 - val_acc: 0.7091\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.06757 to 1.04720, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 59s 5ms/step - loss: 0.7113 - acc: 0.7737 - val_loss: 1.0339 - val_acc: 0.7153\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.04720 to 1.03393, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 61s 6ms/step - loss: 0.6572 - acc: 0.7894 - val_loss: 1.0282 - val_acc: 0.7186\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.03393 to 1.02820, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 9/20\n",
      "11032/11032 [==============================] - 66s 6ms/step - loss: 0.6043 - acc: 0.8045 - val_loss: 1.0093 - val_acc: 0.7281\n",
      "\n",
      "Epoch 00009: val_loss improved from 1.02820 to 1.00927, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 10/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.5576 - acc: 0.8214 - val_loss: 1.0086 - val_acc: 0.7289\n",
      "\n",
      "Epoch 00010: val_loss improved from 1.00927 to 1.00865, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 11/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.5410 - acc: 0.8192 - val_loss: 0.9889 - val_acc: 0.7349\n",
      "\n",
      "Epoch 00011: val_loss improved from 1.00865 to 0.98892, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 12/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.5057 - acc: 0.8342 - val_loss: 0.9948 - val_acc: 0.7431\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.98892\n",
      "Epoch 13/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.4990 - acc: 0.8359 - val_loss: 1.0205 - val_acc: 0.7333\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.98892\n",
      "Epoch 14/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.4853 - acc: 0.8387 - val_loss: 0.9795 - val_acc: 0.7558\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.98892 to 0.97947, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 15/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.4776 - acc: 0.8404 - val_loss: 0.9784 - val_acc: 0.7569\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.97947 to 0.97841, saving model to model_rnn_fullname222.hdf5\n",
      "Epoch 16/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.4429 - acc: 0.8541 - val_loss: 1.0040 - val_acc: 0.7499\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.97841\n",
      "Epoch 17/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.4529 - acc: 0.8496 - val_loss: 0.9924 - val_acc: 0.7518\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.97841\n",
      "Epoch 18/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.4526 - acc: 0.8479 - val_loss: 1.0556 - val_acc: 0.7523\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.97841\n",
      "Epoch 19/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.4279 - acc: 0.8546 - val_loss: 1.0399 - val_acc: 0.7520\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 0.97841\n",
      "Epoch 00019: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x28852232b00>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_fullname3.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator2(X_train,y_train), validation_data=rnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "\n",
    "fn_inp = Input(batch_shape=(1,None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "fn_x = GRU(64, return_sequences=True)(fn_x)\n",
    "fn_x = BatchNormalization()(fn_x)\n",
    "fn_x = Dropout(0.2)(fn_x)\n",
    "fn_x = GRU(64)(fn_x)\n",
    "\n",
    "sn_inp = Input(batch_shape=(1,None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "sn_x = GRU(64, return_sequences=True)(sn_x)\n",
    "sn_x = BatchNormalization()(sn_x)\n",
    "sn_x = Dropout(0.2)(sn_x)\n",
    "sn_x = GRU(64)(sn_x)\n",
    "\n",
    "x = Concatenate()([fn_x, sn_x])\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 83s 8ms/step - loss: 1.7107 - acc: 0.4748 - val_loss: 1.9981 - val_acc: 0.4078\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.99810, saving model to model_rnn_fullname3ii.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 72s 7ms/step - loss: 1.2214 - acc: 0.6290 - val_loss: 1.9271 - val_acc: 0.4347\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.99810 to 1.92705, saving model to model_rnn_fullname3ii.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 63s 6ms/step - loss: 1.0192 - acc: 0.6875 - val_loss: 1.9319 - val_acc: 0.4331\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 1.92705\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.8922 - acc: 0.7208 - val_loss: 1.8791 - val_acc: 0.4505\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.92705 to 1.87907, saving model to model_rnn_fullname3ii.hdf5\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.8090 - acc: 0.7438 - val_loss: 2.0118 - val_acc: 0.4318\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 1.87907\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.7469 - acc: 0.7607 - val_loss: 2.0417 - val_acc: 0.4263\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.87907\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 63s 6ms/step - loss: 0.7126 - acc: 0.7694 - val_loss: 2.0057 - val_acc: 0.4241\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 1.87907\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.6890 - acc: 0.7765 - val_loss: 2.0640 - val_acc: 0.4407\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 1.87907\n",
      "Epoch 00008: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x28868511780>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_fullname3ii.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator2(X_train,y_train), validation_data=rnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Discussions on performance decrease when using Dropout and Batchnormalization together;\n",
    "* https://stackoverflow.com/questions/39691902/ordering-of-batch-normalization-and-dropout\n",
    "* https://www.reddit.com/r/MachineLearning/comments/67gonq/d_batch_normalization_before_or_after_relu/\n",
    "\n",
    "Papers;\n",
    "* https://arxiv.org/pdf/1801.05134.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "\n",
    "fn_inp = Input(batch_shape=(1,None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "fn_x = GRU(64, return_sequences=True)(fn_x)\n",
    "fn_x = Dropout(0.2)(fn_x)\n",
    "fn_x = BatchNormalization()(fn_x)\n",
    "fn_x = GRU(64)(fn_x)\n",
    "\n",
    "sn_inp = Input(batch_shape=(1,None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "sn_x = GRU(64, return_sequences=True)(sn_x)\n",
    "sn_x = Dropout(0.2)(sn_x)\n",
    "sn_x = BatchNormalization()(sn_x)\n",
    "sn_x = GRU(64)(sn_x)\n",
    "\n",
    "x = Concatenate()([fn_x, sn_x])\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 83s 8ms/step - loss: 1.7655 - acc: 0.4546 - val_loss: 1.8470 - val_acc: 0.4383\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.84701, saving model to model_rnn_fullname3iii.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 80s 7ms/step - loss: 1.3082 - acc: 0.6022 - val_loss: 1.6570 - val_acc: 0.4943\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.84701 to 1.65701, saving model to model_rnn_fullname3iii.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 74s 7ms/step - loss: 1.1097 - acc: 0.6557 - val_loss: 1.5901 - val_acc: 0.5204\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.65701 to 1.59012, saving model to model_rnn_fullname3iii.hdf5\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 74s 7ms/step - loss: 0.9915 - acc: 0.6935 - val_loss: 1.5833 - val_acc: 0.5321\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.59012 to 1.58331, saving model to model_rnn_fullname3iii.hdf5\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 73s 7ms/step - loss: 0.8965 - acc: 0.7157 - val_loss: 1.5243 - val_acc: 0.5476\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.58331 to 1.52430, saving model to model_rnn_fullname3iii.hdf5\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 69s 6ms/step - loss: 0.8325 - acc: 0.7365 - val_loss: 1.6010 - val_acc: 0.5375\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.52430\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 69s 6ms/step - loss: 0.7764 - acc: 0.7494 - val_loss: 1.5096 - val_acc: 0.5593\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.52430 to 1.50957, saving model to model_rnn_fullname3iii.hdf5\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 68s 6ms/step - loss: 0.7301 - acc: 0.7626 - val_loss: 1.6388 - val_acc: 0.5367\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 1.50957\n",
      "Epoch 9/20\n",
      "11032/11032 [==============================] - 109s 10ms/step - loss: 0.7002 - acc: 0.7744 - val_loss: 1.6024 - val_acc: 0.5633\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 1.50957\n",
      "Epoch 10/20\n",
      "11032/11032 [==============================] - 79s 7ms/step - loss: 0.6950 - acc: 0.7795 - val_loss: 1.6417 - val_acc: 0.5459\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 1.50957\n",
      "Epoch 11/20\n",
      "11032/11032 [==============================] - 73s 7ms/step - loss: 0.6718 - acc: 0.7820 - val_loss: 1.6163 - val_acc: 0.5506\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 1.50957\n",
      "Epoch 00011: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2886b04bc88>"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_fullname3iii.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator2(X_train,y_train), validation_data=rnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Insignificant improvement from order reversal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "\n",
    "fn_inp = Input(batch_shape=(1,None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "fn_x = GRU(64, return_sequences=True)(fn_x)\n",
    "fn_x = BatchNormalization()(fn_x)\n",
    "fn_x = GRU(64)(fn_x)\n",
    "\n",
    "sn_inp = Input(batch_shape=(1,None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "sn_x = GRU(64, return_sequences=True)(sn_x)\n",
    "sn_x = BatchNormalization()(sn_x)\n",
    "sn_x = GRU(64)(sn_x)\n",
    "\n",
    "x = Concatenate()([fn_x, sn_x])\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 94s 9ms/step - loss: 1.6582 - acc: 0.4953 - val_loss: 1.9758 - val_acc: 0.3983\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.97582, saving model to model_rnn_fullname3iv.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 79s 7ms/step - loss: 1.1345 - acc: 0.6508 - val_loss: 1.8180 - val_acc: 0.4405\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.97582 to 1.81799, saving model to model_rnn_fullname3iv.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 69s 6ms/step - loss: 0.8917 - acc: 0.7201 - val_loss: 2.0618 - val_acc: 0.3896\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 1.81799\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 74s 7ms/step - loss: 0.7258 - acc: 0.7700 - val_loss: 2.1574 - val_acc: 0.3812\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 1.81799\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 76s 7ms/step - loss: 0.6204 - acc: 0.8031 - val_loss: 2.2313 - val_acc: 0.3915\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 1.81799\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 74s 7ms/step - loss: 0.5459 - acc: 0.8251 - val_loss: 2.3177 - val_acc: 0.3812\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.81799\n",
      "Epoch 00006: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x28870fefef0>"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_fullname3iv.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator2(X_train,y_train), validation_data=rnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Batchnormalisation does not provide large enough regularisation effect "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trying out Bidirectional layers;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "\n",
    "fn_inp = Input(batch_shape=(1,None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "fn_x = Bidirectional(GRU(64, return_sequences=True))(fn_x)\n",
    "fn_x = Dropout(0.2)(fn_x)\n",
    "fn_x = Bidirectional(GRU(64))(fn_x)\n",
    "\n",
    "sn_inp = Input(batch_shape=(1,None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "sn_x = Bidirectional(GRU(64, return_sequences=True))(sn_x)\n",
    "sn_x = Dropout(0.2)(sn_x)\n",
    "sn_x = Bidirectional(GRU(64))(sn_x)\n",
    "\n",
    "x = Concatenate()([fn_x, sn_x])\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 118s 11ms/step - loss: 1.8821 - acc: 0.4107 - val_loss: 1.5547 - val_acc: 0.5131\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.55472, saving model to checkpoints\\model_rnn_fullname4.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 105s 9ms/step - loss: 1.3477 - acc: 0.5944 - val_loss: 1.2364 - val_acc: 0.6278\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.55472 to 1.23643, saving model to checkpoints\\model_rnn_fullname4.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 105s 10ms/step - loss: 1.0593 - acc: 0.6722 - val_loss: 1.1256 - val_acc: 0.6659\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.23643 to 1.12561, saving model to checkpoints\\model_rnn_fullname4.hdf5\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 102s 9ms/step - loss: 0.8608 - acc: 0.7308 - val_loss: 1.0150 - val_acc: 0.7045\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.12561 to 1.01499, saving model to checkpoints\\model_rnn_fullname4.hdf5\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 99s 9ms/step - loss: 0.7151 - acc: 0.7716 - val_loss: 0.9898 - val_acc: 0.7159\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.01499 to 0.98978, saving model to checkpoints\\model_rnn_fullname4.hdf5\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 114s 10ms/step - loss: 0.6248 - acc: 0.7970 - val_loss: 0.9889 - val_acc: 0.7281\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.98978 to 0.98893, saving model to checkpoints\\model_rnn_fullname4.hdf5\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 112s 10ms/step - loss: 0.5423 - acc: 0.8253 - val_loss: 0.9906 - val_acc: 0.7393\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.98893\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 103s 9ms/step - loss: 0.5017 - acc: 0.8346 - val_loss: 0.9879 - val_acc: 0.7450\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.98893 to 0.98790, saving model to checkpoints\\model_rnn_fullname4.hdf5\n",
      "Epoch 9/20\n",
      "11032/11032 [==============================] - 101s 9ms/step - loss: 0.4853 - acc: 0.8426 - val_loss: 1.0408 - val_acc: 0.7442\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.98790\n",
      "Epoch 10/20\n",
      "11032/11032 [==============================] - 99s 9ms/step - loss: 0.4608 - acc: 0.8486 - val_loss: 1.0175 - val_acc: 0.7447\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.98790\n",
      "Epoch 11/20\n",
      "11032/11032 [==============================] - 101s 9ms/step - loss: 0.4562 - acc: 0.8513 - val_loss: 1.0238 - val_acc: 0.7569\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.98790\n",
      "Epoch 12/20\n",
      "11032/11032 [==============================] - 96s 9ms/step - loss: 0.4220 - acc: 0.8583 - val_loss: 1.0203 - val_acc: 0.7599\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.98790\n",
      "Epoch 00012: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x250b2d75c88>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_fullname4.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator2(X_train,y_train), validation_data=rnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Insignificant improvement"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Attempting to fit training data perfectly;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_vocab=26\n",
    "EMBEDDING_DIM=5\n",
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "def gru_block(x):\n",
    "    x = Bidirectional(GRU(64, return_sequences=True))(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    x = Bidirectional(GRU(64, return_sequences=True))(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    x = Bidirectional(GRU(64, return_sequences=False))(x)\n",
    "    return x        \n",
    "\n",
    "fn_inp = Input(batch_shape=(1,None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "fn_x = gru_block(fn_x)\n",
    "\n",
    "sn_inp = Input(batch_shape=(1,None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "sn_x = gru_block(sn_x)\n",
    "x = Concatenate()([fn_x, sn_x])\n",
    "x = Dense(128)(x)\n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 158s 14ms/step - loss: 2.0005 - acc: 0.3661 - val_loss: 1.7365 - val_acc: 0.4679\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.73645, saving model to checkpoints\\model_rnn_fullname5.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 148s 13ms/step - loss: 1.5409 - acc: 0.5366 - val_loss: 1.4962 - val_acc: 0.5579\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.73645 to 1.49618, saving model to checkpoints\\model_rnn_fullname5.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 141s 13ms/step - loss: 1.3116 - acc: 0.6054 - val_loss: 1.3212 - val_acc: 0.6066\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.49618 to 1.32123, saving model to checkpoints\\model_rnn_fullname5.hdf5\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 141s 13ms/step - loss: 1.1584 - acc: 0.6470 - val_loss: 1.3567 - val_acc: 0.6123\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 1.32123\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 140s 13ms/step - loss: 1.0386 - acc: 0.6784 - val_loss: 1.2914 - val_acc: 0.6476\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.32123 to 1.29140, saving model to checkpoints\\model_rnn_fullname5.hdf5\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 146s 13ms/step - loss: 0.9311 - acc: 0.7065 - val_loss: 1.1970 - val_acc: 0.6580\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.29140 to 1.19699, saving model to checkpoints\\model_rnn_fullname5.hdf5\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 143s 13ms/step - loss: 0.8462 - acc: 0.7354 - val_loss: 1.1384 - val_acc: 0.6746\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.19699 to 1.13835, saving model to checkpoints\\model_rnn_fullname5.hdf5\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 147s 13ms/step - loss: 0.7859 - acc: 0.7536 - val_loss: 1.1778 - val_acc: 0.6879\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 1.13835\n",
      "Epoch 9/20\n",
      "11032/11032 [==============================] - 163s 15ms/step - loss: 0.7445 - acc: 0.7651 - val_loss: 1.2480 - val_acc: 0.6822\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 1.13835\n",
      "Epoch 10/20\n",
      "11032/11032 [==============================] - 161s 15ms/step - loss: 0.7175 - acc: 0.7699 - val_loss: 1.1926 - val_acc: 0.6909\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 1.13835\n",
      "Epoch 11/20\n",
      "11032/11032 [==============================] - 163s 15ms/step - loss: 0.7019 - acc: 0.7750 - val_loss: 1.1415 - val_acc: 0.6979\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 1.13835\n",
      "Epoch 00011: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x259ac8b8c88>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_rnn_fullname5.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(rnn_generator2(X_train,y_train), validation_data=rnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding more layers gives worse performance on training data (?)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical\n",
    "from keras.layers import concatenate, Conv1D, GlobalMaxPooling1D, Lambda, Flatten\n",
    "from keras import backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_vocab = n_vocab + 1\n",
    "n_vocab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Over last names only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(df['Surname'][surnames_mask].values, labels[surnames_mask], random_state = RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cnn_name_processor(name, const_val=n_vocab-1, pad=2):\n",
    "    return np.pad(get_name_val(name), pad_width=pad, mode='constant', constant_values=const_val).reshape(1,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# padding of 2 so minimum word length is 5\n",
    "def cnn_generator(X, y, one_iter=False, x_processor=None):\n",
    "    if x_processor is None:\n",
    "        x_processor = cnn_name_processor\n",
    "        \n",
    "    while True:\n",
    "        for x_i, y_i in zip(X,y):\n",
    "            yield (x_processor(x_i), to_categorical(y_i, num_classes=n_classes).reshape(1,-1))\n",
    "    \n",
    "        if one_iter:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cnn_predict(names, model, argmax=True, x_processor=None):\n",
    "    if x_processor is None:\n",
    "        x_processor = cnn_name_processor\n",
    "        \n",
    "    predictions = []\n",
    "    for name in names:\n",
    "        result = model.predict( x_processor(name) )\n",
    "        if argmax:\n",
    "            result = np.argmax(result)\n",
    "        predictions += [result]\n",
    "    return np.array(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1D Convolution / convolution over time with filter sizes from 1 to 5 followed by max pooling over time (giving one element for each filter). Output is stacked into a single vector followed by a softmax."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_layers = []\n",
    "for i in range(1,6):\n",
    "    conv_layers.append( Conv1D(filters=10, kernel_size=(i,), strides=(1,)) )\n",
    "    \n",
    "inp = Input(shape=(None,))\n",
    "x = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)(inp)\n",
    "conv_outputs = []\n",
    "for conv_layer in conv_layers:\n",
    "    conv_outputs.append( GlobalMaxPooling1D() (conv_layer(x)) )\n",
    "x = Lambda(lambda x: backend.stack(x, axis=1))(conv_outputs)\n",
    "x = Flatten()(x) \n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=inp, outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)         (None, None, 5)      135         input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, None, 10)     60          embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_7 (Conv1D)               (None, None, 10)     110         embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_8 (Conv1D)               (None, None, 10)     160         embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_9 (Conv1D)               (None, None, 10)     210         embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_10 (Conv1D)              (None, None, 10)     260         embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_6 (GlobalM (None, 10)           0           conv1d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_7 (GlobalM (None, 10)           0           conv1d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_8 (GlobalM (None, 10)           0           conv1d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_9 (GlobalM (None, 10)           0           conv1d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_10 (Global (None, 10)           0           conv1d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 5, 10)        0           global_max_pooling1d_6[0][0]     \n",
      "                                                                 global_max_pooling1d_7[0][0]     \n",
      "                                                                 global_max_pooling1d_8[0][0]     \n",
      "                                                                 global_max_pooling1d_9[0][0]     \n",
      "                                                                 global_max_pooling1d_10[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 50)           0           lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 25)           1275        flatten_2[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 2,210\n",
      "Trainable params: 2,210\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11180/11180 [==============================] - 68s 6ms/step - loss: 2.0208 - acc: 0.3788 - val_loss: 1.7786 - val_acc: 0.4671\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.77857, saving model to model_cnn_surname.hdf5\n",
      "Epoch 2/20\n",
      "11180/11180 [==============================] - 67s 6ms/step - loss: 1.6976 - acc: 0.5002 - val_loss: 1.6202 - val_acc: 0.5272\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.77857 to 1.62019, saving model to model_cnn_surname.hdf5\n",
      "Epoch 3/20\n",
      "11180/11180 [==============================] - 68s 6ms/step - loss: 1.5609 - acc: 0.5405 - val_loss: 1.5437 - val_acc: 0.5500\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.62019 to 1.54365, saving model to model_cnn_surname.hdf5\n",
      "Epoch 4/20\n",
      "11180/11180 [==============================] - 68s 6ms/step - loss: 1.4834 - acc: 0.5647 - val_loss: 1.4943 - val_acc: 0.5707\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.54365 to 1.49427, saving model to model_cnn_surname.hdf5\n",
      "Epoch 5/20\n",
      "11180/11180 [==============================] - 68s 6ms/step - loss: 1.4263 - acc: 0.5833 - val_loss: 1.4628 - val_acc: 0.5830\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.49427 to 1.46277, saving model to model_cnn_surname.hdf5\n",
      "Epoch 6/20\n",
      "11180/11180 [==============================] - 69s 6ms/step - loss: 1.3884 - acc: 0.5933 - val_loss: 1.4412 - val_acc: 0.5892\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.46277 to 1.44119, saving model to model_cnn_surname.hdf5\n",
      "Epoch 7/20\n",
      "11180/11180 [==============================] - 71s 6ms/step - loss: 1.3601 - acc: 0.6016 - val_loss: 1.4223 - val_acc: 0.5962\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.44119 to 1.42228, saving model to model_cnn_surname.hdf5\n",
      "Epoch 8/20\n",
      "11180/11180 [==============================] - 72s 6ms/step - loss: 1.3382 - acc: 0.6115 - val_loss: 1.4106 - val_acc: 0.6010\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.42228 to 1.41058, saving model to model_cnn_surname.hdf5\n",
      "Epoch 9/20\n",
      "11180/11180 [==============================] - 73s 7ms/step - loss: 1.3221 - acc: 0.6175 - val_loss: 1.4001 - val_acc: 0.6042\n",
      "\n",
      "Epoch 00009: val_loss improved from 1.41058 to 1.40012, saving model to model_cnn_surname.hdf5\n",
      "Epoch 10/20\n",
      "11180/11180 [==============================] - 72s 6ms/step - loss: 1.3096 - acc: 0.6213 - val_loss: 1.3912 - val_acc: 0.6112\n",
      "\n",
      "Epoch 00010: val_loss improved from 1.40012 to 1.39122, saving model to model_cnn_surname.hdf5\n",
      "Epoch 11/20\n",
      "11180/11180 [==============================] - 72s 6ms/step - loss: 1.2991 - acc: 0.6221 - val_loss: 1.3821 - val_acc: 0.6166\n",
      "\n",
      "Epoch 00011: val_loss improved from 1.39122 to 1.38215, saving model to model_cnn_surname.hdf5\n",
      "Epoch 12/20\n",
      "11180/11180 [==============================] - 73s 7ms/step - loss: 1.2906 - acc: 0.6266 - val_loss: 1.3766 - val_acc: 0.6166\n",
      "\n",
      "Epoch 00012: val_loss improved from 1.38215 to 1.37658, saving model to model_cnn_surname.hdf5\n",
      "Epoch 13/20\n",
      "11180/11180 [==============================] - 58s 5ms/step - loss: 1.2823 - acc: 0.6284 - val_loss: 1.3716 - val_acc: 0.6177\n",
      "\n",
      "Epoch 00013: val_loss improved from 1.37658 to 1.37159, saving model to model_cnn_surname.hdf5\n",
      "Epoch 14/20\n",
      "11180/11180 [==============================] - 56s 5ms/step - loss: 1.2753 - acc: 0.6271 - val_loss: 1.3685 - val_acc: 0.6158\n",
      "\n",
      "Epoch 00014: val_loss improved from 1.37159 to 1.36852, saving model to model_cnn_surname.hdf5\n",
      "Epoch 15/20\n",
      "11180/11180 [==============================] - 59s 5ms/step - loss: 1.2690 - acc: 0.6301 - val_loss: 1.3660 - val_acc: 0.6166\n",
      "\n",
      "Epoch 00015: val_loss improved from 1.36852 to 1.36600, saving model to model_cnn_surname.hdf5\n",
      "Epoch 16/20\n",
      "11180/11180 [==============================] - 64s 6ms/step - loss: 1.2647 - acc: 0.6300 - val_loss: 1.3660 - val_acc: 0.6152\n",
      "\n",
      "Epoch 00016: val_loss improved from 1.36600 to 1.36598, saving model to model_cnn_surname.hdf5\n",
      "Epoch 17/20\n",
      "11180/11180 [==============================] - 56s 5ms/step - loss: 1.2596 - acc: 0.6313 - val_loss: 1.3655 - val_acc: 0.6150\n",
      "\n",
      "Epoch 00017: val_loss improved from 1.36598 to 1.36551, saving model to model_cnn_surname.hdf5\n",
      "Epoch 18/20\n",
      "11180/11180 [==============================] - 56s 5ms/step - loss: 1.2558 - acc: 0.6336 - val_loss: 1.3602 - val_acc: 0.6219\n",
      "\n",
      "Epoch 00018: val_loss improved from 1.36551 to 1.36015, saving model to model_cnn_surname.hdf5\n",
      "Epoch 19/20\n",
      "11180/11180 [==============================] - 70s 6ms/step - loss: 1.2514 - acc: 0.6353 - val_loss: 1.3605 - val_acc: 0.6225\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 1.36015\n",
      "Epoch 20/20\n",
      "11180/11180 [==============================] - 70s 6ms/step - loss: 1.2469 - acc: 0.6356 - val_loss: 1.3583 - val_acc: 0.6236\n",
      "\n",
      "Epoch 00020: val_loss improved from 1.36015 to 1.35833, saving model to model_cnn_surname.hdf5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x16ef3a079b0>"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_cnn_surname.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(cnn_generator(X_train,y_train), validation_data=cnn_generator(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(\"checkpoints\\\\model_cnn_surname.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict['accuracy'] = accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train..\n",
      "f1_all = [0.36363636 0.73938336 0.09448819 0.         0.68428373 0.\n",
      " 0.61535088 0.52924791 0.         0.         0.         0.59277108\n",
      " 0.60622463 0.4955157  0.         0.         0.74190871 0.\n",
      " 0.         0.79279794 0.         0.54890788 0.52925764 0.\n",
      " 0.59797297]\n",
      "f1_micro = 0.6480322003577818\n",
      "f1_macro = 0.31726987963465136\n",
      "f1_weighted = 0.6312551048743819\n",
      "accuracy = 0.6480322003577818\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "yhat_train = cnn_predict(X_train, model)\n",
    "print('Train..')\n",
    "for metric, metric_fn in metrics_dict.items():\n",
    "    print(metric,\"=\", metric_fn(y_train, yhat_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dev..\n",
      "f1_all = [0.18181818 0.70097604 0.05063291 0.         0.65989848 0.\n",
      " 0.58964143 0.37777778 0.         0.         0.         0.65454545\n",
      " 0.52884615 0.40860215 0.         0.         0.68894602 0.\n",
      " 0.         0.78792822 0.         0.45592705 0.55086849 0.\n",
      " 0.57971014]\n",
      "f1_micro = 0.6235578213039978\n",
      "f1_macro = 0.36080592520821814\n",
      "f1_weighted = 0.6063165505549567\n",
      "accuracy = 0.6235578213039978\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "yhat_dev = cnn_predict(X_dev, model)\n",
    "print('Dev..')\n",
    "for metric, metric_fn in metrics_dict.items():\n",
    "    print(metric,\"=\", metric_fn(y_dev, yhat_dev))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exploring name representations (last GRU state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import adjusted_rand_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_output = Model(inputs=model.input, outputs=model.layers[-2].output)\n",
    "name_vecs = cnn_predict(df['Surname'][surnames_mask].values, layer_output, argmax=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "name_vecs = np.squeeze(name_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "km = KMeans(n_clusters=n_classes, random_state=RANDOM_STATE).fit(name_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_vecs_clusters = km.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09718642072415555"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#how well do name vector clusters correspond with community\n",
    "adjusted_rand_score(name_vecs_clusters, labels[surnames_mask])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Significantly worse than RNN embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Over full names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(df[['Name','Surname']][fullnames_mask].values, labels[fullnames_mask], random_state = RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# padding of 2 so minimum word length is 5\n",
    "def cnn_generator2(X, y, one_iter=False, x_processor=None):\n",
    "    if x_processor is None:\n",
    "        x_processor = cnn_name_processor\n",
    "        \n",
    "    while True:\n",
    "        for x_i, y_i in zip(X,y):\n",
    "            yield ({'fn': x_processor(x_i[0]), 'sn':x_processor(x_i[1])}, to_categorical(y_i, num_classes=n_classes).reshape(1,-1))\n",
    "    \n",
    "        if one_iter:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(n_vocab, EMBEDDING_DIM, trainable=True)\n",
    "fn_inp = Input(shape=(None,), name='fn')\n",
    "fn_x = embedding_layer(fn_inp)\n",
    "\n",
    "sn_inp = Input(shape=(None,), name='sn')\n",
    "sn_x = embedding_layer(sn_inp)\n",
    "\n",
    "conv_layers = {'fn':[], 'sn':[]}\n",
    "for inp_x in ['fn','sn']:\n",
    "    for i in range(1,6):\n",
    "        conv_layers[inp_x].append( Conv1D(filters=10, kernel_size=(i,), strides=(1,)) )\n",
    "    \n",
    "conv_outputs = []\n",
    "for conv_layer in conv_layers['fn']:\n",
    "    conv_outputs.append( GlobalMaxPooling1D() (conv_layer(fn_x)) )\n",
    "for conv_layer in conv_layers['sn']:\n",
    "    conv_outputs.append( GlobalMaxPooling1D() (conv_layer(sn_x)) )\n",
    "    \n",
    "x = Lambda(lambda x: backend.stack(x, axis=1))(conv_outputs)\n",
    "x = Flatten()(x) \n",
    "out = Dense(n_classes, activation='softmax')(x)\n",
    "model = Model(inputs=[fn_inp, sn_inp], outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "fn (InputLayer)                 (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sn (InputLayer)                 (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (None, None, 5)      135         fn[0][0]                         \n",
      "                                                                 sn[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_21 (Conv1D)              (None, None, 10)     60          embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_22 (Conv1D)              (None, None, 10)     110         embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_23 (Conv1D)              (None, None, 10)     160         embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_24 (Conv1D)              (None, None, 10)     210         embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_25 (Conv1D)              (None, None, 10)     260         embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_26 (Conv1D)              (None, None, 10)     60          embedding_6[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_27 (Conv1D)              (None, None, 10)     110         embedding_6[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_28 (Conv1D)              (None, None, 10)     160         embedding_6[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_29 (Conv1D)              (None, None, 10)     210         embedding_6[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_30 (Conv1D)              (None, None, 10)     260         embedding_6[1][0]                \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_21 (Global (None, 10)           0           conv1d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_22 (Global (None, 10)           0           conv1d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_23 (Global (None, 10)           0           conv1d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_24 (Global (None, 10)           0           conv1d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_25 (Global (None, 10)           0           conv1d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_26 (Global (None, 10)           0           conv1d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_27 (Global (None, 10)           0           conv1d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_28 (Global (None, 10)           0           conv1d_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_29 (Global (None, 10)           0           conv1d_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_30 (Global (None, 10)           0           conv1d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, 10, 10)       0           global_max_pooling1d_21[0][0]    \n",
      "                                                                 global_max_pooling1d_22[0][0]    \n",
      "                                                                 global_max_pooling1d_23[0][0]    \n",
      "                                                                 global_max_pooling1d_24[0][0]    \n",
      "                                                                 global_max_pooling1d_25[0][0]    \n",
      "                                                                 global_max_pooling1d_26[0][0]    \n",
      "                                                                 global_max_pooling1d_27[0][0]    \n",
      "                                                                 global_max_pooling1d_28[0][0]    \n",
      "                                                                 global_max_pooling1d_29[0][0]    \n",
      "                                                                 global_max_pooling1d_30[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 100)          0           lambda_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 25)           2525        flatten_4[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 4,260\n",
      "Trainable params: 4,260\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 72s 7ms/step - loss: 1.9704 - acc: 0.3765 - val_loss: 1.7582 - val_acc: 0.4600\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.75816, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 46s 4ms/step - loss: 1.5862 - acc: 0.5141 - val_loss: 1.5492 - val_acc: 0.5294\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.75816 to 1.54921, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 75s 7ms/step - loss: 1.4156 - acc: 0.5693 - val_loss: 1.4298 - val_acc: 0.5693\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.54921 to 1.42981, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 76s 7ms/step - loss: 1.3075 - acc: 0.6032 - val_loss: 1.3560 - val_acc: 0.5949\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.42981 to 1.35603, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 75s 7ms/step - loss: 1.2344 - acc: 0.6210 - val_loss: 1.3132 - val_acc: 0.6049\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.35603 to 1.31321, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 75s 7ms/step - loss: 1.1813 - acc: 0.6347 - val_loss: 1.2841 - val_acc: 0.6117\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.31321 to 1.28409, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 75s 7ms/step - loss: 1.1438 - acc: 0.6440 - val_loss: 1.2593 - val_acc: 0.6215\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.28409 to 1.25925, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 74s 7ms/step - loss: 1.1139 - acc: 0.6516 - val_loss: 1.2455 - val_acc: 0.6251\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.25925 to 1.24549, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 9/20\n",
      "11032/11032 [==============================] - 74s 7ms/step - loss: 1.0897 - acc: 0.6606 - val_loss: 1.2321 - val_acc: 0.6283\n",
      "\n",
      "Epoch 00009: val_loss improved from 1.24549 to 1.23212, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 10/20\n",
      "11032/11032 [==============================] - 73s 7ms/step - loss: 1.0697 - acc: 0.6701 - val_loss: 1.2211 - val_acc: 0.6340\n",
      "\n",
      "Epoch 00010: val_loss improved from 1.23212 to 1.22114, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 11/20\n",
      "11032/11032 [==============================] - 58s 5ms/step - loss: 1.0535 - acc: 0.6756 - val_loss: 1.2124 - val_acc: 0.6400\n",
      "\n",
      "Epoch 00011: val_loss improved from 1.22114 to 1.21239, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 12/20\n",
      "11032/11032 [==============================] - 56s 5ms/step - loss: 1.0387 - acc: 0.6795 - val_loss: 1.2062 - val_acc: 0.6487\n",
      "\n",
      "Epoch 00012: val_loss improved from 1.21239 to 1.20625, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 13/20\n",
      "11032/11032 [==============================] - 56s 5ms/step - loss: 1.0259 - acc: 0.6828 - val_loss: 1.2024 - val_acc: 0.6514\n",
      "\n",
      "Epoch 00013: val_loss improved from 1.20625 to 1.20242, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 14/20\n",
      "11032/11032 [==============================] - 57s 5ms/step - loss: 1.0151 - acc: 0.6853 - val_loss: 1.1982 - val_acc: 0.6512\n",
      "\n",
      "Epoch 00014: val_loss improved from 1.20242 to 1.19819, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 15/20\n",
      "11032/11032 [==============================] - 57s 5ms/step - loss: 1.0049 - acc: 0.6894 - val_loss: 1.1904 - val_acc: 0.6531\n",
      "\n",
      "Epoch 00015: val_loss improved from 1.19819 to 1.19043, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 16/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.9967 - acc: 0.6887 - val_loss: 1.1904 - val_acc: 0.6536\n",
      "\n",
      "Epoch 00016: val_loss improved from 1.19043 to 1.19040, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 17/20\n",
      "11032/11032 [==============================] - 58s 5ms/step - loss: 0.9895 - acc: 0.6922 - val_loss: 1.1877 - val_acc: 0.6563\n",
      "\n",
      "Epoch 00017: val_loss improved from 1.19040 to 1.18767, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 18/20\n",
      "11032/11032 [==============================] - 50s 5ms/step - loss: 0.9820 - acc: 0.6950 - val_loss: 1.1848 - val_acc: 0.6577\n",
      "\n",
      "Epoch 00018: val_loss improved from 1.18767 to 1.18482, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 19/20\n",
      "11032/11032 [==============================] - 52s 5ms/step - loss: 0.9747 - acc: 0.6973 - val_loss: 1.1826 - val_acc: 0.6577\n",
      "\n",
      "Epoch 00019: val_loss improved from 1.18482 to 1.18260, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 20/20\n",
      "11032/11032 [==============================] - 50s 5ms/step - loss: 0.9681 - acc: 0.6992 - val_loss: 1.1824 - val_acc: 0.6580\n",
      "\n",
      "Epoch 00020: val_loss improved from 1.18260 to 1.18239, saving model to checkpoints\\model_cnn_fullname.hdf5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x259c0704160>"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_cnn_fullname.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(cnn_generator2(X_train,y_train), validation_data=cnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "11032/11032 [==============================] - 67s 6ms/step - loss: 0.9625 - acc: 0.7001 - val_loss: 1.1837 - val_acc: 0.6547\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.18367, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 2/20\n",
      "11032/11032 [==============================] - 63s 6ms/step - loss: 0.9573 - acc: 0.7019 - val_loss: 1.1832 - val_acc: 0.6574\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.18367 to 1.18323, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 3/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.9524 - acc: 0.7042 - val_loss: 1.1831 - val_acc: 0.6580\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.18323 to 1.18306, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 4/20\n",
      "11032/11032 [==============================] - 67s 6ms/step - loss: 0.9486 - acc: 0.7046 - val_loss: 1.1828 - val_acc: 0.6555\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.18306 to 1.18282, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 5/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.9452 - acc: 0.7052 - val_loss: 1.1804 - val_acc: 0.6580\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.18282 to 1.18045, saving model to checkpoints\\model_cnn_fullname.hdf5\n",
      "Epoch 6/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.9415 - acc: 0.7058 - val_loss: 1.1834 - val_acc: 0.6574\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.18045\n",
      "Epoch 7/20\n",
      "11032/11032 [==============================] - 66s 6ms/step - loss: 0.9380 - acc: 0.7082 - val_loss: 1.1923 - val_acc: 0.6566\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 1.18045\n",
      "Epoch 8/20\n",
      "11032/11032 [==============================] - 65s 6ms/step - loss: 0.9348 - acc: 0.7091 - val_loss: 1.1945 - val_acc: 0.6547\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 1.18045\n",
      "Epoch 9/20\n",
      "11032/11032 [==============================] - 64s 6ms/step - loss: 0.9310 - acc: 0.7102 - val_loss: 1.1964 - val_acc: 0.6582\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 1.18045\n",
      "Epoch 00009: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x259b3cc3550>"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_cnn_fullname.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(cnn_generator2(X_train,y_train), validation_data=cnn_generator2(X_dev,y_dev), steps_per_epoch=len(X_train), validation_steps=len(X_dev), epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(\"checkpoints\\\\model_cnn_fullname.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train..\n",
      "f1_all = [0.73170732 0.7862157  0.41975309 0.         0.76328872 0.6\n",
      " 0.70252324 0.63959391 1.         0.71428571 0.66666667 0.6389414\n",
      " 0.71032746 0.63166954 1.         0.61538462 0.76356192 1.\n",
      " 1.         0.81926242 0.5620915  0.61884368 0.68497711 1.\n",
      " 0.67080745]\n",
      "f1_micro = 0.7230783176214648\n",
      "f1_macro = 0.7095960581032001\n",
      "f1_weighted = 0.7222755482005473\n",
      "accuracy = 0.7230783176214648\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "yhat_train = cnn_predict(X_train, model, x_processor=lambda x:{'fn': cnn_name_processor(x[0]), 'sn':cnn_name_processor(x[1])})\n",
    "print('Train..')\n",
    "for metric, metric_fn in metrics_dict.items():\n",
    "    print(metric,\"=\", metric_fn(y_train, yhat_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dev..\n",
      "f1_all = [0.16666667 0.75347222 0.25352113 0.         0.68965517 0.\n",
      " 0.66278317 0.53631285 0.         0.46153846 0.66666667 0.59119497\n",
      " 0.65254237 0.55525606 0.         0.5        0.73142857 0.\n",
      " 0.         0.76363636 0.44067797 0.51590106 0.56225931 0.\n",
      " 0.65198238]\n",
      "f1_micro = 0.6579662860250136\n",
      "f1_macro = 0.48359501857008497\n",
      "f1_weighted = 0.6555693772422725\n",
      "accuracy = 0.6579662860250136\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\anush\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "yhat_dev = cnn_predict(X_dev, model, x_processor=lambda x:{'fn': cnn_name_processor(x[0]), 'sn':cnn_name_processor(x[1])})\n",
    "print('Dev..')\n",
    "for metric, metric_fn in metrics_dict.items():\n",
    "    print(metric,\"=\", metric_fn(y_dev, yhat_dev))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exploring name representations (last GRU state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import adjusted_rand_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer_output = Model(inputs=model.input, outputs=model.layers[-2].output)\n",
    "name_vecs = cnn_predict(df[['Name','Surname']][fullnames_mask].values, layer_output, argmax=False, x_processor=lambda x:{'fn': cnn_name_processor(x[0]), 'sn':cnn_name_processor(x[1])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "name_vecs = np.squeeze(name_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "km = KMeans(n_clusters=n_classes, random_state=RANDOM_STATE).fit(name_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_vecs_clusters = km.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06774717855884567"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#how well do name vector clusters correspond with community\n",
    "adjusted_rand_score(name_vecs_clusters, labels[fullnames_mask])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Supervised Learning Conclusion\n",
    "\n",
    "Out of all the tried models, Logistic Regression seems to be the best option, taking both dev scores and speed of training and predicting into consideration.\n",
    "\n",
    "The neural network models may have better performance with more data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unsupervised Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Autoencoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "from keras.layers import Lambda, RepeatVector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_vocab = 26"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_dev, y_train, y_dev = train_test_split(df[['Name','Surname']][fullnames_mask].values, labels[fullnames_mask], random_state = RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def autoencoder_generator(X, one_iter=False, x_processor=None, n_vocab=n_vocab):\n",
    "    if x_processor is None:\n",
    "        x_processor = lambda x: to_categorical(get_name_val(x).reshape(1,-1), num_classes=n_vocab)\n",
    "        \n",
    "    while True:\n",
    "        for x_i in X:\n",
    "            yield (x_processor(x_i), x_processor(x_i))\n",
    "    \n",
    "        if one_iter:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def autoencoder_generator2(X, one_iter=False, x_processor=None):\n",
    "    while True:\n",
    "        for i in range(X.shape[-1]):\n",
    "            for x_i, y_i  in autoencoder_generator(X[:,i], one_iter=True, x_processor=x_processor):\n",
    "                yield (x_i, y_i)\n",
    "                \n",
    "        if one_iter:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def repeat_vector(args):\n",
    "        layer_to_repeat = args[0]\n",
    "        sequence_layer = args[1]\n",
    "        return RepeatVector(K.shape(sequence_layer)[1])(layer_to_repeat)\n",
    "\n",
    "inp = Input(batch_shape=(1,None,n_vocab))\n",
    "x = Bidirectional(GRU(32, return_sequences=False))(inp)\n",
    "x = Dense(4)(x)\n",
    "x = Lambda(repeat_vector, output_shape=(None,4))([x, inp])\n",
    "x = GRU(64, return_sequences=True)(x)\n",
    "out = Dense(n_vocab, activation='softmax')(x)\n",
    "model = Model(inputs=inp, outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_26 (InputLayer)           (1, None, 26)        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_26 (Bidirectional (1, 64)              11328       input_26[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (1, 4)               260         bidirectional_26[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "lambda_22 (Lambda)              (1, None, 4)         0           dense_12[0][0]                   \n",
      "                                                                 input_26[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "gru_34 (GRU)                    (1, None, 64)        13248       lambda_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (1, None, 26)        1690        gru_34[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 26,526\n",
      "Trainable params: 26,526\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "22064/22064 [==============================] - 135s 6ms/step - loss: 1.6573 - acc: 0.4964 - val_loss: 1.5519 - val_acc: 0.5594\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.55193, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 2/20\n",
      "22064/22064 [==============================] - 128s 6ms/step - loss: 1.2322 - acc: 0.6362 - val_loss: 1.3296 - val_acc: 0.6196\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.55193 to 1.32963, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 3/20\n",
      "22064/22064 [==============================] - 102s 5ms/step - loss: 1.0844 - acc: 0.6829 - val_loss: 1.2156 - val_acc: 0.6513\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.32963 to 1.21561, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 4/20\n",
      "22064/22064 [==============================] - 113s 5ms/step - loss: 1.0056 - acc: 0.7082 - val_loss: 1.1611 - val_acc: 0.6749\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.21561 to 1.16112, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 5/20\n",
      "22064/22064 [==============================] - 133s 6ms/step - loss: 0.9521 - acc: 0.7265 - val_loss: 1.1296 - val_acc: 0.6854\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.16112 to 1.12962, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 6/20\n",
      "22064/22064 [==============================] - 123s 6ms/step - loss: 0.9169 - acc: 0.7366 - val_loss: 1.0989 - val_acc: 0.6928\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.12962 to 1.09889, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 7/20\n",
      "22064/22064 [==============================] - 118s 5ms/step - loss: 0.8902 - acc: 0.7461 - val_loss: 1.0870 - val_acc: 0.7012\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.09889 to 1.08696, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 8/20\n",
      "22064/22064 [==============================] - 110s 5ms/step - loss: 0.8692 - acc: 0.7513 - val_loss: 1.0708 - val_acc: 0.7072\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.08696 to 1.07077, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 9/20\n",
      "22064/22064 [==============================] - 110s 5ms/step - loss: 0.8521 - acc: 0.7563 - val_loss: 1.0455 - val_acc: 0.7140\n",
      "\n",
      "Epoch 00009: val_loss improved from 1.07077 to 1.04546, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 10/20\n",
      "22064/22064 [==============================] - 110s 5ms/step - loss: 0.8377 - acc: 0.7603 - val_loss: 1.0341 - val_acc: 0.7199\n",
      "\n",
      "Epoch 00010: val_loss improved from 1.04546 to 1.03410, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 11/20\n",
      "22064/22064 [==============================] - 115s 5ms/step - loss: 0.8296 - acc: 0.7642 - val_loss: 1.0273 - val_acc: 0.7191\n",
      "\n",
      "Epoch 00011: val_loss improved from 1.03410 to 1.02728, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 12/20\n",
      "22064/22064 [==============================] - 115s 5ms/step - loss: 0.8183 - acc: 0.7675 - val_loss: 0.9803 - val_acc: 0.7309\n",
      "\n",
      "Epoch 00012: val_loss improved from 1.02728 to 0.98029, saving model to checkpoints\\model_autoencoder3.hdf5\n",
      "Epoch 13/20\n",
      "22064/22064 [==============================] - 119s 5ms/step - loss: 0.8066 - acc: 0.7699 - val_loss: 1.0154 - val_acc: 0.7216\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.98029\n",
      "Epoch 14/20\n",
      "22064/22064 [==============================] - 107s 5ms/step - loss: 0.8016 - acc: 0.7719 - val_loss: 1.0075 - val_acc: 0.7221\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.98029\n",
      "Epoch 15/20\n",
      "22064/22064 [==============================] - 117s 5ms/step - loss: 0.7967 - acc: 0.7745 - val_loss: 1.0149 - val_acc: 0.7214\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.98029\n",
      "Epoch 16/20\n",
      "22064/22064 [==============================] - 111s 5ms/step - loss: 0.7882 - acc: 0.7768 - val_loss: 1.0972 - val_acc: 0.7013\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.98029\n",
      "Epoch 00016: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x222dbe20a90>"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_autoencoder1.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(autoencoder_generator2(X_train), validation_data=autoencoder_generator2(X_dev), steps_per_epoch=len(X_train)*2, validation_steps=len(X_dev)*2, epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "def repeat_vector(args):\n",
    "        layer_to_repeat = args[0]\n",
    "        sequence_layer = args[1]\n",
    "        return RepeatVector(K.shape(sequence_layer)[1])(layer_to_repeat)\n",
    "\n",
    "inp = Input(batch_shape=(1,None,n_vocab))\n",
    "x = Bidirectional(GRU(32, return_sequences=True))(inp)\n",
    "x = Bidirectional(GRU(32, return_sequences=False))(x)\n",
    "x = Dense(4)(x)\n",
    "x = Lambda(repeat_vector, output_shape=(None,4))([x, inp])\n",
    "x = GRU(64, return_sequences=True)(x)\n",
    "x = GRU(64, return_sequences=True)(x)\n",
    "out = Dense(n_vocab, activation='softmax')(x)\n",
    "model = Model(inputs=inp, outputs=out)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_28 (InputLayer)           (1, None, 26)        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_29 (Bidirectional (1, None, 64)        11328       input_28[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_30 (Bidirectional (1, 64)              18624       bidirectional_29[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (1, 4)               260         bidirectional_30[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "lambda_23 (Lambda)              (1, None, 4)         0           dense_14[0][0]                   \n",
      "                                                                 input_28[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "gru_39 (GRU)                    (1, None, 64)        13248       lambda_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "gru_40 (GRU)                    (1, None, 64)        24768       gru_39[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (1, None, 26)        1690        gru_40[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 69,918\n",
      "Trainable params: 69,918\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "22064/22064 [==============================] - 212s 10ms/step - loss: 1.5442 - acc: 0.5386 - val_loss: 1.4169 - val_acc: 0.6010\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.41691, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 2/20\n",
      "22064/22064 [==============================] - 190s 9ms/step - loss: 1.1047 - acc: 0.6822 - val_loss: 1.1969 - val_acc: 0.6627\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.41691 to 1.19685, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 3/20\n",
      "22064/22064 [==============================] - 189s 9ms/step - loss: 0.9646 - acc: 0.7239 - val_loss: 1.1028 - val_acc: 0.6900\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.19685 to 1.10280, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 4/20\n",
      "22064/22064 [==============================] - 188s 9ms/step - loss: 0.8942 - acc: 0.7445 - val_loss: 1.0213 - val_acc: 0.7139\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.10280 to 1.02128, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 5/20\n",
      "22064/22064 [==============================] - 189s 9ms/step - loss: 0.8451 - acc: 0.7596 - val_loss: 1.0167 - val_acc: 0.7188\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.02128 to 1.01673, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 6/20\n",
      "22064/22064 [==============================] - 189s 9ms/step - loss: 0.8104 - acc: 0.7708 - val_loss: 0.9773 - val_acc: 0.7298\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.01673 to 0.97730, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 7/20\n",
      "22064/22064 [==============================] - 190s 9ms/step - loss: 0.7838 - acc: 0.7785 - val_loss: 0.9478 - val_acc: 0.7349\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.97730 to 0.94777, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 8/20\n",
      "22064/22064 [==============================] - 189s 9ms/step - loss: 0.7678 - acc: 0.7831 - val_loss: 0.9207 - val_acc: 0.7464\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.94777 to 0.92068, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 9/20\n",
      "22064/22064 [==============================] - 189s 9ms/step - loss: 0.7475 - acc: 0.7891 - val_loss: 0.9767 - val_acc: 0.7279\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.92068\n",
      "Epoch 10/20\n",
      "22064/22064 [==============================] - 189s 9ms/step - loss: 0.7308 - acc: 0.7942 - val_loss: 0.9019 - val_acc: 0.7520\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.92068 to 0.90193, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 11/20\n",
      "22064/22064 [==============================] - 190s 9ms/step - loss: 0.7203 - acc: 0.7976 - val_loss: 0.9188 - val_acc: 0.7449\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.90193\n",
      "Epoch 12/20\n",
      "22064/22064 [==============================] - 187s 8ms/step - loss: 0.7178 - acc: 0.7989 - val_loss: 0.8942 - val_acc: 0.7557\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.90193 to 0.89417, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 13/20\n",
      "22064/22064 [==============================] - 199s 9ms/step - loss: 0.7063 - acc: 0.8014 - val_loss: 0.8563 - val_acc: 0.7646\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.89417 to 0.85625, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 14/20\n",
      "22064/22064 [==============================] - 198s 9ms/step - loss: 0.7022 - acc: 0.8037 - val_loss: 0.8813 - val_acc: 0.7585\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.85625\n",
      "Epoch 15/20\n",
      "22064/22064 [==============================] - 199s 9ms/step - loss: 0.6889 - acc: 0.8065 - val_loss: 0.8454 - val_acc: 0.7679\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.85625 to 0.84540, saving model to checkpoints\\model_autoencoder4.hdf5\n",
      "Epoch 16/20\n",
      "22064/22064 [==============================] - 187s 8ms/step - loss: 0.6916 - acc: 0.8065 - val_loss: 0.8571 - val_acc: 0.7665\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.84540\n",
      "Epoch 17/20\n",
      "22064/22064 [==============================] - 182s 8ms/step - loss: 0.6873 - acc: 0.8085 - val_loss: 0.8819 - val_acc: 0.7554\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.84540\n",
      "Epoch 18/20\n",
      "22064/22064 [==============================] - 232s 11ms/step - loss: 0.6845 - acc: 0.8089 - val_loss: 0.8535 - val_acc: 0.7668\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.84540\n",
      "Epoch 19/20\n",
      "22064/22064 [==============================] - 263s 12ms/step - loss: 0.6773 - acc: 0.8113 - val_loss: 0.8539 - val_acc: 0.7673\n",
      "Restoring model weights from the end of the best epoch\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 0.84540\n",
      "Epoch 00019: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x222f15a0d68>"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_point = ModelCheckpoint('checkpoints\\\\model_autoencoder2.hdf5', verbose=True, save_best_only=True)\n",
    "early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=True, restore_best_weights=True)\n",
    "model.fit_generator(autoencoder_generator2(X_train), validation_data=autoencoder_generator2(X_dev), steps_per_epoch=len(X_train)*2, validation_steps=len(X_dev)*2, epochs=n_epochs, verbose=1, callbacks=[early_stop,check_point])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploring embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_processor = lambda x: to_categorical(get_name_val(x).reshape(1,-1), num_classes=n_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Model(inputs=model.input, outputs=model.layers[3].output)\n",
    "autoencoded_fn_vecs = encoder.predict_generator(autoencoder_generator(df['Name'][fullnames_mask]), steps=sum(fullnames_mask))\n",
    "autoencoded_sn_vecs = encoder.predict_generator(autoencoder_generator(df['Surname'][fullnames_mask]), steps=sum(fullnames_mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoded_vecs = np.hstack([autoencoded_fn_vecs,autoencoded_sn_vecs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14710, 8)"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "autoencoded_vecs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "km = KMeans(n_clusters=n_classes, random_state=RANDOM_STATE).fit(autoencoded_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoded_vecs_clusters = km.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.019626143216108612"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#how well do name vector clusters correspond with community\n",
    "adjusted_rand_score(autoencoded_vecs_clusters, labels[fullnames_mask])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Autoencoder does not learn name representations that map to community very well"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Things to try"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Using clustering algorithms other than KMeans to analyse name vector representations\n",
    "* Using one shot neural network learning techniqus\n",
    "* Using a GAN-like architecture where one network generates names given a community and another classifies whether a name belongs to a particular community or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}