detecting_loanwords/loanwords_feature_engineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loanword Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In this notebook you will:\n",
    "1. Analyze and optimize the featurization matrix from the previous notebook\n",
    "2. Prove that words aligned by suffixes in reverse order are stronger than natural order alignment\n",
    "3. Reduce the size of the feature matrix to increase the efficiency of our classifier without greatly sacrificing precision scores.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import glob\n",
    "import json\n",
    "import logging\n",
    "import multiprocessing\n",
    "import os\n",
    "import site\n",
    "from copy import deepcopy\n",
    "import random\n",
    "from collections import defaultdict\n",
    "import pickle\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from scipy import sparse\n",
    " \n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import BaggingClassifier, ExtraTreesClassifier, RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.utils.extmath import density\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.dummy import DummyClassifier\n",
    "from joblib import dump, load\n",
    "import sklearn\n",
    "from sklearn.preprocessing import FunctionTransformer, LabelEncoder\n",
    "from cltk.stem.latin.j_v import JVReplacer\n",
    "from cltk.prosody.latin.scansion_constants import ScansionConstants\n",
    "from cltk.prosody.latin.string_utils import remove_punctuation_dict\n",
    "from cltk.tokenize.word import WordTokenizer\n",
    "from cltk.corpus.readers import get_corpus_reader\n",
    "from cltk.utils.featurization import word_to_features\n",
    "from cltk.utils.file_operations import md5\n",
    "from cltk.utils.matrix_corpus_fun import (\n",
    "    distinct_words,\n",
    "    separate_camel_cases,\n",
    "    drop_empty_lists,\n",
    "    drop_non_lower,\n",
    "    drop_arabic_numeric,\n",
    "    drop_all_caps,\n",
    "    drop_empty_strings,\n",
    "    jv_transform,\n",
    "    splice_hyphens,\n",
    "    accept_editorial,\n",
    "    profile_chars,\n",
    "    demacronize,\n",
    "    drop_enclitics,\n",
    "    drop_fringe_punctuation,\n",
    "    divide_separate_words,\n",
    "    drop_all_punctuation)\n",
    "plt.style.use('fivethirtyeight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add parent directory to path so we can access our common code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import inspect\n",
    "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
    "parentdir = os.path.dirname(currentdir)\n",
    "sys.path.insert(0, parentdir) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlyoucanuse.romanizer import Romanizer, romanizer_transform  \n",
    "from mlyoucanuse.aeoe_replacer import aeoe_transform\n",
    "from mlyoucanuse.matrix_fun import (run_length_encoding,\n",
    "                                    extract_words,                                     \n",
    "                                    patch_cluster_holes)\n",
    "from mlyoucanuse.featurize_text_fun import word_to_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Turn on logging, primarily so that library methods may report warnings, if any"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOG = logging.getLogger('make_model')\n",
    "LOG.addHandler(logging.NullHandler())\n",
    "logging.basicConfig(level=logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a CorpusReader and select only the text files of Prudentius, Caesar and Eutropius. \n",
    "#### As shown in the appendix of this notebook, the authors of this seed set have a low incidence of using transliterated Greek words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:make_model:available good files 41\n"
     ]
    }
   ],
   "source": [
    "latin_reader = get_corpus_reader('latin_text_latin_library', language='latin')\n",
    "ALL_FILE_IDS = list(latin_reader.fileids())\n",
    "good_files = [file for file in ALL_FILE_IDS\n",
    "              if 'prudentius' in file or\n",
    "              'caesar' in file or\n",
    "              'eutropius' in file]\n",
    "LOG.info('available good files %s', len(good_files))\n",
    "latin_reader._fileids = good_files\n",
    "# good_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove some unfamiliar entries\n",
    "questionable = ['caesar/alex.txt',\n",
    "                'caesar/hisp.txt',\n",
    "                'prudentius/prud.psycho.txt',\n",
    "                'suetonius/suet.caesar.txt',\n",
    "                'xylander/caesar.txt']\n",
    "for file in questionable:\n",
    "    good_files.remove(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a custom Scikit-learn Pipeline, and call the CorpusReader `words()` method to process the texts\n",
    "#### The functions used in the pipelines are doctest documented in the `corpus_cleaning` module\n",
    "#### The functions used and their order was developed iteratively by running the pipelines on actual data and carefully inspecting the results prior to runnning it through featurization. Always know your data!\n",
    "\n",
    "#### Lastly, we use the joblib library to save/pickle the pipeline so that it can be reloaded and reused."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process_latin_text_pipeline = Pipeline([\n",
    "#     ('separate_camel_cases', FunctionTransformer(separate_camel_cases, validate=False)),\n",
    "#     ('splice_hyphens', FunctionTransformer(splice_hyphens, validate=False)),\n",
    "#     ('jv_transform', FunctionTransformer(jv_transform, validate=False)),\n",
    "#     ('aeoe_transform', FunctionTransformer(aeoe_transform, validate=False)),\n",
    "#     ('accept_editorial', FunctionTransformer(accept_editorial, validate=False)),\n",
    "#     ('drop_enclitics', FunctionTransformer(drop_enclitics, validate=False)),\n",
    "#     ('drop_fringe_punctuation', FunctionTransformer(drop_fringe_punctuation, validate=False)),\n",
    "#     ('drop_all_punctuation', FunctionTransformer(drop_all_punctuation, validate=False)),\n",
    "#     ('drop_non_lower', FunctionTransformer(drop_non_lower, validate=False)),\n",
    "#     ('drop_arabic_numeric', FunctionTransformer(drop_arabic_numeric, validate=False)),\n",
    "#     ('drop_all_caps', FunctionTransformer(drop_all_caps, validate=False)),\n",
    "#     ('divide_separate_words', FunctionTransformer(divide_separate_words, validate=False)),\n",
    "#     ('drop_empty_lists', FunctionTransformer(drop_empty_lists, validate=False)),\n",
    "#     ('drop_empty_strings', FunctionTransformer(drop_empty_strings, validate=False))])\n",
    "\n",
    "process_latin_text_pipeline_file = 'process_latin_text_pipeline.{}.joblib'.format(\n",
    "    sklearn.__version__)\n",
    "process_latin_text_pipeline = load(process_latin_text_pipeline_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  4.98it/s]\n"
     ]
    }
   ],
   "source": [
    "X = process_latin_text_pipeline.fit_transform(tqdm([list(latin_reader.words())]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Analyze the resulting matrix, by profiling the character occurences\n",
    "* Go back and adjust the pipeline as necessary\n",
    "* Turn the output into a distinct set of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_count = profile_chars(X)\n",
    "# print('Character distribution profile, total chars:', sum(char_count.values()))\n",
    "# print(char_count)\n",
    "distinct_good_latin = distinct_words(X)\n",
    "# print(f'Number of distinct words in Eutropius/Prudentius/Caesar sample: {len(distinct_good_latin):,}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### After running this notebook several times, we've decided to load in more training data, which is provide by the notebook:\n",
    "* `boosting_training_data.ipynb`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "additional_latin_words = []\n",
    "with open('latin.lemma.forms.txt', 'rt') as reader:\n",
    "    additional_latin_words = reader.read().split('\\n')\n",
    "# random.sample(additional_latin_words, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "additional_latin_words: 175,970\n",
      "distinct_good_latin now: 180,068\n"
     ]
    }
   ],
   "source": [
    "print(f'additional_latin_words: {len(additional_latin_words):,}')\n",
    "distinct_good_latin= list(set(distinct_good_latin) | set(additional_latin_words))\n",
    "print(f'distinct_good_latin now: {len(distinct_good_latin):,}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Load the Greek texts of Homer and Plato (two of the most commonly quoted Greek authors)\n",
    "* Preprocess the text\n",
    "* Transliterate into Classical Latin\n",
    "\n",
    "#### We save this pipeline for reuse too."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "perseus_greek = get_corpus_reader(language='greek', corpus_name='greek_text_perseus')\n",
    "plato = [tmp for tmp in perseus_greek.fileids() if 'plato' in tmp]\n",
    "homer = [tmp for tmp in perseus_greek.fileids() if 'homer' in tmp]\n",
    "greek_texts = plato + homer\n",
    "\n",
    "# process_greek_pipeline = Pipeline([\n",
    "#     ('accept_editorial', FunctionTransformer(accept_editorial, validate=False)),  # problematic\n",
    "#     ('romanizer', FunctionTransformer(romanizer_transform, validate=False)),\n",
    "#     ('drop_fringe_punctuation', FunctionTransformer(drop_fringe_punctuation, validate=False)),\n",
    "#     ('drop_all_punctuation', FunctionTransformer(drop_all_punctuation, validate=False)),\n",
    "#     ('drop_arabic_numeric', FunctionTransformer(drop_arabic_numeric, validate=False)),  #ok\n",
    "#     ('drop_empty_lists', FunctionTransformer(drop_empty_lists, validate=False)),  # problem?\n",
    "#     ('drop_empty_strings', FunctionTransformer(drop_empty_strings, validate=False))  # problem?\n",
    "# ])\n",
    "\n",
    "process_greek_text_pipeline_file = 'process_greek_text_pipeline.{}.joblib'.format(\n",
    "    sklearn.__version__)\n",
    "\n",
    "process_greek_pipeline = load(   process_greek_text_pipeline_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:35<00:00, 35.99s/it]\n"
     ]
    }
   ],
   "source": [
    "X_greek_transliterated = process_greek_pipeline.fit_transform(tqdm([list(perseus_greek.words(greek_texts))]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Analyze the transliterated Greek examples\n",
    "* Check character profiles for tuning\n",
    "* Create a set distinct words, with and without macrons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48,478 distinct_transliterated_greek_examples\n"
     ]
    }
   ],
   "source": [
    "# print('Character distribution profile of transliterated Greek: ', profile_chars(X_greek_transliterated))\n",
    "distinct_transliterated_greek_examples = distinct_words(X_greek_transliterated)\n",
    "print(f'{len(distinct_transliterated_greek_examples):,} distinct_transliterated_greek_examples')\n",
    "distinct_demacronized_greek = distinct_words(demacronize(X_greek_transliterated))\n",
    "# print(f'{len(distinct_demacronized_greek):,} distinct_demacronized_greek')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See how many words from the transliterated Greek words which have also appear in the Latin corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_words = distinct_demacronized_greek & set(distinct_good_latin)\n",
    "# print(f'Shared_words: {len(shared_words)} : {shared_words}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### These shared words appear in both language corpora; however, intuitively, we know each word will have a different probability of occurrence in each language. So, rather than arbitrarily excluding some or all of the words from one language or the other, we should split them into the most common probable groups. We can do this by loading the probability distribution pickle objects we have created in the notebooks:\n",
    "* `building_language_model/make_frequency_distribution.ipynb` \n",
    "* `detecting_loanwords/make_frequency_distribution_greek_transliterated.ipynb`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Frequency Distributions for Latin and transliterated Greek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "greek_transliterated_word_probs = {}\n",
    "with open('freq_dist.greek.transliterated.pkl', 'rb') as reader:\n",
    "    greek_transliterated_word_probs = pickle.load(reader)\n",
    "    \n",
    "latin_word_probs = {}\n",
    "with open(os.path.join('../building_language_model', 'freq_dist.latin.pkl'), 'rb') as reader:\n",
    "    latin_word_probs = pickle.load(reader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We'll create a list of tuples containing (the word, the words probability in Latin, the words probability in Greek)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_latin_greek = [(word, \n",
    "                       latin_word_probs.get(word, 0.000001),\n",
    "                       greek_transliterated_word_probs.get(word, 0.000001)) \n",
    "                      for word in shared_words]\n",
    "\n",
    "shared_latin_greek.sort(key=lambda a: a[1], reverse=True)\n",
    "# for item in shared_latin_greek:\n",
    "#     print(item)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`kai` is the most common word in the Greek corpus, so we could also divide the shared words by the threshold of this probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0011142407253193212"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "latin_word_probs.get('kai')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94\n"
     ]
    }
   ],
   "source": [
    "likely_latin = [word for word, latin_prob, greek_prob \n",
    "                in shared_latin_greek \n",
    "                if latin_prob >= latin_word_probs['kai']]\n",
    "print(len(likely_latin))\n",
    "# likely_latin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "greater_prob_latin = [word for word, latin_prob, greek_prob\n",
    "                     in shared_latin_greek\n",
    "                     if latin_prob >= greek_prob]\n",
    "# print(len(greater_prob_latin))\n",
    "# greater_prob_latin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46,929 distinct transliterated Greek words without matches in the Latin corpus\n"
     ]
    }
   ],
   "source": [
    "# We'll remove the words that have a high probability of being Latin \n",
    "# from the collection of demacronized transliterated Greek words\n",
    "only_greek_transliterated = distinct_demacronized_greek - set(likely_latin)\n",
    "likely_greek = shared_words - set(likely_latin)\n",
    "\n",
    "# Likewise, let's remove the transliterated words that are likely Greek \n",
    "# from the collection of good Latin words\n",
    "distinct_good_latin = set(distinct_good_latin) - likely_greek\n",
    "\n",
    "print(f'{len(only_greek_transliterated):,} distinct transliterated Greek words without matches in the Latin corpus')\n",
    "\n",
    "# NOTE: we are toggling this on to see the difference\n",
    "# only_greek_transliterated=  distinct_demacronized_greek\n",
    "# distinct_latin_wo_greek_matches = distinct_good_latin - shared_words\n",
    "# The following had low precision high recall:\n",
    "# only_greek_transliterated = distinct_demacronized_greek - shared_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normally these gaps might concern us, but since we are more interested in groups of loanwords in phrases, we can rely on smoothing over clusters of loanwords to screen out misses."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a simple data matrix of the single words, transliterated Greek examples followed by the Latin words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "226601"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = [list(only_greek_transliterated) + list(distinct_good_latin)]\n",
    "len(X[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Before we featurize our data matrix, let's check on the max word lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28 Max word length in distinct good Latin sample\n",
      "25 Max word length in transliterated Greek sample\n"
     ]
    }
   ],
   "source": [
    "print(f'{sorted([len(tmp) for tmp in distinct_good_latin])[-1]} Max word length in distinct good Latin sample')\n",
    "print(f'{sorted([len(tmp) for tmp in only_greek_transliterated])[-1] } Max word length in transliterated Greek sample')\n",
    "max_len = sorted([len(tmp) for tmp in only_greek_transliterated])[-1]  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Featurization\n",
    "#### We'll use a generic character to integer transform so that we can reuse our encoding process for unseen character data combinations (as opposed to building a dictionary mapping for a discrete sample space)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's create the X,y feature matrix and labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y shape: (226601,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:mlyoucanuse.featurize_text_fun:Excessive word length 28 for Thensaurochrysonicochrysides, truncating to 25\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X shape: (226601, 25)\n"
     ]
    }
   ],
   "source": [
    "all_y = np.array([1] * len(only_greek_transliterated) + [0] * len(distinct_good_latin), dtype=float)\n",
    "print(f'y shape: {all_y.shape}')\n",
    "# We use a label encoder to automatically capture the range of values for provenance\n",
    "# Although it's true we're only doing binary classification, \n",
    "# it's a good practice to use this so that we can automate recording our model's provenance\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(all_y)\n",
    "all_words = list(only_greek_transliterated) + list(distinct_good_latin)\n",
    "all_X = np.array([word_to_features(word, max_len) for word in all_words])\n",
    "print(f'X shape: {all_X.shape}')\n",
    "# all_X = sparse.csr_matrix(all_X)\n",
    "num_samples = all_y.shape[0] # to be used later by model provenance\n",
    "num_features = all_X.shape[1] # to be used later by model provenance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=0.99)\n",
    "features_pca = pca.fit_transform(all_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original features:25\n",
      "Reduced features: 14\n"
     ]
    }
   ],
   "source": [
    "print(f'Original features:{all_X.shape[1]}')\n",
    "print(f'Reduced features: {features_pca.shape[1]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from sklearn.feature_selection import RFECV\n",
    "# warnings.filterwarnings(action='ignore', module='scipy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
    "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=1, min_samples_split=2,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 25 features.\n",
      "Fitting estimator with 24 features.\n",
      "Fitting estimator with 23 features.\n",
      "Fitting estimator with 22 features.\n",
      "Fitting estimator with 21 features.\n",
      "Fitting estimator with 20 features.\n",
      "Fitting estimator with 19 features.\n",
      "Fitting estimator with 18 features.\n",
      "Fitting estimator with 17 features.\n",
      "Fitting estimator with 16 features.\n",
      "Fitting estimator with 15 features.\n",
      "Fitting estimator with 14 features.\n",
      "Fitting estimator with 13 features.\n",
      "Fitting estimator with 12 features.\n",
      "Fitting estimator with 11 features.\n",
      "Fitting estimator with 10 features.\n",
      "Fitting estimator with 9 features.\n",
      "Fitting estimator with 8 features.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[105, 111, 105, ..., 112,   0,   0],\n",
       "       [ 97, 108, 108, ..., 105,  97, 116],\n",
       "       [110,  97, 115, ..., 111, 100, 105],\n",
       "       ...,\n",
       "       [115, 105, 116, ..., 116, 105, 109],\n",
       "       [105, 115, 112, ..., 108,  99, 101],\n",
       "       [109, 117, 105, ..., 110, 101, 114]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv = RFECV(estimator=rfc,\n",
    "              step=1, \n",
    "              scoring='neg_mean_squared_error', cv=5, n_jobs = multiprocessing.cpu_count()-1,\n",
    "             verbose=1)\n",
    "rfecv.fit(all_X, all_y)\n",
    "rfecv.transform(all_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv.n_features_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True,  True,  True, False, False,\n",
       "       False, False, False, False, False, False, False, False, False,\n",
       "       False, False, False, False, False, False, False])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv.support_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  1,  1,  1,  1,  1,  1,  2,  3,  4,  6,  7,  5,  8,  9, 10, 11,\n",
       "       12, 13, 15, 14, 18, 19, 17, 16])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv.ranking_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:mlyoucanuse.featurize_text_fun:Excessive word length 28 for Thensaurochrysonicochrysides, truncating to 25\n"
     ]
    }
   ],
   "source": [
    "all_X_non_reversed = np.array([word_to_features(word, max_len, reverse=False) for word in all_words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
    "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=1, min_samples_split=2,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting estimator with 25 features.\n",
      "Fitting estimator with 24 features.\n",
      "Fitting estimator with 23 features.\n",
      "Fitting estimator with 22 features.\n",
      "Fitting estimator with 21 features.\n",
      "Fitting estimator with 20 features.\n",
      "Fitting estimator with 19 features.\n",
      "Fitting estimator with 18 features.\n",
      "Fitting estimator with 17 features.\n",
      "Fitting estimator with 16 features.\n",
      "Fitting estimator with 15 features.\n",
      "Fitting estimator with 14 features.\n",
      "Fitting estimator with 13 features.\n",
      "Fitting estimator with 12 features.\n",
      "Fitting estimator with 11 features.\n",
      "Fitting estimator with 10 features.\n",
      "Fitting estimator with 9 features.\n",
      "Fitting estimator with 8 features.\n",
      "Fitting estimator with 7 features.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[112, 111, 105, 111, 105,   0],\n",
       "       [104, 101, 108, 101, 116,  97],\n",
       "       [105, 100, 111, 117, 115,  97],\n",
       "       ...,\n",
       "       [112, 114, 111, 120, 105, 109],\n",
       "       [101,  99, 108, 105, 112, 115],\n",
       "       [ 97, 100, 104,  97, 101, 114]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv_non_reverse = RFECV(estimator=rfc,\n",
    "              step=1, \n",
    "              scoring='neg_mean_squared_error', cv=3, n_jobs = multiprocessing.cpu_count()-1,\n",
    "             verbose=1)\n",
    "rfecv_non_reverse.fit(all_X_non_reversed, all_y)\n",
    "rfecv_non_reverse.transform(all_X_non_reversed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  1,  1,  1,  1,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12,\n",
       "       13, 14, 15, 20, 17, 16, 19, 18])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv_non_reverse.ranking_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing the two ranking scores, we see that the word's natural sequence yields a straight gradient with best fit for the first six letters and then decaying linearly.\n",
    "## The reverse order, or suffix aligned feature matrix yields best fit for seven features, before going into a near linear decay."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'onmlkjihgf cba'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def prepare(word):\n",
    "    # craft a vector: aligning reverse suffix and prefixes\n",
    "    letters =list(word)\n",
    "    letters.reverse()\n",
    "    return ''.join(letters[:10] + [' '] + letters[-3:])\n",
    "\n",
    "prepare('abcdefghijklmno')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len('onmlkjihgf cba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_X_shortened = np.array([word_to_features(prepare(word), max_word_length=14, reverse=False) for word in all_words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
    "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=1, min_samples_split=2,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[105, 111, 105, ...,   0, 105, 111],\n",
       "       [ 97, 108, 108, ...,  97, 116, 101],\n",
       "       [110,  97, 115, ..., 100, 105,   0],\n",
       "       ...,\n",
       "       [115, 105, 116, ..., 105, 109, 105],\n",
       "       [105, 115, 112, ...,  99, 101,   0],\n",
       "       [109, 117, 105, ..., 101, 114, 101]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv_shortened = RFECV(estimator=rfc,\n",
    "              step=1, \n",
    "              scoring='neg_mean_squared_error', cv=3, n_jobs = multiprocessing.cpu_count()-1,\n",
    "             verbose=0)\n",
    "rfecv_shortened.fit(all_X_shortened, all_y)\n",
    "rfecv_shortened.transform(all_X_shortened)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 6, 4, 7, 5])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfecv_shortened.ranking_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'onmlkjihgf'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def shorten(word):\n",
    "    # craft a vector: aligning reverse suffix \n",
    "    letters =list(word)\n",
    "    letters.reverse()\n",
    "    return ''.join(letters[:10])\n",
    "\n",
    "shorten('abcdefghijklmno')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_X_short = np.array([word_to_features(shorten(word), max_word_length=10, reverse=False) for word in all_words])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Remove duplicates from the shortened feature matrix\n",
    "since we have removed features from the matrix, some rows will not be unique, let's make the data distinct."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(226601, 10)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_X_short.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(226601,)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniqs, indices = np.unique(all_X_short, return_index=True, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_X_shorter = all_X_short[ indices ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(216189, 10)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_X_shorter.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_y_shorter = all_y[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(216189,)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_y_shorter.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
      "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
      "            oob_score=False, random_state=None, verbose=0,\n",
      "            warm_start=False) 0.47923783949821946 [0.43370568 0.55012836 0.45331545 0.16860559 0.79043412]\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
    "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=1, min_samples_split=2,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)\n",
    "scores = cross_val_score(rfc, all_X_shorter, all_y_shorter,\n",
    "                             scoring='accuracy',\n",
    "                             n_jobs=multiprocessing.cpu_count()-1,\n",
    "                             cv=5)\n",
    "print(f'{str(rfc)} {scores.mean()} {scores}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## That is quite a drop in expected performance!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
      "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
      "            oob_score=False, random_state=None, verbose=0,\n",
      "            warm_start=False) 0.9621184307382344 [0.96211469 0.96341652 0.96167255 0.9614519  0.96193649]\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
    "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=1, min_samples_split=2,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)\n",
    "scores = cross_val_score(rfc, all_X_shortened, all_y,\n",
    "                             scoring='accuracy',\n",
    "                             n_jobs=multiprocessing.cpu_count()-1,\n",
    "                             cv=5)\n",
    "print(f'{str(rfc)} {scores.mean()} {scores}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# So, the moderately trimmed feature matrix retained high f scores, whereas the aggressively trimmed version was lost too much performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
       "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier =RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
    "            max_depth=None, max_features=0.4, max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=1, min_samples_split=2,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=800, n_jobs=None,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)\n",
    "classifier.fit(all_X_shorter, all_y_shorter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save the classifier, so it can be loaded without training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['is_transliterated_greek.lw.mdl.0.20.2.joblib']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_output_file = 'is_transliterated_greek.lw.mdl.{}.joblib'.format(sklearn.__version__)\n",
    "dump(classifier, model_output_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# That's all for now folks!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}