ch07/Explainable_AI.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[**Blueprints for Text Analysis Using Python**](https://github.com/blueprints-for-text-analytics-python/blueprints-text)  \n",
    "Jens Albrecht, Sidharth Ramachandran, Christian Winkler\n",
    "\n",
    "# Chapter 7: How to Explain a Text Classifier<div class='tocSkip'/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remark<div class='tocSkip'/>\n",
    "\n",
    "The code in this notebook differs slightly from the printed book. \n",
    "\n",
    "Several layout and formatting commands, like `figsize` to control figure size or subplot commands are removed in the book.\n",
    "\n",
    "All of this is done to simplify the code in the book and put the focus on the important parts instead of formatting."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup<div class='tocSkip'/>\n",
    "\n",
    "Set directory locations. If working on Google Colab: copy files and install required libraries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "ON_COLAB = 'google.colab' in sys.modules\n",
    "\n",
    "if ON_COLAB:\n",
    "    GIT_ROOT = 'https://github.com/blueprints-for-text-analytics-python/blueprints-text/raw/master'\n",
    "    os.system(f'wget {GIT_ROOT}/ch07/setup.py')\n",
    "\n",
    "%run -i setup.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Python Settings<div class=\"tocSkip\"/>\n",
    "\n",
    "Common imports, defaults for formatting in Matplotlib, Pandas etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run \"$BASE_DIR/settings.py\"\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%config InlineBackend.figure_format = 'png'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From classification chapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_colwidth', 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# Cleaning up the data to remove special characters - will re-use the blueprint from Chapter 5\n",
    "import html \n",
    "import re\n",
    "# tags like \n",
    "RE_TAG = re.compile(r'<[^<>]*>')\n",
    "# text or code in brackets like [0]\n",
    "RE_BRACKET = re.compile('\\[[^\\[\\]]*\\]')\n",
    "# text or code in brackets like (0)\n",
    "RE_BRACKET_1 = re.compile('\\([^)]*\\)')\n",
    "# specials that are not part of words; matches # but not #cool\n",
    "RE_SPECIAL = re.compile(r'(?:^|\\s)[&#<>{}\\[\\]+]+(?:\\s|$)')\n",
    "# standalone sequences of hyphens like --- or ==\n",
    "RE_HYPHEN_SEQ = re.compile(r'(?:^|\\s)[\\-=\\+]{2,}(?:\\s|$)')\n",
    "# sequences of white spaces\n",
    "RE_MULTI_SPACE = re.compile('\\s+')\n",
    "\n",
    "def clean(text):\n",
    "    text = html.unescape(text)\n",
    "    text = RE_TAG.sub(' ', text)\n",
    "    text = RE_BRACKET.sub(' ', text)\n",
    "    text = RE_BRACKET_1.sub(' ', text)\n",
    "    text = RE_SPECIAL.sub(' ', text)\n",
    "    text = RE_HYPHEN_SEQ.sub(' ', text)\n",
    "    text = RE_MULTI_SPACE.sub(' ', text)\n",
    "    return text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix, classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Final Blueprint for Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the dataframe\n",
    "\n",
    "df = pd.read_csv(BUGS_FILE)\n",
    "df = df.groupby('Component', as_index=False).apply(pd.DataFrame.sample, random_state=42, frac=.2)\n",
    "df = df[['Title','Description','Component']]\n",
    "df = df.dropna()\n",
    "df['text'] = df['Title'] + \" \" + df['Description']\n",
    "df = df.drop(columns=['Title','Description'])\n",
    "\n",
    "# Step 1 - Data Preparation\n",
    "\n",
    "df['text'] = df['text'].apply(clean)\n",
    "\n",
    "# Step 2 - Train-Test Split\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(df['text'], df['Component'], \n",
    "                                                    test_size=0.2, random_state=42,\n",
    "                                                    stratify=df['Component'])\n",
    "print ('Size of Training Data ', X_train.shape[0])\n",
    "print ('Size of Test Data ', X_test.shape[0])\n",
    "\n",
    "# Step 3 - Training the Machine Learning model\n",
    "\n",
    "tfidf = TfidfVectorizer(min_df = 10, ngram_range=(1,2), stop_words=\"english\")\n",
    "X_train_tf = tfidf.fit_transform(X_train)\n",
    "\n",
    "\n",
    "svc = SVC(kernel=\"linear\", C=1, probability=True, random_state=42)\n",
    "svc.fit(X_train_tf, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_tf = tfidf.transform(X_test)\n",
    "Y_pred = svc.predict(X_test_tf)\n",
    "result = pd.DataFrame({ 'text': X_test.values, 'actual': Y_test.values, 'predicted': Y_pred })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explainable AI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "result[result[\"actual\"] != result[\"predicted\"]].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = result.iloc[21][\"text\"]\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svc.predict_proba(X_test_tf[21])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names = [\"APT\", \"Core\", \"Debug\", \"Doc\", \"Text\", \"UI\"]\n",
    "prob = svc.predict_proba(X_test_tf)\n",
    "# new dataframe for explainable results\n",
    "er = result.copy().reset_index()\n",
    "for i, c in enumerate(class_names):\n",
    "    er[c] = prob[:, i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "er[[\"actual\", \"predicted\"] + class_names].sample(5, random_state=99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "er['max_probability'] = er[class_names].max(axis=1)\n",
    "correct = (er[er['actual'] == er['predicted']])\n",
    "wrong   = (er[er['actual'] != er['predicted']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "correct[\"max_probability\"].plot.hist(title=\"Correct\")\n",
    "plt.savefig(\"correct.svg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "wrong[\"max_probability\"].plot.hist(title=\"Wrong\")\n",
    "plt.savefig(\"wrong.svg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(wrong)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "high = er[er[\"max_probability\"] > 0.8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(high)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(high[\"actual\"], high[\"predicted\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(er[\"actual\"], er[\"predicted\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## then with most important components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svc.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coef_[1] yields a matrix, A[0] converts to array and takes first row\n",
    "coef = svc.coef_[8].A[0]\n",
    "vocabulary_positions = coef.argsort()\n",
    "vocabulary = tfidf.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words = 10\n",
    "top_positive_coef = vocabulary_positions[-top_words:].tolist()\n",
    "top_negative_coef = vocabulary_positions[:top_words].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "core_ui = pd.DataFrame([[vocabulary[c], coef[c]] for c in top_positive_coef + top_negative_coef], \n",
    "                          columns=[\"feature\", \"coefficient\"]).sort_values(\"coefficient\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "core_ui.set_index(\"feature\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "10*['r']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "core_ui.set_index(\"feature\").plot.barh(color=[['red']*10 + ['green']*10])\n",
    "plt.savefig(\"coefficients-core-ui.svg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = svc.coef_\n",
    "coef = (c[5] + c[6] + c[7] + c[8] - c[0]).A[0]\n",
    "vocabulary_positions = coef.argsort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words = 20\n",
    "top_positive_coef = vocabulary_positions[-top_words:].tolist()\n",
    "top_negative_coef = vocabulary_positions[:top_words].tolist()\n",
    "core = pd.DataFrame([[vocabulary[c], coef[c]] for c in top_positive_coef + top_negative_coef], \n",
    "                       columns=[\"feature\", \"coefficient\"]).sort_values(\"coefficient\")\n",
    "core.set_index(\"feature\").plot.barh(figsize=(6, 10), color=[['red']*top_words + ['green']*top_words])\n",
    "plt.savefig(\"coefficients-core.svg\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## then with LIME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_pipeline\n",
    "pipeline = make_pipeline(tfidf, svc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline.predict_proba([\"compiler not working\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lime.lime_text import LimeTextExplainer\n",
    "explainer = LimeTextExplainer(class_names=class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "er[er[\"predicted\"] != er[\"actual\"]].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id = 21\n",
    "print('Document id: %d' % id)\n",
    "print('Predicted class =', er.iloc[id][\"predicted\"])\n",
    "print('True class: %s' % er.iloc[id][\"actual\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp = explainer.explain_instance(result.iloc[id][\"text\"], pipeline.predict_proba, num_features=10, labels=[1, 5])\n",
    "print('Explanation for class %s' % class_names[1])\n",
    "print('\\n'.join(map(str, exp.as_list(label=1))))\n",
    "print()\n",
    "print('Explanation for class %s' % class_names[5])\n",
    "print('\\n'.join(map(str, exp.as_list(label=5))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp = explainer.explain_instance(result.iloc[id][\"text\"], pipeline.predict_proba, num_features=6, top_labels=3)\n",
    "print(exp.available_labels())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp.show_in_notebook(text=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lime import submodular_pick"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "lsm = submodular_pick.SubmodularPick(explainer, er[\"text\"].values, pipeline.predict_proba, \n",
    "                                        sample_size=100,\n",
    "                                        num_features=20,\n",
    "                                        num_exps_desired=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsm.explanations[0].show_in_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ELI5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "svm = SGDClassifier(loss='hinge', max_iter=1000, tol=1e-3, random_state=42)\n",
    "svm.fit(X_train_tf, Y_train)\n",
    "Y_pred_svm = svm.predict(X_test_tf)\n",
    "print(classification_report(Y_test, Y_pred_svm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import eli5\n",
    "eli5.show_weights(svm, top=10, vec=tfidf, target_names=class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eli5.show_prediction(svm, X_test.iloc[21],  vec=tfidf, target_names=class_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## then with anchor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from anchor import anchor_text\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hack for spacy 2.3\n",
    "for s in nlp.vocab.vectors:\n",
    "    _ = nlp.vocab[s]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "explainer_unk = anchor_text.AnchorText(nlp, class_names, use_unk_distribution=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = er.iloc[21][\"text\"]\n",
    "actual = er.iloc[21][\"actual\"]\n",
    "# we want the class with the highest probability and must invert the order\n",
    "predicted_class_ids = np.argsort(pipeline.predict_proba([text])[0])[::-1]\n",
    "pred = explainer_unk.class_names[predicted_class_ids[0]]\n",
    "alternative = explainer_unk.class_names[predicted_class_ids[1]]\n",
    "print(f'predicted {pred}, alternative {alternative}, actual {actual}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_unk = explainer_unk.explain_instance(text, pipeline.predict, threshold=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Rule: {\" AND \".join(exp_unk.names())}')\n",
    "print(f'Precision: {exp_unk.precision()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Made-up examples where anchor rule matches and model predicts {pred}\\n')\n",
    "print('\\n'.join([x[0] for x in exp_unk.examples(only_same_prediction=True)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Made-up examples where anchor rule matches and model predicts {alternative}\\n')\n",
    "print('\\n'.join([x[0] for x in exp_unk.examples(partial_index=0, only_different_prediction=True)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Without unknown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "explainer_no_unk = anchor_text.AnchorText(nlp, class_names, use_unk_distribution=False, use_bert=False)\n",
    "exp_no_unk = explainer_no_unk.explain_instance(text, pipeline.predict, threshold=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'Rule: {\" AND \".join(exp_no_unk.names())}')\n",
    "print(f'Precision: {exp_no_unk.precision()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Anchor: %s' % (' AND '.join(exp_no_unk.names())))\n",
    "print('Precision: %.2f' % exp_no_unk.precision())\n",
    "print()\n",
    "print('Examples where anchor applies and model predicts %s:' % pred)\n",
    "print()\n",
    "print('\\n'.join([x[0] for x in exp_no_unk.examples(only_same_prediction=True)]))\n",
    "print()\n",
    "print('Examples where anchor applies and model predicts %s:' % alternative)\n",
    "print()\n",
    "print('\\n'.join([x[0] for x in exp_no_unk.examples(partial_index=0, only_different_prediction=True)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## numerical for show in notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_numerical(text):\n",
    "    res = pipeline.predict(text)\n",
    "    n = np.array([str(class_names.index(r)) for r in res])\n",
    "    return n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "explainer_num = anchor_text.AnchorText(nlp, list(range(6)), use_unk_distribution=False, use_bert=False)\n",
    "exp_num = explainer_num.explain_instance(text, predict_numerical, threshold=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_class_ids = np.argsort(pipeline.predict_proba([text])[0])[::-1]\n",
    "pred = explainer_num.class_names[predicted_class_ids[0]]\n",
    "alternative = explainer_num.class_names[predicted_class_ids[1]]\n",
    "print(f'predicted {pred}, alternative {alternative}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "exp_num.show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}