hcv_finding_optimal_features.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hcv_finding_optimal_features.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "5GuK5AOmQBD1"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/vigilant-umbrella/hcv-prediction/blob/main/hcv_finding_optimal_features.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w8Iv3Z7fD36F"
      },
      "source": [
        "# Initialization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QatIN-hXPkPZ"
      },
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import kendalltau\n",
        "\n",
        "from sklearn.preprocessing import OrdinalEncoder\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.model_selection import cross_validate\n",
        "from sklearn.feature_selection import f_classif\n",
        "\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.neural_network import MLPClassifier"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5GuK5AOmQBD1"
      },
      "source": [
        "# Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-5ujA_ACP0If",
        "outputId": "8d0532b7-445c-497d-d689-7f9568e0958e"
      },
      "source": [
        "%%shell\n",
        "if ! [ -f \"hcvdat0.csv\" ]; then\n",
        "    wget https://archive.ics.uci.edu/ml/machine-learning-databases/00571/hcvdat0.csv \n",
        "fi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-09-27 08:49:35--  https://archive.ics.uci.edu/ml/machine-learning-databases/00571/hcvdat0.csv\n",
            "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
            "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 46183 (45K) [application/x-httpd-php]\n",
            "Saving to: ‘hcvdat0.csv’\n",
            "\n",
            "hcvdat0.csv         100%[===================>]  45.10K  --.-KB/s    in 0.06s   \n",
            "\n",
            "2021-09-27 08:49:36 (819 KB/s) - ‘hcvdat0.csv’ saved [46183/46183]\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pakcz0nYP1a7"
      },
      "source": [
        "data = pd.read_csv('hcvdat0.csv')\n",
        "\n",
        "data = data[(data['Category']!='0=Blood Donor') & (data['Category']!='0s=suspect Blood Donor')].reset_index(drop=True)\n",
        "\n",
        "X = data.drop(['Unnamed: 0', 'Category'], axis=1)\n",
        "category = pd.DataFrame(data['Category'])\n",
        "\n",
        "X = pd.concat([X, pd.get_dummies(X.Sex, drop_first=True)], axis=1)\n",
        "X = X.drop(['Sex'], axis=1)\n",
        "\n",
        "X['ALB'] = X['ALB'].fillna(X['ALB'].median())\n",
        "X['ALP'] = X['ALP'].fillna(X['ALP'].median())\n",
        "X['ALT'] = X['ALT'].fillna(X['ALT'].median())\n",
        "X['CHOL'] = X['CHOL'].fillna(X['CHOL'].mean())\n",
        "X['PROT'] = X['PROT'].fillna(X['PROT'].mean())\n",
        "\n",
        "enc = OrdinalEncoder()\n",
        "y = pd.DataFrame(enc.fit_transform(pd.DataFrame(category)), columns=['category'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7DbErxdNvnRu"
      },
      "source": [
        "# Categorical Column 'Sex'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_dMzkNrswi6v"
      },
      "source": [
        "categorical_result = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lVJC6rBJxtqE"
      },
      "source": [
        "## Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qjOEQWm5wzfS"
      },
      "source": [
        "categorical_result['Logistic Regresssion'] = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R__usTp3xmYJ"
      },
      "source": [
        "### Keeping 'Sex'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-VT0Ys4yvqpY"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X)\n",
        "\n",
        "lr = LogisticRegression(\n",
        "    C=1.25,\n",
        "    fit_intercept=True,\n",
        "    l1_ratio=0.5,\n",
        "    max_iter=50,\n",
        "    penalty='elasticnet',\n",
        "    random_state=221,\n",
        "    solver='saga'\n",
        "    )\n",
        "\n",
        "lrcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\n",
        "\n",
        "lrcv_results = cross_validate(\n",
        "    lr,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=lrcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Logistic Regresssion'][\"Keeping 'Sex'\"] = sum(lrcv_results['test_score'])/len(lrcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R4Rze0EBwCkC"
      },
      "source": [
        "### After removing 'Sex'\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7-vicl07wFrd"
      },
      "source": [
        "X_dropped = X.drop('m', axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "lr = LogisticRegression(\n",
        "    C=1.25,\n",
        "    fit_intercept=True,\n",
        "    l1_ratio=0.5,\n",
        "    max_iter=50,\n",
        "    penalty='elasticnet',\n",
        "    random_state=221,\n",
        "    solver='saga'\n",
        "    )\n",
        "\n",
        "lrcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\n",
        "\n",
        "lrcv_results = cross_validate(\n",
        "    lr,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=lrcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Logistic Regresssion'][\"Removing 'Sex'\"] = sum(lrcv_results['test_score'])/len(lrcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EEALUrsHP2ke"
      },
      "source": [
        "## k-nearest neighbors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qSoCwYadP2ke"
      },
      "source": [
        "categorical_result['k-nearest neighbors'] = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WblgpcEvP2kf"
      },
      "source": [
        "### Keeping 'Sex'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cAsWD2KCP2kf"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X)\n",
        "\n",
        "knclf = KNeighborsClassifier(\n",
        "    n_neighbors=10,\n",
        "    algorithm='ball_tree',\n",
        "    p=4\n",
        "    )\n",
        "\n",
        "kncv = StratifiedKFold(n_splits=10, shuffle=True, random_state=182)\n",
        "\n",
        "kncv_results = cross_validate(\n",
        "    knclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=kncv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['k-nearest neighbors'][\"Keeping 'Sex'\"] = sum(kncv_results['test_score'])/len(kncv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "REDgDDlbP2kf"
      },
      "source": [
        "### After removing 'Sex'\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AL5SQNt7P2kg"
      },
      "source": [
        "X_dropped = X.drop('m', axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "knclf = KNeighborsClassifier(\n",
        "    n_neighbors=10,\n",
        "    algorithm='ball_tree',\n",
        "    p=4\n",
        "    )\n",
        "\n",
        "kncv = StratifiedKFold(n_splits=10, shuffle=True, random_state=182)\n",
        "\n",
        "kncv_results = cross_validate(\n",
        "    knclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=kncv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['k-nearest neighbors'][\"Removing 'Sex'\"] = sum(kncv_results['test_score'])/len(kncv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nMe1IIuMyZKY"
      },
      "source": [
        "## Gaussian naive Bayes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sx8kFL5oyZKY"
      },
      "source": [
        "categorical_result['Gaussian naive Bayes'] = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u2ew2sCcyZKY"
      },
      "source": [
        "### Keeping 'Sex'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mx5xGPGYyZKY"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X)\n",
        "\n",
        "gnb = GaussianNB(var_smoothing=1e-10)\n",
        "\n",
        "gnbcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=302)\n",
        "\n",
        "gnbcv_results = cross_validate(\n",
        "    gnb,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=gnbcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Gaussian naive Bayes'][\"Keeping 'Sex'\"] = sum(gnbcv_results['test_score'])/len(gnbcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XKr2eTUYyZKY"
      },
      "source": [
        "### After removing 'Sex'\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9luGGW24yZKZ"
      },
      "source": [
        "X_dropped = X.drop('m', axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "gnb = GaussianNB(var_smoothing=1e-10)\n",
        "\n",
        "gnbcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=302)\n",
        "\n",
        "gnbcv_results = cross_validate(\n",
        "    gnb,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=gnbcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Gaussian naive Bayes'][\"Removing 'Sex'\"] = sum(gnbcv_results['test_score'])/len(gnbcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pJTnD5Oqy5dk"
      },
      "source": [
        "## Decision Tree"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nU7JrOimy5dm"
      },
      "source": [
        "categorical_result['Decision Tree'] = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zyj8r5Vzy5dl"
      },
      "source": [
        "### Keeping 'Sex'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t5e58U6Qy5dm"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X)\n",
        "\n",
        "dtclf = DecisionTreeClassifier(\n",
        "    criterion='entropy',\n",
        "    max_depth=7,\n",
        "    max_features=None,\n",
        "    min_samples_leaf=3,\n",
        "    min_samples_split=0.4,\n",
        "    random_state=559,\n",
        "    splitter='random'\n",
        "    )\n",
        "\n",
        "dtcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=985)\n",
        "\n",
        "dtcv_results = cross_validate(\n",
        "    dtclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=dtcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Decision Tree'][\"Keeping 'Sex'\"] = sum(dtcv_results['test_score'])/len(dtcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S_Vq37YRy5dm"
      },
      "source": [
        "### After removing 'Sex'\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UCpJrVnoy5dm"
      },
      "source": [
        "X_dropped = X.drop('m', axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "dtclf = DecisionTreeClassifier(\n",
        "    criterion='entropy',\n",
        "    max_depth=7,\n",
        "    max_features=None,\n",
        "    min_samples_leaf=3,\n",
        "    min_samples_split=0.4,\n",
        "    random_state=559,\n",
        "    splitter='random'\n",
        "    )\n",
        "\n",
        "dtcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=985)\n",
        "\n",
        "dtcv_results = cross_validate(\n",
        "    dtclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=dtcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Decision Tree'][\"Removing 'Sex'\"] = sum(dtcv_results['test_score'])/len(dtcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ICbWjkMzUZ6"
      },
      "source": [
        "## Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Grw72c0zUZ7"
      },
      "source": [
        "categorical_result['Random Forest'] = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CoG-Ghp9zUZ6"
      },
      "source": [
        "### Keeping 'Sex'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0FS29bf2zUZ7"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X)\n",
        "\n",
        "rfclf = RandomForestClassifier(\n",
        "    criterion='gini',\n",
        "    max_features='sqrt',\n",
        "    min_samples_leaf=2,\n",
        "    min_samples_split=5,\n",
        "    n_estimators=50,\n",
        "    random_state=67\n",
        "    )\n",
        "\n",
        "rfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=309)\n",
        "\n",
        "rfcv_results = cross_validate(\n",
        "    rfclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=rfcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Random Forest'][\"Keeping 'Sex'\"] = sum(rfcv_results['test_score'])/len(rfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2etI_4iVzUZ7"
      },
      "source": [
        "### After removing 'Sex'\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IOlnHH-czUZ7"
      },
      "source": [
        "X_dropped = X.drop('m', axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "rfclf = RandomForestClassifier(\n",
        "    criterion='gini',\n",
        "    max_features='sqrt',\n",
        "    min_samples_leaf=2,\n",
        "    min_samples_split=5,\n",
        "    n_estimators=50,\n",
        "    random_state=67\n",
        "    )\n",
        "\n",
        "rfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=309)\n",
        "\n",
        "rfcv_results = cross_validate(\n",
        "    rfclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=rfcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Random Forest'][\"Removing 'Sex'\"] = sum(rfcv_results['test_score'])/len(rfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4pwVWWBnzwOs"
      },
      "source": [
        "## Support Vector Machine"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JAxZynoVzwOt"
      },
      "source": [
        "categorical_result['Support Vector Machine'] = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fByMf2opzwOt"
      },
      "source": [
        "### Keeping 'Sex'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CXIIXEmUzwOt"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X)\n",
        "\n",
        "svc = SVC(\n",
        "    C=0.1,\n",
        "    coef0=0.3,\n",
        "    degree=2,\n",
        "    gamma='scale',\n",
        "    kernel='poly',\n",
        "    random_state=98,\n",
        "    shrinking=True\n",
        "    )\n",
        "\n",
        "svccv = StratifiedKFold(n_splits=10, shuffle=True, random_state=911)\n",
        "\n",
        "svccv_results = cross_validate(\n",
        "    svc,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=svccv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Support Vector Machine'][\"Keeping 'Sex'\"] = sum(svccv_results['test_score'])/len(svccv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2AFcImQLzwOu"
      },
      "source": [
        "### After removing 'Sex'\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NmJ49FJszwOu"
      },
      "source": [
        "X_dropped = X.drop('m', axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "svc = SVC(\n",
        "    C=0.1,\n",
        "    coef0=0.3,\n",
        "    degree=2,\n",
        "    gamma='scale',\n",
        "    kernel='poly',\n",
        "    random_state=98,\n",
        "    shrinking=True\n",
        "    )\n",
        "\n",
        "svccv = StratifiedKFold(n_splits=10, shuffle=True, random_state=911)\n",
        "\n",
        "svccv_results = cross_validate(\n",
        "    svc,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=svccv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Support Vector Machine'][\"Removing 'Sex'\"] = sum(rfcv_results['test_score'])/len(rfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_panJdvO0L8p"
      },
      "source": [
        "## Mutli-layer Perceptron"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tvXrqr5b0L8q"
      },
      "source": [
        "categorical_result['Mutli-layer Perceptron'] = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V0T5_mj10L8p"
      },
      "source": [
        "### Keeping 'Sex'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CDgFnOb80L8q"
      },
      "source": [
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X)\n",
        "\n",
        "mlpclf = MLPClassifier(\n",
        "    batch_size=8,\n",
        "    activation='relu',\n",
        "    early_stopping=False,\n",
        "    hidden_layer_sizes=(32, 32, 32),\n",
        "    max_iter=500,\n",
        "    solver='lbfgs',\n",
        "    random_state=377\n",
        "    )\n",
        "\n",
        "mlpclfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=496)\n",
        "\n",
        "mlpclfcv_results = cross_validate(\n",
        "    mlpclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=mlpclfcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Mutli-layer Perceptron'][\"Keeping 'Sex'\"] = sum(mlpclfcv_results['test_score'])/len(mlpclfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KImEaVjx0L8q"
      },
      "source": [
        "### After removing 'Sex'\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7wdtYpqY0L8q"
      },
      "source": [
        "X_dropped = X.drop('m', axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "mlpclf = MLPClassifier(\n",
        "    batch_size=8,\n",
        "    activation='relu',\n",
        "    early_stopping=False,\n",
        "    hidden_layer_sizes=(32, 32, 32),\n",
        "    max_iter=500,\n",
        "    solver='lbfgs',\n",
        "    random_state=377\n",
        "    )\n",
        "\n",
        "mlpclfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=496)\n",
        "\n",
        "mlpclfcv_results = cross_validate(\n",
        "    mlpclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=mlpclfcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "categorical_result['Mutli-layer Perceptron'][\"Removing 'Sex'\"] = sum(mlpclfcv_results['test_score'])/len(mlpclfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q1ljm3f66XRE"
      },
      "source": [
        "## Result"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "NCUHa9Xe0L8q",
        "outputId": "cf62d146-873f-4c75-df4e-314c0756e184"
      },
      "source": [
        "pd.DataFrame(categorical_result.values(), index=categorical_result.keys())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Keeping 'Sex'</th>\n",
              "      <th>Removing 'Sex'</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Logistic Regresssion</th>\n",
              "      <td>0.692857</td>\n",
              "      <td>0.760714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>k-nearest neighbors</th>\n",
              "      <td>0.669643</td>\n",
              "      <td>0.764286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Gaussian naive Bayes</th>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.696429</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Decision Tree</th>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.625000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Random Forest</th>\n",
              "      <td>0.678571</td>\n",
              "      <td>0.760714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Support Vector Machine</th>\n",
              "      <td>0.641071</td>\n",
              "      <td>0.760714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Mutli-layer Perceptron</th>\n",
              "      <td>0.612500</td>\n",
              "      <td>0.582143</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                        Keeping 'Sex'  Removing 'Sex'\n",
              "Logistic Regresssion         0.692857        0.760714\n",
              "k-nearest neighbors          0.669643        0.764286\n",
              "Gaussian naive Bayes         0.708929        0.696429\n",
              "Decision Tree                0.571429        0.625000\n",
              "Random Forest                0.678571        0.760714\n",
              "Support Vector Machine       0.641071        0.760714\n",
              "Mutli-layer Perceptron       0.612500        0.582143"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DZfCs_lN6dDu"
      },
      "source": [
        "From the results we can conclude that 'Sex' should be removed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MXCjYydzS3KK",
        "outputId": "84911904-6837-4964-8dca-543e7526c1f6"
      },
      "source": [
        "to_remove_features = ['m']\n",
        "to_remove_features"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['m']"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p1ackp_xDy3y"
      },
      "source": [
        "# ANOVA F-value"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SBIn2PjlmvfE",
        "outputId": "ca04b7a4-58e0-4f3d-fc10-c8160b400c8e"
      },
      "source": [
        "anova_f_values = f_classif(X.drop(['m'], axis=1), y['category'])[0]\n",
        "\n",
        "linear_corr = pd.Series(anova_f_values, index=X.drop(['m'], axis=1).columns)\n",
        "linear_corr"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Age     15.299046\n",
              "ALB     43.959767\n",
              "ALP      6.677791\n",
              "ALT      4.877729\n",
              "AST      1.842203\n",
              "BIL      8.920381\n",
              "CHE     48.276278\n",
              "CHOL     6.290820\n",
              "CREA     2.095414\n",
              "GGT      1.386249\n",
              "PROT     5.837231\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8FmOndvz1Kuh"
      },
      "source": [
        "anova_result = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RY6hs3Dq1Kuh"
      },
      "source": [
        "## Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xmbx3PQp1Kuh"
      },
      "source": [
        "anova_result['Logistic Regresssion'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HDzZ7Zgm2hoY"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "lr = LogisticRegression(\n",
        "    C=1.25,\n",
        "    fit_intercept=True,\n",
        "    l1_ratio=0.5,\n",
        "    max_iter=50,\n",
        "    penalty='elasticnet',\n",
        "    random_state=221,\n",
        "    solver='saga'\n",
        "    )\n",
        "\n",
        "lrcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\n",
        "\n",
        "lrcv_results = cross_validate(\n",
        "    lr,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=lrcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "anova_result['Logistic Regresssion'][0] = sum(lrcv_results['test_score'])/len(lrcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XRxNne6w1gMo"
      },
      "source": [
        "for col, anova_val in linear_corr.sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    lr = LogisticRegression(\n",
        "        C=1.25,\n",
        "        fit_intercept=True,\n",
        "        l1_ratio=0.5,\n",
        "        max_iter=50,\n",
        "        penalty='elasticnet',\n",
        "        random_state=221,\n",
        "        solver='saga'\n",
        "        )\n",
        "\n",
        "    lrcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\n",
        "\n",
        "    lrcv_results = cross_validate(\n",
        "        lr,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=lrcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    anova_result['Logistic Regresssion'][anova_val] = sum(lrcv_results['test_score'])/len(lrcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5HkvixwnQYQk"
      },
      "source": [
        "## k-nearest neighbors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J2LzaOM6QYQl"
      },
      "source": [
        "anova_result['k-nearest neighbors'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZCZEsc0SQYQm"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "knclf = KNeighborsClassifier(\n",
        "    n_neighbors=10,\n",
        "    algorithm='ball_tree',\n",
        "    p=4\n",
        "    )\n",
        "\n",
        "kncv = StratifiedKFold(n_splits=10, shuffle=True, random_state=182)\n",
        "\n",
        "kncv_results = cross_validate(\n",
        "    knclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=kncv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "anova_result['k-nearest neighbors'][0] = sum(kncv_results['test_score'])/len(kncv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d7CoHB6_QYQn"
      },
      "source": [
        "for col, anova_val in linear_corr.sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    knclf = KNeighborsClassifier(\n",
        "        n_neighbors=10,\n",
        "        algorithm='ball_tree',\n",
        "        p=4\n",
        "        )\n",
        "\n",
        "    kncv = StratifiedKFold(n_splits=10, shuffle=True, random_state=182)\n",
        "\n",
        "    kncv_results = cross_validate(\n",
        "        knclf,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=kncv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    anova_result['k-nearest neighbors'][anova_val] = sum(kncv_results['test_score'])/len(kncv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q_mKQ0HS1Kui"
      },
      "source": [
        "## Gaussian naive Bayes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PBEYdOGD1Kuj"
      },
      "source": [
        "anova_result['Gaussian naive Bayes'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MTufQ6IV1Kuj"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "gnb = GaussianNB(var_smoothing=1e-10)\n",
        "\n",
        "gnbcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=302)\n",
        "\n",
        "gnbcv_results = cross_validate(\n",
        "    gnb,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=gnbcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "anova_result['Gaussian naive Bayes'][0] = sum(gnbcv_results['test_score'])/len(gnbcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qk0O1CWK1Kuj"
      },
      "source": [
        "for col, anova_val in linear_corr.sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    gnb = GaussianNB(var_smoothing=1e-10)\n",
        "\n",
        "    gnbcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=302)\n",
        "\n",
        "    gnbcv_results = cross_validate(\n",
        "        gnb,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=gnbcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    anova_result['Gaussian naive Bayes'][anova_val] = sum(gnbcv_results['test_score'])/len(gnbcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LcywDcDj1Kuk"
      },
      "source": [
        "## Decision Tree"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wweKT8c71Kuk"
      },
      "source": [
        "anova_result['Decision Tree'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gUcmlzY61Kuk"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "dtclf = DecisionTreeClassifier(\n",
        "    criterion='entropy',\n",
        "    max_depth=7,\n",
        "    max_features=None,\n",
        "    min_samples_leaf=3,\n",
        "    min_samples_split=0.4,\n",
        "    random_state=559,\n",
        "    splitter='random'\n",
        "    )\n",
        "\n",
        "dtcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=985)\n",
        "\n",
        "dtcv_results = cross_validate(\n",
        "    dtclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=dtcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "anova_result['Decision Tree'][0] = sum(dtcv_results['test_score'])/len(dtcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DDKmWAVs1Kuk"
      },
      "source": [
        "for col, anova_val in linear_corr.sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    dtclf = DecisionTreeClassifier(\n",
        "        criterion='entropy',\n",
        "        max_depth=7,\n",
        "        max_features=None,\n",
        "        min_samples_leaf=3,\n",
        "        min_samples_split=0.4,\n",
        "        random_state=559,\n",
        "        splitter='random'\n",
        "        )\n",
        "\n",
        "    dtcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=985)\n",
        "\n",
        "    dtcv_results = cross_validate(\n",
        "        dtclf,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=dtcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    anova_result['Decision Tree'][anova_val] = sum(dtcv_results['test_score'])/len(dtcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "if-l8grC1Kuk"
      },
      "source": [
        "## Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b7Csc5J71Kuk"
      },
      "source": [
        "anova_result['Random Forest'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q42zfsTc1Kul"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "rfclf = RandomForestClassifier(\n",
        "    criterion='gini',\n",
        "    max_features='sqrt',\n",
        "    min_samples_leaf=2,\n",
        "    min_samples_split=5,\n",
        "    n_estimators=50,\n",
        "    random_state=67\n",
        "    )\n",
        "\n",
        "rfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=309)\n",
        "\n",
        "rfcv_results = cross_validate(\n",
        "    rfclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=rfcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "anova_result['Random Forest'][0] = sum(rfcv_results['test_score'])/len(rfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JnygrwMU1Kul"
      },
      "source": [
        "for col, anova_val in linear_corr.sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    rfclf = RandomForestClassifier(\n",
        "        criterion='gini',\n",
        "        max_features='sqrt',\n",
        "        min_samples_leaf=2,\n",
        "        min_samples_split=5,\n",
        "        n_estimators=50,\n",
        "        random_state=67\n",
        "        )\n",
        "\n",
        "    rfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=309)\n",
        "\n",
        "    rfcv_results = cross_validate(\n",
        "        rfclf,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=rfcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    anova_result['Random Forest'][anova_val] = sum(rfcv_results['test_score'])/len(rfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3oRsbfiS1Kul"
      },
      "source": [
        "## Support Vector Machine"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDUixxu91Kul"
      },
      "source": [
        "anova_result['Support Vector Machine'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P-NUV7AS1Kul"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "svc = SVC(\n",
        "    C=0.1,\n",
        "    coef0=0.3,\n",
        "    degree=2,\n",
        "    gamma='scale',\n",
        "    kernel='poly',\n",
        "    random_state=98,\n",
        "    shrinking=True\n",
        "    )\n",
        "\n",
        "svccv = StratifiedKFold(n_splits=10, shuffle=True, random_state=911)\n",
        "\n",
        "svccv_results = cross_validate(\n",
        "    svc,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=svccv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "anova_result['Support Vector Machine'][0] = sum(svccv_results['test_score'])/len(svccv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tPTEGKY61Kum"
      },
      "source": [
        "for col, anova_val in linear_corr.sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    svc = SVC(\n",
        "        C=0.1,\n",
        "        coef0=0.3,\n",
        "        degree=2,\n",
        "        gamma='scale',\n",
        "        kernel='poly',\n",
        "        random_state=98,\n",
        "        shrinking=True\n",
        "        )\n",
        "\n",
        "    svccv = StratifiedKFold(n_splits=10, shuffle=True, random_state=911)\n",
        "\n",
        "    svccv_results = cross_validate(\n",
        "        svc,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=svccv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    anova_result['Support Vector Machine'][anova_val] = sum(svccv_results['test_score'])/len(svccv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qoiQhWQO1Kum"
      },
      "source": [
        "## Mutli-layer Perceptron"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TmTLJar71Kum"
      },
      "source": [
        "anova_result['Mutli-layer Perceptron'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wG-a_xF11Kum"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "mlpclf = MLPClassifier(\n",
        "    batch_size=8,\n",
        "    activation='relu',\n",
        "    early_stopping=False,\n",
        "    hidden_layer_sizes=(32, 32, 32),\n",
        "    max_iter=500,\n",
        "    solver='lbfgs',\n",
        "    random_state=377\n",
        "    )\n",
        "\n",
        "mlpclfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=496)\n",
        "\n",
        "mlpclfcv_results = cross_validate(\n",
        "    mlpclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=mlpclfcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "anova_result['Mutli-layer Perceptron'][0] = sum(mlpclfcv_results['test_score'])/len(mlpclfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hy8MR0vZ1Kun"
      },
      "source": [
        "for col, anova_val in linear_corr.sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    mlpclf = MLPClassifier(\n",
        "        batch_size=8,\n",
        "        activation='relu',\n",
        "        early_stopping=False,\n",
        "        hidden_layer_sizes=(32, 32, 32),\n",
        "        max_iter=500,\n",
        "        solver='lbfgs',\n",
        "        random_state=377\n",
        "        )\n",
        "\n",
        "    mlpclfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=496)\n",
        "\n",
        "    mlpclfcv_results = cross_validate(\n",
        "        mlpclf,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=mlpclfcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    anova_result['Mutli-layer Perceptron'][anova_val] = sum(mlpclfcv_results['test_score'])/len(mlpclfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HvaQd44G6nPv"
      },
      "source": [
        "## Result"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "gLhzoLmq1Kun",
        "outputId": "cec4c255-fcca-4688-acee-0338069f46a2"
      },
      "source": [
        "pd.DataFrame(anova_result.values(), index=anova_result.keys())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0.000000</th>\n",
              "      <th>1.386249</th>\n",
              "      <th>1.842203</th>\n",
              "      <th>2.095414</th>\n",
              "      <th>4.877729</th>\n",
              "      <th>5.837231</th>\n",
              "      <th>6.290820</th>\n",
              "      <th>6.677791</th>\n",
              "      <th>8.920381</th>\n",
              "      <th>15.299046</th>\n",
              "      <th>43.959767</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Logistic Regresssion</th>\n",
              "      <td>0.760714</td>\n",
              "      <td>0.789286</td>\n",
              "      <td>0.789286</td>\n",
              "      <td>0.775000</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.612500</td>\n",
              "      <td>0.612500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>k-nearest neighbors</th>\n",
              "      <td>0.764286</td>\n",
              "      <td>0.778571</td>\n",
              "      <td>0.766071</td>\n",
              "      <td>0.791071</td>\n",
              "      <td>0.787500</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.698214</td>\n",
              "      <td>0.737500</td>\n",
              "      <td>0.687500</td>\n",
              "      <td>0.587500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Gaussian naive Bayes</th>\n",
              "      <td>0.696429</td>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.723214</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.721429</td>\n",
              "      <td>0.746429</td>\n",
              "      <td>0.773214</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.723214</td>\n",
              "      <td>0.671429</td>\n",
              "      <td>0.671429</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Decision Tree</th>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.707143</td>\n",
              "      <td>0.616071</td>\n",
              "      <td>0.555357</td>\n",
              "      <td>0.676786</td>\n",
              "      <td>0.650000</td>\n",
              "      <td>0.642857</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.641071</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.555357</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Random Forest</th>\n",
              "      <td>0.760714</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.775000</td>\n",
              "      <td>0.746429</td>\n",
              "      <td>0.746429</td>\n",
              "      <td>0.760714</td>\n",
              "      <td>0.732143</td>\n",
              "      <td>0.721429</td>\n",
              "      <td>0.691071</td>\n",
              "      <td>0.578571</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Support Vector Machine</th>\n",
              "      <td>0.721429</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.762500</td>\n",
              "      <td>0.773214</td>\n",
              "      <td>0.733929</td>\n",
              "      <td>0.764286</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.725000</td>\n",
              "      <td>0.657143</td>\n",
              "      <td>0.662500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Mutli-layer Perceptron</th>\n",
              "      <td>0.582143</td>\n",
              "      <td>0.707143</td>\n",
              "      <td>0.748214</td>\n",
              "      <td>0.666071</td>\n",
              "      <td>0.639286</td>\n",
              "      <td>0.516071</td>\n",
              "      <td>0.598214</td>\n",
              "      <td>0.544643</td>\n",
              "      <td>0.639286</td>\n",
              "      <td>0.569643</td>\n",
              "      <td>0.589286</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                        0.000000   1.386249   ...  15.299046  43.959767\n",
              "Logistic Regresssion     0.760714   0.789286  ...   0.612500   0.612500\n",
              "k-nearest neighbors      0.764286   0.778571  ...   0.687500   0.587500\n",
              "Gaussian naive Bayes     0.696429   0.708929  ...   0.671429   0.671429\n",
              "Decision Tree            0.625000   0.707143  ...   0.625000   0.555357\n",
              "Random Forest            0.760714   0.735714  ...   0.691071   0.578571\n",
              "Support Vector Machine   0.721429   0.735714  ...   0.657143   0.662500\n",
              "Mutli-layer Perceptron   0.582143   0.707143  ...   0.569643   0.589286\n",
              "\n",
              "[7 rows x 11 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tD77s-7a7uO_"
      },
      "source": [
        "We can conclude ANOVA value of 1.84 gives best results so it is better to remove features 'GGT' and 'AST'."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pn2XakLp7998",
        "outputId": "770ffcc1-df34-42a7-a375-8bf05e6f91bc"
      },
      "source": [
        "to_remove_features += ['GGT', 'AST']\n",
        "to_remove_features"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['m', 'GGT', 'AST']"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jD0VpAQomtPq"
      },
      "source": [
        "# Kendall's τ coefficient"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gglpdyMBuVzm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "737ab887-9bee-4c9d-9913-650e88955e59"
      },
      "source": [
        "kendall_corr = [kendalltau(X[col], y).correlation for col in X.drop(to_remove_features, axis=1).columns]\n",
        "\n",
        "non_linear_corr = pd.Series(kendall_corr, index=X.drop(to_remove_features, axis=1).columns)\n",
        "non_linear_corr"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Age     0.377397\n",
              "ALB    -0.597778\n",
              "ALP     0.410298\n",
              "ALT    -0.217547\n",
              "BIL     0.350095\n",
              "CHE    -0.580503\n",
              "CHOL   -0.300940\n",
              "CREA    0.003535\n",
              "PROT   -0.191451\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zrrWz7HJ8Out"
      },
      "source": [
        "kendall_result = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D4AhI3Et8Out"
      },
      "source": [
        "## Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W3NlGyjq8Out"
      },
      "source": [
        "kendall_result['Logistic Regresssion'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tHO0VkI-8Ouu"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "lr = LogisticRegression(\n",
        "    C=1.25,\n",
        "    fit_intercept=True,\n",
        "    l1_ratio=0.5,\n",
        "    max_iter=50,\n",
        "    penalty='elasticnet',\n",
        "    random_state=221,\n",
        "    solver='saga'\n",
        "    )\n",
        "\n",
        "lrcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\n",
        "\n",
        "lrcv_results = cross_validate(\n",
        "    lr,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=lrcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "kendall_result['Logistic Regresssion'][0] = sum(lrcv_results['test_score'])/len(lrcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xIuxT4Hl8Ouu"
      },
      "source": [
        "for col, kendall_val in non_linear_corr.abs().sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    lr = LogisticRegression(\n",
        "        C=1.25,\n",
        "        fit_intercept=True,\n",
        "        l1_ratio=0.5,\n",
        "        max_iter=50,\n",
        "        penalty='elasticnet',\n",
        "        random_state=221,\n",
        "        solver='saga'\n",
        "        )\n",
        "\n",
        "    lrcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)\n",
        "\n",
        "    lrcv_results = cross_validate(\n",
        "        lr,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=lrcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    kendall_result['Logistic Regresssion'][kendall_val] = sum(lrcv_results['test_score'])/len(lrcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2-AX6eChQ1K8"
      },
      "source": [
        "## k-nearest neighbors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "niGOK_FFQ1K8"
      },
      "source": [
        "kendall_result['k-nearest neighbors'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bsYhiJ_bQ1K9"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "knclf = KNeighborsClassifier(\n",
        "    n_neighbors=10,\n",
        "    algorithm='ball_tree',\n",
        "    p=4\n",
        "    )\n",
        "\n",
        "kncv = StratifiedKFold(n_splits=10, shuffle=True, random_state=182)\n",
        "\n",
        "kncv_results = cross_validate(\n",
        "    knclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=kncv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "kendall_result['k-nearest neighbors'][0] = sum(kncv_results['test_score'])/len(kncv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CnGVL1AlQ1K9"
      },
      "source": [
        "for col, kendall_val in non_linear_corr.abs().sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    knclf = KNeighborsClassifier(\n",
        "        n_neighbors=10,\n",
        "        algorithm='ball_tree',\n",
        "        p=4\n",
        "        )\n",
        "\n",
        "    kncv = StratifiedKFold(n_splits=10, shuffle=True, random_state=182)\n",
        "\n",
        "    kncv_results = cross_validate(\n",
        "        knclf,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=kncv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    kendall_result['k-nearest neighbors'][kendall_val] = sum(kncv_results['test_score'])/len(kncv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_3aa7U4d8Ouu"
      },
      "source": [
        "## Gaussian naive Bayes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wVDNV1ym8Ouv"
      },
      "source": [
        "kendall_result['Gaussian naive Bayes'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CWJaYAn08Ouv"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "gnb = GaussianNB(var_smoothing=1e-10)\n",
        "\n",
        "gnbcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=302)\n",
        "\n",
        "gnbcv_results = cross_validate(\n",
        "    gnb,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=gnbcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "kendall_result['Gaussian naive Bayes'][0] = sum(gnbcv_results['test_score'])/len(gnbcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6i2rAIRW8Ouv"
      },
      "source": [
        "for col, kendall_val in non_linear_corr.abs().sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    gnb = GaussianNB(var_smoothing=1e-10)\n",
        "\n",
        "    gnbcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=302)\n",
        "\n",
        "    gnbcv_results = cross_validate(\n",
        "        gnb,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=gnbcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    kendall_result['Gaussian naive Bayes'][kendall_val] = sum(gnbcv_results['test_score'])/len(gnbcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uh3w58Bh8Ouv"
      },
      "source": [
        "## Decision Tree"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zFBWj0kK8Ouw"
      },
      "source": [
        "kendall_result['Decision Tree'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H7PPRmKs8Ouw"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "dtclf = DecisionTreeClassifier(\n",
        "    criterion='entropy',\n",
        "    max_depth=7,\n",
        "    max_features=None,\n",
        "    min_samples_leaf=3,\n",
        "    min_samples_split=0.4,\n",
        "    random_state=559,\n",
        "    splitter='random'\n",
        "    )\n",
        "\n",
        "dtcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=985)\n",
        "\n",
        "dtcv_results = cross_validate(\n",
        "    dtclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=dtcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "kendall_result['Decision Tree'][0] = sum(dtcv_results['test_score'])/len(dtcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "spMe-prp8Ouw"
      },
      "source": [
        "for col, kendall_val in non_linear_corr.abs().sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    dtclf = DecisionTreeClassifier(\n",
        "        criterion='entropy',\n",
        "        max_depth=7,\n",
        "        max_features=None,\n",
        "        min_samples_leaf=3,\n",
        "        min_samples_split=0.4,\n",
        "        random_state=559,\n",
        "        splitter='random'\n",
        "        )\n",
        "\n",
        "    dtcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=985)\n",
        "\n",
        "    dtcv_results = cross_validate(\n",
        "        dtclf,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=dtcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    kendall_result['Decision Tree'][kendall_val] = sum(dtcv_results['test_score'])/len(dtcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ieGWBClu8Ouw"
      },
      "source": [
        "## Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZrNREcgX8Oux"
      },
      "source": [
        "kendall_result['Random Forest'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9R1YMxxV8Ouy"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "rfclf = RandomForestClassifier(\n",
        "    criterion='gini',\n",
        "    max_features='sqrt',\n",
        "    min_samples_leaf=2,\n",
        "    min_samples_split=5,\n",
        "    n_estimators=50,\n",
        "    random_state=67\n",
        "    )\n",
        "\n",
        "rfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=309)\n",
        "\n",
        "rfcv_results = cross_validate(\n",
        "    rfclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=rfcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "kendall_result['Random Forest'][0] = sum(rfcv_results['test_score'])/len(rfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bjwgnlDD8Ouy"
      },
      "source": [
        "for col, kendall_val in non_linear_corr.abs().sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    rfclf = RandomForestClassifier(\n",
        "        criterion='gini',\n",
        "        max_features='sqrt',\n",
        "        min_samples_leaf=2,\n",
        "        min_samples_split=5,\n",
        "        n_estimators=50,\n",
        "        random_state=67\n",
        "        )\n",
        "\n",
        "    rfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=309)\n",
        "\n",
        "    rfcv_results = cross_validate(\n",
        "        rfclf,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=rfcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    kendall_result['Random Forest'][kendall_val] = sum(rfcv_results['test_score'])/len(rfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8lV-mFRG8Ouy"
      },
      "source": [
        "## Support Vector Machine"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "62ctRLXC8Ouy"
      },
      "source": [
        "kendall_result['Support Vector Machine'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ibvz8Cgn8Ouy"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "svc = SVC(\n",
        "    C=0.1,\n",
        "    coef0=0.3,\n",
        "    degree=2,\n",
        "    gamma='scale',\n",
        "    kernel='poly',\n",
        "    random_state=98,\n",
        "    shrinking=True\n",
        "    )\n",
        "\n",
        "svccv = StratifiedKFold(n_splits=10, shuffle=True, random_state=911)\n",
        "\n",
        "svccv_results = cross_validate(\n",
        "    svc,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=svccv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "kendall_result['Support Vector Machine'][0] = sum(svccv_results['test_score'])/len(svccv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-WoJidlH8Ouy"
      },
      "source": [
        "for col, kendall_val in non_linear_corr.abs().sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    svc = SVC(\n",
        "        C=0.1,\n",
        "        coef0=0.3,\n",
        "        degree=2,\n",
        "        gamma='scale',\n",
        "        kernel='poly',\n",
        "        random_state=98,\n",
        "        shrinking=True\n",
        "        )\n",
        "\n",
        "    svccv = StratifiedKFold(n_splits=10, shuffle=True, random_state=911)\n",
        "\n",
        "    svccv_results = cross_validate(\n",
        "        svc,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=svccv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    kendall_result['Support Vector Machine'][kendall_val] = sum(svccv_results['test_score'])/len(svccv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "us1YC0xg8Ouz"
      },
      "source": [
        "## Mutli-layer Perceptron"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lm2La3SO8Ouz"
      },
      "source": [
        "kendall_result['Mutli-layer Perceptron'] = {}\n",
        "cols_removed = to_remove_features.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MplQMN1t8Ouz"
      },
      "source": [
        "X_dropped = X.drop(cols_removed, axis=1)\n",
        "scaler = MinMaxScaler()\n",
        "scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "mlpclf = MLPClassifier(\n",
        "    batch_size=8,\n",
        "    activation='relu',\n",
        "    early_stopping=False,\n",
        "    hidden_layer_sizes=(32, 32, 32),\n",
        "    max_iter=500,\n",
        "    random_state=377,\n",
        "    solver='lbfgs'\n",
        "    )\n",
        "\n",
        "mlpclfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=496)\n",
        "\n",
        "mlpclfcv_results = cross_validate(\n",
        "    mlpclf,\n",
        "    scaled_X,\n",
        "    y['category'],\n",
        "    cv=mlpclfcv,\n",
        "    scoring='accuracy',\n",
        "    n_jobs=-1\n",
        "    )\n",
        "\n",
        "kendall_result['Mutli-layer Perceptron'][0] = sum(mlpclfcv_results['test_score'])/len(mlpclfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0xY8d5jQ8Ouz"
      },
      "source": [
        "for col, kendall_val in non_linear_corr.abs().sort_values()[:-1].iteritems():\n",
        "    cols_removed.append(col)\n",
        "    X_dropped = X.drop(cols_removed, axis=1)\n",
        "    scaler = MinMaxScaler()\n",
        "    scaled_X = scaler.fit_transform(X_dropped)\n",
        "\n",
        "    mlpclf = MLPClassifier(\n",
        "        batch_size=8,\n",
        "        activation='relu',\n",
        "        early_stopping=False,\n",
        "        hidden_layer_sizes=(32, 32, 32),\n",
        "        max_iter=500,\n",
        "        random_state=377,\n",
        "        solver='lbfgs'\n",
        "        )\n",
        "\n",
        "    mlpclfcv = StratifiedKFold(n_splits=10, shuffle=True, random_state=496)\n",
        "\n",
        "    mlpclfcv_results = cross_validate(\n",
        "        mlpclf,\n",
        "        scaled_X,\n",
        "        y['category'],\n",
        "        cv=mlpclfcv,\n",
        "        scoring='accuracy',\n",
        "        n_jobs=-1\n",
        "        )\n",
        "\n",
        "    kendall_result['Mutli-layer Perceptron'][kendall_val] = sum(mlpclfcv_results['test_score'])/len(mlpclfcv_results['test_score'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qLvRzjvN8Ouz"
      },
      "source": [
        "## Result"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "gYIVzkTt8Ouz",
        "outputId": "0d2a1339-923b-4bc1-d186-b026846acae3"
      },
      "source": [
        "pd.DataFrame(kendall_result.values(), index=kendall_result.keys())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0.000000</th>\n",
              "      <th>0.003535</th>\n",
              "      <th>0.191451</th>\n",
              "      <th>0.217547</th>\n",
              "      <th>0.300940</th>\n",
              "      <th>0.350095</th>\n",
              "      <th>0.377397</th>\n",
              "      <th>0.410298</th>\n",
              "      <th>0.580503</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Logistic Regresssion</th>\n",
              "      <td>0.789286</td>\n",
              "      <td>0.775000</td>\n",
              "      <td>0.803571</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.612500</td>\n",
              "      <td>0.612500</td>\n",
              "      <td>0.650000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>k-nearest neighbors</th>\n",
              "      <td>0.766071</td>\n",
              "      <td>0.791071</td>\n",
              "      <td>0.816071</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.748214</td>\n",
              "      <td>0.664286</td>\n",
              "      <td>0.687500</td>\n",
              "      <td>0.653571</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Gaussian naive Bayes</th>\n",
              "      <td>0.723214</td>\n",
              "      <td>0.750000</td>\n",
              "      <td>0.787500</td>\n",
              "      <td>0.746429</td>\n",
              "      <td>0.773214</td>\n",
              "      <td>0.733929</td>\n",
              "      <td>0.639286</td>\n",
              "      <td>0.671429</td>\n",
              "      <td>0.621429</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Decision Tree</th>\n",
              "      <td>0.616071</td>\n",
              "      <td>0.555357</td>\n",
              "      <td>0.748214</td>\n",
              "      <td>0.650000</td>\n",
              "      <td>0.642857</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.619643</td>\n",
              "      <td>0.625000</td>\n",
              "      <td>0.401786</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Random Forest</th>\n",
              "      <td>0.708929</td>\n",
              "      <td>0.775000</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>0.746429</td>\n",
              "      <td>0.760714</td>\n",
              "      <td>0.733929</td>\n",
              "      <td>0.650000</td>\n",
              "      <td>0.691071</td>\n",
              "      <td>0.623214</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Support Vector Machine</th>\n",
              "      <td>0.762500</td>\n",
              "      <td>0.773214</td>\n",
              "      <td>0.817857</td>\n",
              "      <td>0.764286</td>\n",
              "      <td>0.735714</td>\n",
              "      <td>0.733929</td>\n",
              "      <td>0.682143</td>\n",
              "      <td>0.657143</td>\n",
              "      <td>0.614286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Mutli-layer Perceptron</th>\n",
              "      <td>0.748214</td>\n",
              "      <td>0.666071</td>\n",
              "      <td>0.867857</td>\n",
              "      <td>0.516071</td>\n",
              "      <td>0.598214</td>\n",
              "      <td>0.707143</td>\n",
              "      <td>0.605357</td>\n",
              "      <td>0.569643</td>\n",
              "      <td>0.626786</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                        0.000000  0.003535  ...  0.410298  0.580503\n",
              "Logistic Regresssion    0.789286  0.775000  ...  0.612500  0.650000\n",
              "k-nearest neighbors     0.766071  0.791071  ...  0.687500  0.653571\n",
              "Gaussian naive Bayes    0.723214  0.750000  ...  0.671429  0.621429\n",
              "Decision Tree           0.616071  0.555357  ...  0.625000  0.401786\n",
              "Random Forest           0.708929  0.775000  ...  0.691071  0.623214\n",
              "Support Vector Machine  0.762500  0.773214  ...  0.657143  0.614286\n",
              "Mutli-layer Perceptron  0.748214  0.666071  ...  0.569643  0.626786\n",
              "\n",
              "[7 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0SZZJZ-L8Ouz"
      },
      "source": [
        "We can conclude Kendall's value of 0.191451 gives best results so it is better to remove features CREA and PROT."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ATmICQAtNkVB",
        "outputId": "8f3a90ee-c86c-45ec-ef91-8c6f1e49a349"
      },
      "source": [
        "to_remove_features += ['CREA', 'PROT']\n",
        "to_remove_features"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['m', 'GGT', 'AST', 'CREA', 'PROT']"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    }
  ]
}