ML_model_train_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ebef34a-b387-44eb-b7aa-a00aa5e201d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
    "\n",
    "# setup for multiple output display\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89745583-1609-4642-a785-e947d11322a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import glob\n",
    "import keras\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from scipy.io import loadmat\n",
    "from keras.layers import *\n",
    "from keras.models import *\n",
    "from keras.optimizers import *\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "from sklearn.compose import make_column_transformer\n",
    "from  sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, LabelEncoder # Convert data to oneHotEncoder\n",
    "from sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler, MaxAbsScaler\n",
    "\n",
    "from sklearn.svm import SVR\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.multioutput import MultiOutputRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c899bd39-4a11-488b-800f-eea0711e38d3",
   "metadata": {},
   "source": [
    "# Set up the GPU device [optional]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66797044-830b-42e8-86d9-23351a192462",
   "metadata": {},
   "outputs": [],
   "source": [
    "# A tensor environment should be setup for GPU, read instructions for details.\n",
    "# Set up the GPU device\n",
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "physical_devices\n",
    "# tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
    "\n",
    "# Check if GPU is available\n",
    "if tf.test.gpu_device_name():\n",
    "    print('GPU found')\n",
    "else:\n",
    "    print(\"No GPU found\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0d5b857-4a6e-4e73-be2d-280031206c89",
   "metadata": {},
   "source": [
    "# Read the train and test dataset files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f95ff6d-f55d-4bd8-80d9-b041110378bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainDataFilePath = \"data/train_dataset.csv\";\n",
    "testDataFilePath = \"data/test_dataset.csv\";\n",
    "coDataFilePath = \"data/unseen_test_dataset.csv\";\n",
    "\n",
    "df_train = pd.read_csv(trainDataFilePath) # Read train data CSV file\n",
    "df_train = shuffle(df_train)\n",
    "df_train.head()\n",
    "\n",
    "df_test = pd.read_csv(testDataFilePath) # Read test data CSV file\n",
    "df_test.head()\n",
    "\n",
    "df_co = pd.read_csv(coDataFilePath) # Read cohort test data CSV file\n",
    "df_co.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af85aed9-d6f5-4878-a672-e2f86ecbe428",
   "metadata": {},
   "source": [
    "# Checking class counts in respective datasets [for verification]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "079a395f-e29d-443e-8692-2a9f0c46fa9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_counts = df_train['Jettable'].value_counts() #Training Data Class Count\n",
    "print (\"Train: \",class_counts.index,\"| \", class_counts.values)\n",
    "\n",
    "class_counts = df_test['Jettable'].value_counts() #Test Data Class Count\n",
    "print (\"Test: \",class_counts.index,\"| \", class_counts.values)\n",
    "\n",
    "class_counts = df_co['Jettable'].value_counts() #Test Data Class Count\n",
    "print (\"Cohort: \",class_counts.index,\"| \", class_counts.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d44fde8-1b4b-46af-9c50-b9c5c6b5047f",
   "metadata": {},
   "source": [
    "# Drop the unnecessary feature columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7b951ff-2cc3-44d7-8574-b29b2703a106",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df_train.drop(['MaterialName','Printer','Re','We','Ca','Z','Oh','Bo'],axis=1)\n",
    "df_test = df_test.drop(['MaterialName','Printer','Re','We','Ca','Z','Oh','Bo'],axis=1)\n",
    "df_co = df_co.drop(['MaterialName','Printer','Re','We','Ca','Z','Oh','Bo'],axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2e0cf02-5f22-4b81-a9db-7d9c47deffb3",
   "metadata": {},
   "source": [
    "# Check for NA values in each column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d29b66cb-b891-49f3-b690-b0b7fa0ed60f",
   "metadata": {},
   "outputs": [],
   "source": [
    "has_nan_per_column = df_train.isnull().any(axis=0)\n",
    "print(has_nan_per_column)\n",
    "\n",
    "print ('--------------------------------------------------')\n",
    "has_nan_per_column = df_test.isnull().any(axis=0)\n",
    "print(has_nan_per_column)\n",
    "\n",
    "print ('--------------------------------------------------')\n",
    "has_nan_per_column = df_co.isnull().any(axis=0)\n",
    "print(has_nan_per_column)\n",
    "\n",
    "# replace the na values with zero\n",
    "df_train = df_train.fillna(0)\n",
    "df_test = df_test.fillna(0)\n",
    "df_co = df_co.fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40606391-c742-4ad3-9cf8-8a09c256ec61",
   "metadata": {},
   "source": [
    "# Data normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fbec397-16dc-45ad-884f-a7ea54c63808",
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate features and target variable\n",
    "df_train_num = df_train.drop(['PulseDuration','Voltage','Trise','Tfall','Jettable','WaveformType'], axis=1) # target columns are also dropped\n",
    "df_test_num = df_test.drop(['PulseDuration','Voltage','Trise','Tfall','Jettable','WaveformType'], axis=1)\n",
    "df_co_num = df_co.drop(['PulseDuration','Voltage','Trise','Tfall','Jettable','WaveformType'], axis=1)\n",
    "\n",
    "# save non-numeric columns\n",
    "non_numeric_cols_train = df_train.select_dtypes(exclude='number').columns\n",
    "non_numeric_cols_test = df_test.select_dtypes(exclude='number').columns\n",
    "non_numeric_cols_co = df_co.select_dtypes(exclude='number').columns\n",
    "\n",
    "# apply MinMaxScaler to all numeric columns except target column\n",
    "scaler = MinMaxScaler()\n",
    "df_train_num_scaled = scaler.fit_transform(df_train_num)\n",
    "df_test_num_scaled = scaler.transform(df_test_num)\n",
    "df_co_num_scaled = scaler.transform(df_co_num)\n",
    "\n",
    "# replace the original numeric columns with scaled data\n",
    "df_train[df_train_num.columns] = df_train_num_scaled\n",
    "df_test[df_test_num.columns] = df_test_num_scaled\n",
    "df_co[df_co_num.columns] = df_co_num_scaled\n",
    "\n",
    "# restore non-numeric columns\n",
    "df_train[non_numeric_cols_train] = df_train[non_numeric_cols_train]\n",
    "df_test[non_numeric_cols_test] = df_test[non_numeric_cols_test]\n",
    "df_co[non_numeric_cols_co] = df_co[non_numeric_cols_co]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24a2f0f2-e9c6-4343-8b33-c0783346b594",
   "metadata": {},
   "source": [
    "# Applying the Label Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8ffbf0d-3cdf-4217-a63e-643411e68b9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_column(df_train, df_test, df_co, column_name):\n",
    "    # Select the column by name\n",
    "    train_selected_column = df_train[column_name]\n",
    "    test_selected_column = df_test[column_name]\n",
    "    co_selected_column = df_co[column_name]\n",
    "\n",
    "    # Initialize the label encoder\n",
    "    encoder = LabelEncoder()\n",
    "\n",
    "    # Convert string values to numerical values using the encoder\n",
    "    train_encoded_values = encoder.fit_transform(train_selected_column)\n",
    "    test_encoded_values = encoder.transform(test_selected_column)\n",
    "    co_encoded_values = encoder.transform(co_selected_column)\n",
    "\n",
    "    # Replace the original column with the encoded values\n",
    "    df_train[column_name] = train_encoded_values\n",
    "    df_test[column_name] = test_encoded_values\n",
    "    df_co[column_name] = co_encoded_values\n",
    "\n",
    "    # Get the categories and their corresponding labels\n",
    "    categories = encoder.classes_\n",
    "    labels = encoder.transform(categories)\n",
    "\n",
    "    # Print the categories and labels\n",
    "    for category, label in zip(categories, labels):\n",
    "        print(f\"{category} -> {label}\")\n",
    "\n",
    "    return categories, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2522ec1-ad13-4809-b715-7fc5aaff3930",
   "metadata": {},
   "outputs": [],
   "source": [
    "encode_column(df_train, df_test, df_co, 'Jettable')\n",
    "encode_column(df_train, df_test, df_co, 'WaveformType')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65ccfaa5-39f5-40b9-9b03-7d7bff1b4d0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = df_train.drop(['PulseDuration','Voltage','Trise','Tfall'],axis=1) \n",
    "y_train = df_train[['PulseDuration','Voltage','Trise','Tfall']] \n",
    "\n",
    "x_test = df_test.drop(['PulseDuration','Voltage','Trise','Tfall'],axis=1)\n",
    "y_test = df_test[['PulseDuration','Voltage','Trise','Tfall']] \n",
    "\n",
    "\n",
    "x_co = df_co.drop(['PulseDuration','Voltage','Trise','Tfall'],axis=1)\n",
    "y_co = df_co[['PulseDuration','Voltage','Trise','Tfall']] \n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train,\n",
    "                                                  y_train,\n",
    "                                                  test_size=ratioTestTrain,\n",
    "                                                  random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15b28c9c-2ebf-4128-9741-ac7d6e4b177c",
   "metadata": {},
   "source": [
    "# Exporting results custom function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11bcb872-f64a-4a08-bd27-7d8639eceaba",
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_output_metrics(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Calculate performance metrics (RMSE, MAE, MAPE) for multiple outputs in a regression task.\n",
    "\n",
    "    Parameters:\n",
    "    y_true: The true target values for each output.\n",
    "    y_pred: The predicted target values for each output.\n",
    "\n",
    "    Returns:\n",
    "    tuple: A dictionary (`metrics`) with performance metrics for each output and \n",
    "           a pandas DataFrame (`metrics_df`) summarizing the metrics.\n",
    "    \"\"\"\n",
    "    metrics = {}\n",
    "    for i, col in enumerate(y_true.columns):\n",
    "        rmse = np.sqrt(mean_squared_error(y_true[col], y_pred[:, i]))\n",
    "        mae = mean_absolute_error(y_true[col], y_pred[:, i])\n",
    "        r2 = r2_score(y_true[col], y_pred[:, i])\n",
    "        mape = np.mean(np.abs((y_true[col] - y_pred[:, i]) / y_true[col])) * 100  # Convert to percentage\n",
    "\n",
    "        # metrics[col] = {'RMSE': rmse, 'R2 Score': r2, 'MAE': mae, 'MAPE': mape}\n",
    "        metrics[col] = {'RMSE': rmse, 'MAE': mae, 'MAPE': mape}\n",
    "        print(f\"Output: {col}\")\n",
    "        print(f\"RMSE: {rmse:.6f}\")\n",
    "        # print(f\"R2 Score: {r2:.6f}\")\n",
    "        print(f\"MAE: {mae:.6f}\")\n",
    "        print(f\"MAPE: {mape:.6f}%\")  # Display MAPE as percentage\n",
    "        print()\n",
    "    \n",
    "    metrics = pd.DataFrame(metrics)\n",
    "    # print(metrics)\n",
    "    metrics_df = pd.DataFrame(metrics).T\n",
    "    print(metrics_df)\n",
    "    return metrics, metrics_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c83b1a7-b5b2-414b-a225-d38b24d1b063",
   "metadata": {},
   "source": [
    "# Setup models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb8ae6b4-6ad2-4792-bc17-c7494e9233fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_evaluate_model(model_name, model, x_train, y_train, x_test, y_test, testType):\n",
    "    \"\"\"\n",
    "    Train a regression model, evaluate its performance, and save predictions and metrics to files.\n",
    "\n",
    "    Parameters:\n",
    "    model_name (str): The name of the model being trained.\n",
    "    model (object): The regression model to be trained, implementing the `fit` and `predict` methods.\n",
    "    x_train (numpy.ndarray or pandas.DataFrame): The input features for training.\n",
    "    y_train (numpy.ndarray or pandas.DataFrame): The target values for training.\n",
    "    x_test (numpy.ndarray or pandas.DataFrame): The input features for testing.\n",
    "    y_test (numpy.ndarray or pandas.DataFrame): The target values for testing.\n",
    "    testType (str): A label for the type of test (used to name output files).\n",
    "\n",
    "    Returns:\n",
    "    None\n",
    "\n",
    "    Functionality:\n",
    "    1. Trains the model using the provided training data (`x_train`, `y_train`).\n",
    "    2. Generates predictions (`y_pred`) for the test set (`x_test`).\n",
    "    3. Saves predictions to a CSV file named `<testType>_<model_name>Prediction.csv`.\n",
    "    4. Computes evaluation metrics (RMSE, MAE, MAPE) for the predictions against the true test labels (`y_test`).\n",
    "    5. Saves the computed metrics to:\n",
    "       - A text file `<testType>_<model_name>PerformanceScores.txt`.\n",
    "       - A CSV file `<testType>_<model_name>PerformanceScores.csv`.\n",
    "\n",
    "    Notes:\n",
    "    - The predictions are saved with column names: `['PulseDuration', 'Voltage', 'Trise', 'Tfall']`.\n",
    "    - The `multi_output_metrics` function is used to calculate the evaluation metrics.\n",
    "    - The `save_data_with_name` function appends metrics and text to a file.\n",
    "    \"\"\"\n",
    "    # Train the model\n",
    "    model.fit(x_train, y_train)\n",
    "        \n",
    "    # Predict the classes for the test set\n",
    "    y_pred = model.predict(x_test)\n",
    "    \n",
    "    predFilePath = f\"{testType}_{model_name}Prediction.csv\"\n",
    "    column_names = ['PulseDuration','Voltage','Trise','Tfall']\n",
    "    pd.DataFrame(y_pred, columns=column_names).to_csv(predFilePath)\n",
    "\n",
    "    # Compute and display metrics\n",
    "    output_metrics, output_metrics_df  = multi_output_metrics(y_test, y_pred)\n",
    "    \n",
    "\n",
    "    # Save the performance results to txt file\n",
    "    txtFilePath = f\"{testType}_{model_name}PerformanceScores.txt\"\n",
    "    csvFilePath = f\"{testType}_{model_name}PerformanceScores.csv\"\n",
    "\n",
    "    # Function to append data and name to the file\n",
    "    def save_data_with_name(name, data, file_path):\n",
    "        with open(file_path, 'a') as f:\n",
    "            f.write(f\"# {name}\")\n",
    "            f.write(str(data) + \"\\n\")\n",
    "\n",
    "    # Append performance metrics data to txt file\n",
    "    save_data_with_name(\"-------------------------\", '', txtFilePath)\n",
    "    save_data_with_name(\"\", output_metrics, txtFilePath) \n",
    "    \n",
    "    # Append performance metrics data to csv file\n",
    "    output_metrics_df.to_csv(csvFilePath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb29aa09-98b4-47c4-8753-1864a932a470",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ''\n",
    "modelName = ''\n",
    "\n",
    "train = False\n",
    "if train == True: \n",
    "    # # SVR\n",
    "    # model = SVR(C=5, degree = 1, gamma='scale', kernel='poly', max_iter=15)\n",
    "    # # model = SVR(C=5, degree = 1, gamma='scale', kernel='poly', max_iter=15) # SELECTED\n",
    "    # model = MultiOutputRegressor(model)\n",
    "    # modelName = 'svr'\n",
    "\n",
    "    # # RF\n",
    "    # model = RandomForestRegressor(max_depth=1, min_samples_leaf=2, min_samples_split=5, n_estimators=4)\n",
    "    # # model = RandomForestRegressor(max_depth=2, min_samples_leaf=2, min_samples_split=5, n_estimators=4) # SELECTED\n",
    "    # model = MultiOutputRegressor(model)\n",
    "    # modelName = 'rf'\n",
    "\n",
    "    # # GB\n",
    "    # model = GradientBoostingRegressor(learning_rate=0.1, max_depth=1, n_estimators=20)\n",
    "    # # model = GradientBoostingRegressor(learning_rate=0.1, max_depth=1, n_estimators=20) # SELECTED\n",
    "    # model = MultiOutputRegressor(model)\n",
    "    # modelName = 'gb'\n",
    "\n",
    "    # # XGBoost\n",
    "    # model = XGBRegressor(objective='reg:squarederror', learning_rate=0.1, max_depth=2, n_estimators=35)\n",
    "    # # model = XGBRegressor(objective='reg:squarederror', learning_rate=0.1, max_depth=2, n_estimators=35) # SELECTED\n",
    "    # model = MultiOutputRegressor(model)\n",
    "    # modelName = 'gbx'\n",
    "\n",
    "    # # DT\n",
    "    # model = DecisionTreeRegressor(max_depth=1, min_samples_leaf=5, min_samples_split=10)\n",
    "    # # model = DecisionTreeRegressor(max_depth=1, min_samples_leaf=5, min_samples_split=10) # SELECTED\n",
    "    # model = MultiOutputRegressor(model)\n",
    "    # modelName = 'dt'\n",
    "    \n",
    "    print(f\"{modelName} has been trained. . . \")\n",
    "else:\n",
    "    print(\"training is false\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb29713e-dadf-41c3-871f-6e5e36199350",
   "metadata": {},
   "source": [
    "# Model Performance Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac494194-5386-4b83-ab19-1ff21bda17da",
   "metadata": {},
   "outputs": [],
   "source": [
    "if train == True: \n",
    "    # Train and evaluate the model\n",
    "    train_and_evaluate_model(modelName, model, x_train, y_train, x_test, y_test, testType = 'test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa026849-17c3-4449-80e6-28e7c15e517d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if train == True:\n",
    "    # Train and evaluate the model\n",
    "    train_and_evaluate_model(modelName, model, x_train, y_train, x_co, y_co, testType = 'co')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}