cooling_subsystem_analysis.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Notes:\n",
        "*   <mark>For confidentiality reasons</mark>, no figure is shown as a result of any block execution and the actual dataset naming and input/output addresses are replaced by aliases.\n",
        "*   The lookup tables implemented in the notebook are all taken from \"<mark>Van der Horst, S. A. M. (2019). Economically optimizing maintenance of air handling units (Master’s thesis). Technische Universiteit Eindhoven</mark>\".\n",
        "*   The ARM approach implemented in this notebook was inspired and mainly learned from: <mark>Isaiah Hull. Market Basket Analysis in Python. https://www.datacamp.com/courses/\n",
        "market-basket-analysis-in-python</mark>."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7595UIXVFovr"
      },
      "source": [
        "# Utils"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xPUgLKbTr4tE"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FzcugnMcFtfO"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import string\n",
        "import re\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "from nltk.tokenize import word_tokenize\n",
        "from mlxtend.preprocessing import TransactionEncoder\n",
        "from mlxtend.frequent_patterns import apriori\n",
        "from mlxtend.frequent_patterns import association_rules\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from wordcloud import WordCloud\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "utbh3w3Nr8jG"
      },
      "source": [
        "## Download Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kAqZ8u70Fxwc"
      },
      "outputs": [],
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-jed0F0KsC0F"
      },
      "source": [
        "## Implement Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r8q14WZMF0rB"
      },
      "outputs": [],
      "source": [
        "def correct_types(df, columns):\n",
        "    \"\"\"\n",
        "    Preprocess the specified columns in a DataFrame by replacing NaN values with an empty string\n",
        "    and converting non-string values to string objects.\n",
        "\n",
        "    Parameters:\n",
        "    - df (pd.DataFrame): The input DataFrame.\n",
        "    - column (str): The column to be preprocessed. Default is 'Description'.\n",
        "\n",
        "    Returns:\n",
        "    - pd.DataFrame: The DataFrame with the specified column preprocessed.\n",
        "    \"\"\"\n",
        "\n",
        "    df[columns] = df[columns].fillna('')\n",
        "    df[columns] = df[columns].astype(str)\n",
        "    return df\n",
        "\n",
        "def replace_punctuation_text(text):\n",
        "    \"\"\"\n",
        "    Replace punctuation in the input text.\n",
        "\n",
        "    Parameters:\n",
        "    - text (str): The input text in which punctuation will be replaced.\n",
        "\n",
        "    Returns:\n",
        "    - str: The input text with punctuation replaced.\n",
        "    \"\"\"\n",
        "    # Replace \"'s\" with empty space.\n",
        "    text = text.replace(\"'s\", '')\n",
        "\n",
        "    # Create a translation table to replace '.' and \"'\" with empty space, and other punctuation with spaces.\n",
        "    translator = str.maketrans({'.': '', \"'\": '', **{p: ' ' for p in string.punctuation if p not in ['.', \"'\"]}})\n",
        "\n",
        "    cleaned_text = text.translate(translator)\n",
        "\n",
        "    return cleaned_text\n",
        "\n",
        "def replace_punctuation(df, columns):\n",
        "    \"\"\"\n",
        "    Replaces specific punctuation in the specified columns of a DataFrame.\n",
        "\n",
        "    This function is designed to clean textual data in the specified columns of a DataFrame by:\n",
        "    - Replacing occurrences of \"'s\" with an empty string.\n",
        "    - Replacing periods (.) and apostrophes (') with an empty string.\n",
        "    - Replacing other punctuation marks with a space.\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): The DataFrame containing the columns to be cleaned.\n",
        "        columns (list or str): The column name(s) of the DataFrame where the replacements should be applied.\n",
        "\n",
        "    Returns:\n",
        "        pd.DataFrame: The DataFrame with the specified columns cleaned of certain punctuation marks.\n",
        "    \"\"\"\n",
        "    # Replace \"'s\" with an empty string in the specified columns.\n",
        "    df[columns] = df[columns].replace(\"'s\", '', regex=True)\n",
        "\n",
        "    # Create a translation table to replace '.' and \"'\" with empty space,\n",
        "    # and other punctuation with spaces.\n",
        "    translator = str.maketrans({\n",
        "        '.': '',                # Remove periods\n",
        "        \"'\": '',                # Remove apostrophes\n",
        "        **{p: ' ' for p in string.punctuation if p not in ['.', \"'\"]}  # Replace other punctuation with space\n",
        "    })\n",
        "\n",
        "    # Apply the translation table to each element in the specified columns.\n",
        "    df[columns] = df[columns].applymap(lambda x: x.translate(translator))\n",
        "\n",
        "    return df\n",
        "\n",
        "def remove_numerical_values_text(text):\n",
        "    \"\"\"\n",
        "    Remove numbers and model-related patterns from the input text.\n",
        "\n",
        "    Parameters:\n",
        "    - text (str): The input text from which numbers and model-related patterns will be removed.\n",
        "\n",
        "    Returns:\n",
        "    - str: The input text with numbers and model-related patterns removed.\n",
        "    \"\"\"\n",
        "    # Remove standalone numbers with optional floating points; such as 154, 99.31.\n",
        "    text = re.sub(r'\\b\\d+(\\.\\d+)?\\b', '', text)\n",
        "\n",
        "    # Remove ordinal numbers; such as 1e, 2ste, 3de, 4e.\n",
        "    text = re.sub(r'\\b\\d+(e|ste|de|e)\\b', '', text)\n",
        "\n",
        "    # Remove numerical quantifiers; such as 1x OH.\n",
        "    text = re.sub(r'\\b\\d+[xX]\\b', '', text)\n",
        "\n",
        "    return text\n",
        "\n",
        "def remove_numerical_values(df, columns):\n",
        "    \"\"\"\n",
        "    Removes various forms of numerical values from the specified columns in a DataFrame.\n",
        "\n",
        "    This function cleans textual data in the specified columns by:\n",
        "    - Removing standalone numbers with optional floating points (e.g., 154, 99.31).\n",
        "    - Removing ordinal numbers commonly used in Dutch (e.g., 1e, 2ste, 3de, 4e).\n",
        "    - Removing numerical quantifiers with 'x' (e.g., 1x, 2X).\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): The DataFrame containing the columns to be cleaned.\n",
        "        columns (list or str): The column name(s) of the DataFrame where the numerical values should be removed.\n",
        "\n",
        "    Returns:\n",
        "        pd.DataFrame: The DataFrame with the specified columns cleaned of numerical values.\n",
        "    \"\"\"\n",
        "    # Remove standalone numbers with optional floating points (e.g., 154, 99.31).\n",
        "    df[columns] = df[columns].applymap(lambda x: re.sub(r'\\b\\d+(\\.\\d+)?\\b', '', x))\n",
        "\n",
        "    # Remove ordinal numbers (e.g., 1e, 2ste, 3de, 4e).\n",
        "    df[columns] = df[columns].applymap(lambda x: re.sub(r'\\b\\d+(e|ste|de)\\b', '', x))\n",
        "\n",
        "    # Remove numerical quantifiers (e.g., 1x, 2X).\n",
        "    df[columns] = df[columns].applymap(lambda x: re.sub(r'\\b\\d+[xX]\\b', '', x))\n",
        "\n",
        "    return df\n",
        "\n",
        "def stemize(text):\n",
        "    \"\"\"\n",
        "    Perform stemming on the input text using the Dutch Snowball Stemmer.\n",
        "\n",
        "    Parameters:\n",
        "    - text (str): The input text to be stemmed.\n",
        "\n",
        "    Returns:\n",
        "    str: The stemmed text.\n",
        "\n",
        "    Example:\n",
        "    >>> stemize(\"This is an example text for stemming.\")\n",
        "    'thi is an exampl text for stem.'\n",
        "    \"\"\"\n",
        "    stemmer = SnowballStemmer(\"dutch\")\n",
        "    tokens = word_tokenize(text)\n",
        "    text = ' '.join([stemmer.stem(token) for token in tokens])\n",
        "    return text\n",
        "\n",
        "def normalize_lookups():\n",
        "    \"\"\"\n",
        "    Normalize and preprocess the global lookup lists.\n",
        "\n",
        "    This function performs the following operations on each global lookup list:\n",
        "    1. Orders lookup lists by length in descending order.\n",
        "    2. Initializes the Dutch Snowball Stemmer.\n",
        "    3. Replaces punctuations using the `replace_punctuation` function.\n",
        "    4. Removes numerical values using the `remove_numerical_values` function.\n",
        "    5. Applies stemming using the `stemize` function.\n",
        "    6. Converts the resulting list to a set.\n",
        "\n",
        "    Note: The original lookup lists are modified in-place.\n",
        "\n",
        "    Returns:\n",
        "    None\n",
        "    \"\"\"\n",
        "    # Get all lookup lists using regular expression.\n",
        "    lookup_lists = [var for var in globals() if re.match(r'^lookup_', var)]\n",
        "    lookup_ventilation_lists = [var for var in globals() if re.match(r'^sublookup_', var)]\n",
        "\n",
        "    for lookup_list_name in lookup_lists:\n",
        "\n",
        "        lookup_list = globals()[lookup_list_name]\n",
        "\n",
        "        # Replace punctuations, remove numerical values, and apply stemming.\n",
        "        normalized_list = [stemize(remove_numerical_values_text(replace_punctuation_text(word))) for word in lookup_list]\n",
        "\n",
        "        # Convert to set.\n",
        "        lookup_list.clear()\n",
        "        lookup_list.extend(set(normalized_list))\n",
        "\n",
        "    for lookup_list_name in lookup_ventilation_lists:\n",
        "\n",
        "        lookup_list = globals()[lookup_list_name]\n",
        "\n",
        "        # Replace punctuations, remove numerical values, and apply stemming.\n",
        "        normalized_list = [stemize(remove_numerical_values_text(replace_punctuation_text(word))) for word in lookup_list]\n",
        "\n",
        "        # Convert to set.\n",
        "        lookup_list.clear()\n",
        "        lookup_list.extend(set(normalized_list))\n",
        "\n",
        "def group_modules(description, lookup_lists):\n",
        "    \"\"\"\n",
        "    Classifies a description into a module type based on predefined lookup lists.\n",
        "\n",
        "    This function checks if the given description contains any words from a set of lookup lists\n",
        "    and assigns a module type accordingly. The lookup lists are accessed dynamically using their\n",
        "    names, and the corresponding module type is identified using a separate mapping.\n",
        "\n",
        "    Args:\n",
        "        description (str): The text description to be classified.\n",
        "        lookup_lists (list of str): A list of lookup list names to check against the description.\n",
        "\n",
        "    Returns:\n",
        "        str: The identified module type name if a match is found; otherwise, 'Other'.\n",
        "    \"\"\"\n",
        "    # Convert the description to lowercase to ensure case-insensitive matching.\n",
        "    description = description.lower()\n",
        "\n",
        "    # Iterate through each lookup list name in the provided list.\n",
        "    for lookup_list_name in lookup_lists:\n",
        "        # Access the actual lookup list using the global variable name.\n",
        "        lookup_list = globals().get(lookup_list_name)\n",
        "\n",
        "        # Check if any word in the lookup list is present in the description.\n",
        "        for word in lookup_list:\n",
        "            if word.lower() in description:\n",
        "                # Map the lookup list name to its corresponding module type.\n",
        "                type_name = module_names[lookup_list_name.split('_')[1]]\n",
        "                return type_name  # Return the matched module type.\n",
        "\n",
        "    # Return 'Other' if no matching word is found in the lookup lists.\n",
        "    return 'Other'\n",
        "\n",
        "def rules_to_coordinates(rules):\n",
        "    \"\"\"\n",
        "    Convert association rules to coordinates.\n",
        "\n",
        "    Parameters:\n",
        "    - rules (pd.DataFrame): DataFrame containing association rules with 'antecedents', 'consequents', and other columns.\n",
        "\n",
        "    Returns:\n",
        "    - pd.DataFrame: DataFrame with 'antecedent', 'consequent', and 'rule' columns representing coordinates.\n",
        "    \"\"\"\n",
        "\n",
        "    # Extract the first item from antecedents and consequents.\n",
        "    rules['antecedent'] = rules['antecedents'].apply(lambda antecedent: list(antecedent)[0])\n",
        "    rules['consequent'] = rules['consequents'].apply(lambda consequent: list(consequent)[0])\n",
        "\n",
        "    # Assign rule index to a new column 'rule'.\n",
        "    rules['rule'] = rules.index\n",
        "\n",
        "    # Select relevant columns for coordinates.\n",
        "    coords = rules[['antecedent', 'consequent', 'rule']]\n",
        "\n",
        "    return coords\n",
        "\n",
        "def contains_word_regex(sentence, target_word):\n",
        "    \"\"\"\n",
        "    Check if a given word is present in a sentence using regex.\n",
        "\n",
        "    Parameters:\n",
        "    - sentence (str): The input sentence to check.\n",
        "    - target_word (str): The word to look for in the sentence.\n",
        "\n",
        "    Returns:\n",
        "    - bool: True if the word is found, False otherwise.\n",
        "    \"\"\"\n",
        "    # Construct a regex pattern to match the whole word, case-insensitive.\n",
        "    pattern = r'\\b' + re.escape(target_word) + r'\\b'\n",
        "\n",
        "    # Use re.search to find the pattern in the sentence.\n",
        "    match = re.search(pattern, sentence, flags=re.IGNORECASE)\n",
        "\n",
        "    # Return True if a match is found, False otherwise.\n",
        "    return bool(match)\n",
        "\n",
        "def convert_str_to_set(itemset_str):\n",
        "    \"\"\"\n",
        "    Converts a comma-separated string into a set of items.\n",
        "\n",
        "    This function takes a string containing items separated by commas,\n",
        "    strips any surrounding whitespace from each item, and returns a set\n",
        "    containing the unique items. This is useful for converting textual\n",
        "    representations of itemsets into Python set objects for further analysis.\n",
        "\n",
        "    Args:\n",
        "        itemset_str (str): A string of items separated by commas.\n",
        "\n",
        "    Returns:\n",
        "        set: A set containing the stripped items from the input string.\n",
        "    \"\"\"\n",
        "    # Split the string by commas and strip whitespace from each item.\n",
        "    items = [item.strip() for item in itemset_str.split(',')]\n",
        "\n",
        "    # Convert the list of items to a set to ensure uniqueness.\n",
        "    return set(items)\n",
        "\n",
        "def get_season(month):\n",
        "    \"\"\"\n",
        "    Determine the season corresponding to a given month.\n",
        "    Args:\n",
        "        month (int): The month as an integer (1 for January, 2 for February, ..., 12 for December).\n",
        "    Returns:\n",
        "        str: The season name corresponding to the input month:\n",
        "            - 'Winter' for December (12), January (1), February (2)\n",
        "            - 'Spring' for March (3), April (4), May (5)\n",
        "            - 'Summer' for June (6), July (7), August (8)\n",
        "            - 'Fall' for September (9), October (10), November (11)\n",
        "    Raises:\n",
        "        ValueError: If the input month is not between 1 and 12.\n",
        "    \"\"\"\n",
        "    if month in [12, 1, 2]:\n",
        "        return 'Winter'\n",
        "    elif month in [3, 4, 5]:\n",
        "        return 'Spring'\n",
        "    elif month in [6, 7, 8]:\n",
        "        return 'Summer'\n",
        "    elif month in [9, 10, 11]:\n",
        "        return 'Fall'\n",
        "    else:\n",
        "        raise ValueError(\"Month must be an integer between 1 and 12.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "in6RcmHhsRMk"
      },
      "source": [
        "## Define Lookup Tables"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7sbfEVkPGB0P"
      },
      "outputs": [],
      "source": [
        "lookup_ventilation = [\n",
        "    \"lbk\",\n",
        "    \"luchtbehandeling\",\n",
        "    \"luchtbehandelen\",\n",
        "    \"luchbehandeling\",\n",
        "    \"luchtbehandeling\",\n",
        "    \"ventilatiesysteem\",\n",
        "    \"ventilatie\",\n",
        "    \"luchtklep\",\n",
        "    \"stoombevochtiger\",\n",
        "    \"stoombevochtiging\",\n",
        "    \"bevochtiger\",\n",
        "    \"toevoerventilator\",\n",
        "    \"afvoerventilator\",\n",
        "    \"ventilatormotor\",\n",
        "    \"dakventilatoren\",\n",
        "    \"Dakventilator\",\n",
        "    \"toevoer ventilator\",\n",
        "    \"dak ventilatoren\",\n",
        "    \"Afzuiventilator\",\n",
        "    \"wiel\",\n",
        "    \"afzuigvent\",\n",
        "    \"V-snaren\",\n",
        "    \"V snaren\",\n",
        "    \"filters\",\n",
        "    \"snaarbreuk\",\n",
        "    \"condensafvoer\",\n",
        "    \"condensor\",\n",
        "    \"filter\",\n",
        "    \"Luchtbeh\",\n",
        "    \"LBH\",\n",
        "    \"vorst\",\n",
        "    \"verwarmingsbatterij\",\n",
        "    \"stoomvochtiger\",\n",
        "    \"luchtdebiet\",\n",
        "    \"luchtzakken\",\n",
        "    \"fancoil\",\n",
        "    \"fan coil\",\n",
        "    \"fan-coil\"\n",
        "]\n",
        "\n",
        "lookup_cooling = [\n",
        "    \"airco\",\n",
        "    \"drogekoeler\",\n",
        "    \"koelmachine\",\n",
        "    \"koeling\",\n",
        "    \"KM\",\n",
        "    \"koelunit\",\n",
        "    \"koelinstallatie\",\n",
        "    \"topcooling\",\n",
        "    \"gkw\",\n",
        "    \"koeltoren\",\n",
        "    \"chillers\",\n",
        "    \"dry-cooler\",\n",
        "    \"drycooler\",\n",
        "    \"koelplafond\",\n",
        "    \"koelklep\",\n",
        "    \"koelwaterpomp\",\n",
        "    \"carrier\",\n",
        "    \"koel-unit\",\n",
        "    \"DX koeler\",\n",
        "    \"DX-koeler\",\n",
        "    \"draaikoeler\",\n",
        "    \"condensventilator\",\n",
        "    \"chiller\",\n",
        "    \"koelventilator\",\n",
        "    \"Condensorventilator\",\n",
        "    \"condensor\",\n",
        "    \"split unit\",\n",
        "    \"split-unit\",\n",
        "    \"splitunit\"\n",
        "]\n",
        "\n",
        "lookup_heating = [\n",
        "    \"ketel\",\n",
        "    \"CV\",\n",
        "    \"c.v.\",\n",
        "    \"c.v\",\n",
        "    \"Kachel\",\n",
        "    \"verwarming\",\n",
        "    \"radiatoren\",\n",
        "    \"radiator\",\n",
        "    \"vloerverwarming\",\n",
        "    \"rookgasventilator\"\n",
        "]\n",
        "\n",
        "lookup_fireSafety = [\n",
        "    \"brandmeld\",\n",
        "    \"rookmelder\",\n",
        "    \"branddeur\",\n",
        "    \"brandklep\",\n",
        "    \"brandweer\",\n",
        "    \"brandhaspel\",\n",
        "    \"brandblus\",\n",
        "    \"ontruiming\",\n",
        "    \"Brandventilatoren\",\n",
        "    \"trappenhuis\"\n",
        "]\n",
        "\n",
        "lookup_entrance = [\n",
        "    \"toegangspoort\",\n",
        "    \"tourniqet\",\n",
        "    \"paslezer\",\"tourniquet\",\n",
        "    \"tourniqeut\",\n",
        "    \"tourniqut\",\n",
        "    \"toegang\",\n",
        "    \"Toerniqet\",\n",
        "    \"tourniget\",\n",
        "    \"garagedeur\",\n",
        "    \"schuifhek\",\n",
        "    \"slagboom\"\n",
        "]\n",
        "\n",
        "lookup_shading = [\n",
        "    \"zonwering\",\n",
        "    \"zonneschermen\",\n",
        "    \"zonnewering\"\n",
        "]\n",
        "\n",
        "lookup_sanitary = [\n",
        "    \"toilet\",\n",
        "    \"WC\",\n",
        "    \"urinoir\",\n",
        "    \"wastafel\",\n",
        "    \"sanitair\",\n",
        "    \"wasbak\",\n",
        "    \"afvoer\"\n",
        "]\n",
        "\n",
        "lookup_heatPump = [\n",
        "    \"warmtepomp\",\n",
        "    \"warmte pomp\",\n",
        "    \"WKO\"\n",
        "]\n",
        "\n",
        "lookup_lighting = [\n",
        "    \"verlichting\",\n",
        "    \"lamp\",\n",
        "    \"Tl-buis\",\n",
        "    \"licht\",\n",
        "    \"tlarmatuur\",\n",
        "    \"tlbuizen\",\n",
        "    \"armatuur\",\n",
        "    \"armaturen\",\n",
        "    \"armanturen\"\n",
        "]\n",
        "\n",
        "lookup_elevator = [\n",
        "    \"Lift\"\n",
        "]\n",
        "\n",
        "lookup_wkk = [\n",
        "    \"WKK\"\n",
        "]\n",
        "\n",
        "lookup_bms = [\n",
        "    \"GBS\",\n",
        "    \"Data\",\n",
        "    \"logger\",\n",
        "    \"lon\",\n",
        "    \"BMC\",\n",
        "    \"priva\",\n",
        "    \"software\",\n",
        "    \"regeling\",\n",
        "    \"hardware\",\n",
        "    \"regelkast\",\n",
        "    \"RK 1\",\n",
        "    \"rk5\",\n",
        "    \"rk2\",\n",
        "    \"RK5\",\n",
        "    \"rk3\",\n",
        "    \"Rk 3\",\n",
        "    \"rk4\",\n",
        "    \"rk 4\",\n",
        "    \"rk1\",\n",
        "    \"rk2\",\n",
        "    \"rk6\",\n",
        "    \"rk 6\",\n",
        "    \"RK7\",\n",
        "    \"sensor\",\n",
        "    \"regelaar\",\n",
        "    \"opnemer\",\n",
        "    \"thermostaat\",\n",
        "    \"meting\",\n",
        "    \"onderstation\",\n",
        "    \"Kloktijden\",\n",
        "    \"kastventilatoren\",\n",
        "    \"kastventilatoren\",\n",
        "    \"kastventilator\",\n",
        "    \"Kastventilator\"\n",
        "]\n",
        "\n",
        "lookup_waterDistribution = [\n",
        "    \"regelklep\",\n",
        "    \"driewegklep\",\n",
        "    \"TSA\",\n",
        "    \"Warmtewisselaar\",\n",
        "    \"pomp\",\n",
        "    \"hydrofoor\",\n",
        "    \"expansie\",\n",
        "    \"drukvat\",\n",
        "    \"waterleiding\",\n",
        "    \"3wegklep\",\n",
        "    \"transportnet\"\n",
        "]\n",
        "\n",
        "lookup_office = [\n",
        "    \"werkvoorbereiding\",\n",
        "    \"Contractbeheerder\",\n",
        "    \"materiaal\",\n",
        "    \"Onderaanneming\",\n",
        "    \"Werkvoorbereider\",\n",
        "    \"kantoor\",\n",
        "    \"Materiaalbon\",\n",
        "    \"Urenbon\",\n",
        "    \"Contractbegeleiding\",\n",
        "    \"Contractbeheer\",\n",
        "    \"overleg\",\n",
        "    \"Meet-enregeltechniek\",\n",
        "    \"Inlenen\",\n",
        "    \"Weekplanning\",\n",
        "    \"calculatie\",\n",
        "    \"Onderaannemering\",\n",
        "    \"contractmanager\",\n",
        "    \"Contractmanagement\",\n",
        "    \"Onderaannemer\"\n",
        "]\n",
        "\n",
        "lookup_domesticWater = [\n",
        "    \"warm water\",\n",
        "    \"ww\",\n",
        "    \"w.w.\",\n",
        "    \"w.w\",\n",
        "    \"warmtapwater\",\n",
        "    \"warmwater\",\n",
        "    \"boiler\"\n",
        "]\n",
        "\n",
        "lookup_faultRedemption = [\n",
        "    \"Storingsafkoop\",\n",
        "    \"Afkoopstoringen\",\n",
        "    \"Verrekening afkoop\"\n",
        "]\n",
        "\n",
        "lookup_regularMaintenance = [\n",
        "    \"onderhoud\",\n",
        "    \"inspectie\",\n",
        "    \"OH\",\n",
        "    \"OHD\",\n",
        "    \"controle\",\n",
        "    \"Preventief\",\n",
        "    \"testen\",\n",
        "    \"Bedrijfvoering\",\n",
        "    \"Bedrijfsvoering\"\n",
        "]\n",
        "\n",
        "lookup_complaints = [\n",
        "    \"klachten\",\n",
        "    \"klacht\",\n",
        "    \"te warm\",\n",
        "    \"tekoud\",\n",
        "    \"luchtvochtigheid\",\n",
        "    \"klimaatbeheersing\",\n",
        "    \"klimaat\",\n",
        "    \"teheet\",\n",
        "    \"benauwd\",\n",
        "    \"tocht\",\n",
        "    \"R.V.\",\n",
        "    \"ergkoud\",\n",
        "    \"RV telaag\",\n",
        "    \"lekkage\",\n",
        "    \"Ruimtevochtigheid\",\n",
        "    \"erg warm\",\n",
        "    \"erg koud\"\n",
        "]\n",
        "\n",
        "module_names = {\n",
        "    \"complaints\": \"Complaints\",\n",
        "    \"regularMaintenance\": \"Regular Maintenance\",\n",
        "    \"faultRedemption\": \"Fault Redemption\",\n",
        "    \"domesticWater\": \"Domestic Water\",\n",
        "    \"office\": \"Office\",\n",
        "    \"waterDistribution\": \"Water Distribution\",\n",
        "    \"bms\": \"BMS\",\n",
        "    \"wkk\": \"WKK\",\n",
        "    \"elevator\": \"Elevator\",\n",
        "    \"lighting\": \"Lighting\",\n",
        "    \"heatPump\": \"Heat Pump\",\n",
        "    \"sanitary\": \"Sanitary\",\n",
        "    \"fireSafety\": \"Fire Safety\",\n",
        "    \"shading\": \"Shading\",\n",
        "    \"entrance\": \"Entrance\",\n",
        "    \"ventilation\": \"Ventilation\",\n",
        "    \"heating\": \"Heating\",\n",
        "    \"cooling\": \"Cooling\",\n",
        "}\n",
        "\n",
        "normalize_lookups()\n",
        "\n",
        "# Get the list of all of the lookups.\n",
        "list_lookups = [var for var in globals() if re.match(r'^lookup_', var)]\n",
        "\n",
        "# Sort the lookups descendingly based on their lengths.\n",
        "list_lookups = sorted(list_lookups, key=lambda x: len(globals()[x]), reverse=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MFYjHZpCGigl"
      },
      "source": [
        "# Data Reading"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7aqqRwUyGqOe"
      },
      "outputs": [],
      "source": [
        "# Define the path to the dataset here.\n",
        "path = 'dataset_exel_file.xlsx'\n",
        "\n",
        "df_so = pd.read_excel(path)\n",
        "df_so.reset_index(drop=True, inplace=True)\n",
        "\n",
        "# Define the columns that you want to extract from the dataset here.\n",
        "columns_to_select = ['Description', 'Problem Text', 'Causation Text', 'Solution Text', 'Start Date']\n",
        "df_so = df_so[columns_to_select]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TZRZ0oZgG3gB"
      },
      "source": [
        "# Exploratory Data Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VDJL0LMGIC6r"
      },
      "source": [
        "## Data Cleaning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vANUeU_EIGs0"
      },
      "outputs": [],
      "source": [
        "textual_columns = ['Description', 'Problem Text', 'Causation Text', 'Solution Text']\n",
        "\n",
        "# Correct cell value types.\n",
        "df_so_cleaned = correct_types(df_so.copy(), textual_columns)\n",
        "\n",
        "# Replace punctuations.\n",
        "df_so_cleaned = replace_punctuation(df_so_cleaned, textual_columns)\n",
        "\n",
        "# Remove numerical values.\n",
        "df_so_cleaned = remove_numerical_values(df_so_cleaned, textual_columns)\n",
        "\n",
        "# Remove Dutch stop words.\n",
        "stop_words = set(stopwords.words('dutch'))\n",
        "stop_words.add('via')\n",
        "stop_words.remove('niet')\n",
        "df_so_cleaned[textual_columns] = df_so_cleaned[textual_columns].applymap(lambda x: ' '.join([word for word in x.split() if word.lower() not in stop_words]))\n",
        "\n",
        "# Stemize the text.\n",
        "df_so_cleaned[textual_columns] = df_so_cleaned[textual_columns].applymap(stemize)\n",
        "\n",
        "df_so_cleaned.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j9xAymHkIRnv"
      },
      "source": [
        "## Data Annotation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kjkC1P61IUjs"
      },
      "outputs": [],
      "source": [
        "# Apply the 'group_modules' function to categorize each service order (SO) description into a main subsystem (module).\n",
        "# The 'target' column will store the assigned subsystem label for each service order based on the lookup tables.\n",
        "df_so_cleaned['target'] = df_so_cleaned['Description'].apply(lambda description: group_modules(str(description), list_lookups))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "07xNgpNqLQzz"
      },
      "outputs": [],
      "source": [
        "# Calculate the counts for each target category.\n",
        "target_counts = df_so_cleaned[\"target\"].value_counts()\n",
        "\n",
        "# Calculate the total number of samples.\n",
        "total_samples = len(df_so_cleaned)\n",
        "\n",
        "# Plotting the bar chart.\n",
        "plt.figure(figsize=(10, 6))\n",
        "bars = plt.bar(target_counts.index, target_counts, color=plt.cm.viridis(np.linspace(0, 1, len(target_counts))))\n",
        "plt.title(\"Frequency of Target Categories\")\n",
        "plt.xlabel(\"Target Categories\")\n",
        "plt.ylabel(\"Frequency\")\n",
        "plt.xticks(rotation=45, ha='right')\n",
        "plt.yticks(minor=True)\n",
        "\n",
        "# Add counts on top of the bars.\n",
        "for bar, count in zip(bars, target_counts):\n",
        "    plt.text(bar.get_x() + bar.get_width() / 2, count + 1, str(count), ha='center', va='bottom')\n",
        "\n",
        "# Calculate and format percentages for legend labels.\n",
        "percentages = [(f\"{category} = {count/total_samples*100:.3f}%\") for category, count in target_counts.items()]\n",
        "\n",
        "# Add legend with custom labels.\n",
        "plt.legend(bars, percentages, loc='upper right', bbox_to_anchor=(1, 1), ncol=2, fontsize='small')\n",
        "\n",
        "plt.savefig('./figures/Annotation (Classification)/module_categories.png', bbox_inches = 'tight')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o18HZVcFGRF4"
      },
      "source": [
        "## Keep Original Description Column (Description)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1QAVZ-q3GaT_"
      },
      "outputs": [],
      "source": [
        "df_so_cleaned['original_sentence'] = df_so['Description']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yi9-Fr-CunVf"
      },
      "source": [
        "## Get Cooling Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5hjbTZKluu_9"
      },
      "outputs": [],
      "source": [
        "df_so_cleaned_cooling = df_so_cleaned[df_so_cleaned['target'] == 'Cooling']\n",
        "\n",
        "df_so_cleaned_cooling.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tg7AC71J7dxV"
      },
      "outputs": [],
      "source": [
        "df_so_cleaned_cooling.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AzaIeJSxRzNO"
      },
      "source": [
        "## Visualization on Description"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xmOFCaXpR3jt"
      },
      "outputs": [],
      "source": [
        "# Extract the 'Description' (service order description) for all service orders categorized under 'Cooling'.\n",
        "# Store them as a list for further processing in text analysis.\n",
        "corpus_cooling = df_so_cleaned_cooling[df_so_cleaned_cooling['target'] == 'Cooling']['Description'].tolist()\n",
        "\n",
        "# Initialize a CountVectorizer to convert the heating-related descriptions into a document-term matrix (DTM).\n",
        "# We are specifying that we only want to extract unigrams (individual words) with ngram_range=(1, 1).\n",
        "vectorizer_cooling = CountVectorizer(ngram_range=(1, 1))\n",
        "\n",
        "# Fit the vectorizer to the corpus and transform it into a DTM (a matrix of word counts).\n",
        "dtm_cooling = vectorizer_cooling.fit_transform(corpus_cooling)\n",
        "\n",
        "# Retrieve the feature names (terms) from the vectorizer (i.e., the words in the DTM).\n",
        "terms_cooling = vectorizer_cooling.get_feature_names_out()\n",
        "\n",
        "# Convert the DTM into a DataFrame where each column corresponds to a word and each row represents a service order.\n",
        "dtm_cooling = pd.DataFrame(dtm_cooling.toarray(), columns=terms_cooling)\n",
        "\n",
        "# Calculate the frequency of each term by summing over the columns of the DTM.\n",
        "term_frequencies_cooling = dtm_cooling.sum()\n",
        "\n",
        "# Generate a word cloud based on the term frequencies for the heating-related service orders.\n",
        "wordcloud = WordCloud(width=1000, height=600, background_color='white').generate_from_frequencies(term_frequencies_cooling)\n",
        "\n",
        "# Display the word cloud.\n",
        "plt.title('Cooling Word Cloud')\n",
        "plt.imshow(wordcloud, interpolation='bilinear')\n",
        "plt.axis('off')\n",
        "\n",
        "# Save the generated word cloud image to the specified directory. Change this in case of need!\n",
        "plt.savefig('./figures/EDA/cooling_terms_word_cloud.png')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ajp0qWDCP9YK"
      },
      "source": [
        "# ARM"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ucV1vvVYR7S"
      },
      "source": [
        "## Obtain Pruned Association Rules"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UO-FL6SmjmvI"
      },
      "outputs": [],
      "source": [
        "# Step 1: Split each service order description in the 'Description' column into individual words for the ventilation component.\n",
        "transactions_cooling = df_so_cleaned_cooling[df_so_cleaned_cooling['target'] == 'Cooling']['Description'].apply(lambda t: t.split(' '))\n",
        "\n",
        "# Convert the series of lists into a list of transactions (each transaction is a list of words).\n",
        "transactions_cooling = list(transactions_cooling)\n",
        "\n",
        "# Step 2: Apply one-hot encoding to the transactions using the TransactionEncoder.\n",
        "encoder_cooling = TransactionEncoder().fit(transactions_cooling)\n",
        "onehot_cooling = encoder_cooling.transform(transactions_cooling)\n",
        "onehot_cooling = pd.DataFrame(onehot_cooling, columns=encoder_cooling.columns_)\n",
        "\n",
        "# Step 3: Define the minimum support threshold for finding frequent itemsets.\n",
        "'''\n",
        "Define the minimum number of occurrences for each itemset.\n",
        "This indirectly acts as a hyperparamether, so tune it as you pefer.\n",
        "Increase the value of \"n\" if you want to find more frequent itemsets.\n",
        "Decrease the value of \"n\" if you want to find less frequent itemsets.\n",
        "'''\n",
        "n = 20\n",
        "minimum_support = n/len(onehot_cooling) # Calculate the minimum support as the ratio of 'n' to the total number of transactions.\n",
        "\n",
        "# Step 4: Run the Apriori algorithm to identify frequent itemsets based on the minimum support threshold.\n",
        "frequent_itemsets_cooling = apriori(onehot_cooling,\n",
        "                            min_support =  minimum_support,\n",
        "                            use_colnames = True,\n",
        "                            verbose = 1)\n",
        "\n",
        "# Step 5: Generate association rules from the frequent itemsets using the 'lift' metric with a minimum threshold of 1.\n",
        "rules_cooling = association_rules(frequent_itemsets_cooling, metric = 'lift', min_threshold = 1)\n",
        "\n",
        "# Print summary statistics of the process.\n",
        "print(f'Number of transactions: {len(onehot_cooling)}')\n",
        "print(f'Minimum number of occurrence for each itemset: {n}')\n",
        "print(f'Minimum support threshold for itemsets: {minimum_support}')\n",
        "print(f'number of frequent itemsets: {len(frequent_itemsets_cooling)}')\n",
        "print(f'Number of rules: {len(rules_cooling)}\\n')\n",
        "\n",
        "# Step 6: Replace frozen sets with strings for easier readability in the rules.\n",
        "rules_cooling['antecedents'] = rules_cooling['antecedents'].apply(lambda a: ','.join(list(a)))\n",
        "rules_cooling['consequents'] = rules_cooling['consequents'].apply(lambda a: ','.join(list(a)))\n",
        "\n",
        "# Display the final association rules.\n",
        "rules_cooling"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0RcmlB7A0SEf"
      },
      "source": [
        "## Heatmap of support"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K0x0OQ-JkuHq"
      },
      "outputs": [],
      "source": [
        "# Step 1: Transform the association rules data into a pivot table for the heatmap.\n",
        "# The pivot table uses 'consequents' as the row index and 'antecedents' as the columns, with 'support' values as the matrix.\n",
        "pivot = rules_cooling.pivot(index='consequents', columns='antecedents', values='support')\n",
        "\n",
        "# Step 2: Create a mask for the upper triangle of the heatmap to avoid duplicate values being displayed.\n",
        "mask = np.triu(np.ones_like(pivot, dtype=bool))\n",
        "\n",
        "# Step 3: Set the font scale for the heatmap.\n",
        "sns.set(font_scale=0.9)\n",
        "\n",
        "# Step 4: Create the heatmap figure.\n",
        "plt.figure(figsize=(15, 8))\n",
        "\n",
        "# Step 5: Generate the heatmap using seaborn.\n",
        "heatmap = sns.heatmap(pivot,  # The pivot table containing support values.\n",
        "                      cmap='coolwarm',  # Use the 'coolwarm' color palette.\n",
        "                      mask=mask,  # Apply the mask to hide the upper triangle of the heatmap.\n",
        "                      cbar=True,  # Show the color bar indicating the support values.\n",
        "                      linewidths=0.5,  # Set the width of the grid lines.\n",
        "                      linecolor='black',  # Set the color of the grid lines.\n",
        "                      annot=True,  # Annotate each cell with the support value.\n",
        "                      fmt='.2f')  # Format the annotations to two decimal places.\n",
        "\n",
        "# Step 6: Customize the heatmap.\n",
        "plt.xticks(rotation=45) # Rotate the x-axis labels 45 degrees for better readability.\n",
        "plt.title(f'Cooling Rules Heatmap: {len(onehot_cooling)} transactions, occurrence min-value of {n}, support threshold of {minimum_support:.2f}') # Set the heatmap title.\n",
        "plt.yticks(rotation=0) # Keep the y-axis labels horizontal.\n",
        "\n",
        "# Step 7: Save the heatmap as an image file. Change the output path in case of need!\n",
        "plt.savefig('./figures/Association Rule Mining/cooling/support_heatmap.png')\n",
        "\n",
        "# Step 8: Display the heatmap.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "estESIcU0byi"
      },
      "source": [
        "## Heatmap of leverage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q-C8mH070h5Y"
      },
      "outputs": [],
      "source": [
        "# Step 1: Transform the association rules data into a pivot table for the heatmap.\n",
        "# The pivot table uses 'consequents' as the row index and 'antecedents' as the columns, with 'leverage' values as the matrix.\n",
        "pivot = rules_cooling.pivot(index='consequents', columns='antecedents', values='leverage')\n",
        "\n",
        "# Step 2: Create a mask for the upper triangle of the heatmap to avoid duplicate values being displayed.\n",
        "mask = np.triu(np.ones_like(pivot, dtype=bool))\n",
        "\n",
        "# Step 3: Set the font scale for the heatmap.\n",
        "sns.set(font_scale=0.9)\n",
        "\n",
        "# Step 4: Create the heatmap figure.\n",
        "plt.figure(figsize=(15, 8))\n",
        "\n",
        "# Step 5: Generate the heatmap using seaborn.\n",
        "heatmap = sns.heatmap(pivot,  # The pivot table containing support values.\n",
        "                      cmap='coolwarm',  # Use the 'coolwarm' color palette.\n",
        "                      mask=mask,  # Apply the mask to hide the upper triangle of the heatmap.\n",
        "                      cbar=True,  # Show the color bar indicating the support values.\n",
        "                      linewidths=0.5,  # Set the width of the grid lines.\n",
        "                      linecolor='black',  # Set the color of the grid lines.\n",
        "                      annot=True,  # Annotate each cell with the support value.\n",
        "                      fmt='.2f')  # Format the annotations to two decimal places.\n",
        "\n",
        "# Step 6: Customize the heatmap.\n",
        "plt.xticks(rotation=45) # Rotate the x-axis labels 45 degrees for better readability.\n",
        "plt.title(f'Cooling Rules Heatmap: {len(onehot_cooling)} transactions, occurrence min-value of {n}, support threshold of {minimum_support:.2f}') # Set the heatmap title.\n",
        "plt.yticks(rotation=0) # Keep the y-axis labels horizontal.\n",
        "\n",
        "# Step 7: Save the heatmap as an image file. Change the output path in case of need!\n",
        "plt.savefig('./figures/Association Rule Mining/cooling/leverage_heatmap.png')\n",
        "\n",
        "# Step 8: Display the heatmap.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sEJi49ya0zFE"
      },
      "source": [
        "## Heatmap of conviction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RmW1dUaT04PH"
      },
      "outputs": [],
      "source": [
        "# Step 1: Transform the association rules data into a pivot table for the heatmap.\n",
        "# The pivot table uses 'consequents' as the row index and 'antecedents' as the columns, with 'conviction' values as the matrix.\n",
        "pivot = rules_cooling.pivot(index='consequents', columns='antecedents', values='conviction')\n",
        "\n",
        "# Step 2: Create a mask for the upper triangle of the heatmap to avoid duplicate values being displayed.\n",
        "mask = np.triu(np.ones_like(pivot, dtype=bool))\n",
        "\n",
        "# Step 3: Set the font scale for the heatmap.\n",
        "sns.set(font_scale=0.9)\n",
        "\n",
        "# Step 4: Create the heatmap figure.\n",
        "plt.figure(figsize=(15, 8))\n",
        "\n",
        "# Step 5: Generate the heatmap using seaborn.\n",
        "heatmap = sns.heatmap(pivot,  # The pivot table containing support values.\n",
        "                      cmap='coolwarm',  # Use the 'coolwarm' color palette.\n",
        "                      mask=mask,  # Apply the mask to hide the upper triangle of the heatmap.\n",
        "                      cbar=True,  # Show the color bar indicating the support values.\n",
        "                      linewidths=0.5,  # Set the width of the grid lines.\n",
        "                      linecolor='black',  # Set the color of the grid lines.\n",
        "                      annot=True,  # Annotate each cell with the support value.\n",
        "                      fmt='.2f')  # Format the annotations to two decimal places.\n",
        "\n",
        "# Step 6: Customize the heatmap.\n",
        "plt.xticks(rotation=45) # Rotate the x-axis labels 45 degrees for better readability.\n",
        "plt.title(f'Cooling Rules Heatmap: {len(onehot_cooling)} transactions, occurrence min-value of {n}, support threshold of {minimum_support:.2f}') # Set the heatmap title.\n",
        "plt.yticks(rotation=0) # Keep the y-axis labels horizontal.\n",
        "\n",
        "# Step 7: Save the heatmap as an image file. Change the output path in case of need!\n",
        "plt.savefig('./figures/Association Rule Mining/cooling/conviction_heatmap.png')\n",
        "\n",
        "# Step 8: Display the heatmap.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ObBCjFgh1TWG"
      },
      "source": [
        "## Frequency table"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z_Szvvds7Tbb"
      },
      "source": [
        "### Prepare items"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "01pvJ5SO7X-4"
      },
      "outputs": [],
      "source": [
        "# Step 1: Combine 'antecedents' and 'consequents' of each rule into a single 'itemset' string for easier handling.\n",
        "rules_cooling['itemset'] = rules_cooling.apply(lambda row: f\"{row['antecedents']},{row['consequents']}\", axis=1)\n",
        "\n",
        "# Step 2: Calculate the absolute count of each itemset's occurrence by multiplying the support value by the total number of transactions.\n",
        "# The number of transactions is determined by the length of the onehot_cooling matrix.\n",
        "rules_cooling['count'] = rules_cooling.apply(lambda row: int(row['support'] * len(onehot_cooling)), axis=1)\n",
        "\n",
        "# Step 3: Convert the 'itemset' string into a Python set to facilitate further itemset manipulations.\n",
        "rules_cooling['itemset'] = rules_cooling['itemset'].apply(convert_str_to_set)\n",
        "\n",
        "# Step 4: Format the support values to three decimal places and store them in a new column 'rounded_support'.\n",
        "rules_cooling['rounded_support'] = rules_cooling.apply(lambda row: f\"{row['support']:.3f}\", axis=1)\n",
        "\n",
        "# Step 5: Calculate the percentage occurrence of each itemset and format the result with two decimal places and a '%' symbol.\n",
        "rules_cooling['percentage'] = rules_cooling.apply(lambda row: f\"{row['count'] * 100 / len(onehot_cooling):.2f}%\", axis=1)\n",
        "\n",
        "# Step 6: Round the conviction values to three decimal places and store them in a new column 'rounded_conviction' for easier reading.\n",
        "rules_cooling['rounded_conviction'] = rules_cooling.apply(lambda row: f\"{row['conviction']:.3f}\", axis=1)\n",
        "\n",
        "# Step 7: Sort the DataFrame based on conviction values in descending order to prioritize rules with higher conviction.\n",
        "rules_cooling.sort_values(by='conviction', ascending=False, inplace=True)\n",
        "\n",
        "# Step 8: Remove duplicate itemsets based on the 'itemset' column, keeping only the first occurrence.\n",
        "rules_cooling.drop_duplicates(subset=['itemset'], keep='first', inplace=True)\n",
        "\n",
        "# Step 9: Reset the index of the DataFrame to maintain consistency after sorting and dropping duplicates.\n",
        "rules_cooling.reset_index(inplace=True)\n",
        "\n",
        "# Output the final processed DataFrame.\n",
        "rules_cooling"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LQ3y9Zbabahx"
      },
      "source": [
        "### Show Heating Rules"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PhqSvjCacYtb"
      },
      "outputs": [],
      "source": [
        "# Sort values by 'support' column.\n",
        "rules_cooling.sort_values(by='support', ascending=False, inplace=True)\n",
        "\n",
        "# Add the index as a column.\n",
        "rules_cooling.reset_index(inplace=True)\n",
        "rules_cooling.rename(columns={'index': 'index'}, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xlk43bIcjmvP"
      },
      "outputs": [],
      "source": [
        "# Step 1: Sort the 'rules_cooling' DataFrame in descending order by the 'support' column.\n",
        "rules_cooling.sort_values(by='support', ascending=False, inplace=True)\n",
        "\n",
        "# Step 2: Create a matplotlib subplot with a figure size of 15x5 inches to accommodate the table.\n",
        "fig, ax = plt.subplots(figsize=(15, 5))\n",
        "\n",
        "# Step 3: Turn off the axis display to hide axis lines and labels.\n",
        "ax.axis('tight')\n",
        "ax.axis('off')\n",
        "\n",
        "# Step 4: Create a table using the sorted 'rules_cooling' DataFrame.\n",
        "# 'cellText' contains the corresponding table data for the specified columns.\n",
        "# 'colLabels' provides the actual column headers for the table.\n",
        "table = ax.table(cellText=rules_cooling[['index', 'itemset', 'rounded_support', 'count', 'percentage', 'rounded_conviction']].values,\n",
        "                 colLabels=rules_cooling[['index', 'itemset', 'support', 'count', 'percentage', 'conviction']].columns,\n",
        "                 cellLoc='center', loc='center', fontsize=15)  # Set table cell font size to 15.\n",
        "\n",
        "# Step 5: Disable auto font size scaling for the table and manually set the font size to 10.\n",
        "table.auto_set_font_size(False)\n",
        "table.set_fontsize(10)\n",
        "\n",
        "# Step 6: Set the title of the table plot, indicating that the rules are sorted by support.\n",
        "plt.title('Cooling Rules - Sorted by Support')\n",
        "\n",
        "# Step 7: Adjust the layout to prevent any overlap and ensure the table fits within the figure dimensions.\n",
        "plt.tight_layout()\n",
        "\n",
        "# Step 8: Save the table as an image file with the specified path and ensure no extra white space around the image.\n",
        "# Change the saving path in case of need!\n",
        "plt.savefig('./figures/Association Rule Mining/cooling/itemsets_frequency_table.png', bbox_inches='tight')\n",
        "\n",
        "# Step 9: Display the table plot.\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a4Nrt2Xx7mVq"
      },
      "outputs": [],
      "source": [
        "# Step 1: Sort the 'rules_cooling' DataFrame in descending order by the 'conviction' column.\n",
        "rules_cooling.sort_values(by='conviction', ascending=False, inplace=True)\n",
        "\n",
        "# Step 2: Create a matplotlib subplot with a figure size of 15x5 inches to accommodate the table.\n",
        "fig, ax = plt.subplots(figsize=(15, 5))\n",
        "\n",
        "# Step 3: Turn off the axis display to hide axis lines and labels.\n",
        "ax.axis('tight')\n",
        "ax.axis('off')\n",
        "\n",
        "# Step 4: Create a table using the sorted 'rules_cooling' DataFrame.\n",
        "# 'cellText' contains the corresponding table data for the specified columns.\n",
        "# 'colLabels' provides the actual column headers for the table.\n",
        "table = ax.table(cellText=rules_cooling[['itemset', 'rounded_support', 'count', 'percentage', 'rounded_conviction']].values,\n",
        "         colLabels=rules_cooling[['itemset', 'support', 'count', 'percentage', 'conviction']].columns,\n",
        "         cellLoc='center', loc='center', fontsize=15)  # Set table cell font size to 15.\n",
        "\n",
        "# Step 5: Set the title of the table plot, indicating that the rules are sorted by conviction.\n",
        "plt.title('Cooling Rules - Sorted by Conviction')\n",
        "\n",
        "# Step 6: Adjust the layout to prevent any overlap and ensure the table fits within the figure dimensions.\n",
        "plt.tight_layout()\n",
        "\n",
        "# Step 7: Display the table plot.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IFLcrdADdUHG"
      },
      "source": [
        "# Frequency Analysis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_82GzgS_fUH7"
      },
      "outputs": [],
      "source": [
        "# Step 1: Extract the year from the 'Start Date' datetime column\n",
        "df_so_cleaned_cooling['Year'] = df_so_cleaned_cooling['Start Date'].dt.year\n",
        "\n",
        "# Step 2: Count the occurrences of each year and sort them in ascending order\n",
        "year_counts = df_so_cleaned_cooling['Year'].value_counts().sort_index()\n",
        "\n",
        "# Step 3: Create a figure for the bar plot with specified dimensions (10x6 inches)\n",
        "plt.figure(figsize=(10, 6))\n",
        "\n",
        "# Step 4: Plot the year counts as a bar chart with green color\n",
        "year_counts.plot(kind='bar', color='green')\n",
        "\n",
        "# Step 5: Set the x-axis label\n",
        "plt.xlabel('Year')\n",
        "\n",
        "# Step 6: Set the y-axis label\n",
        "plt.ylabel('Count of Service Orders')\n",
        "\n",
        "# Step 7: Set the title of the plot to indicate what is being displayed\n",
        "plt.title('Distribution of Service Orders for Cooling Component Across Years')\n",
        "\n",
        "# Step 8: Rotate x-axis tick labels to 0 degrees for better visibility\n",
        "plt.xticks(rotation=0)\n",
        "\n",
        "# Step 9: Save the plot as a PNG file with specified path\n",
        "plt.savefig('./figures/Frequency/cooling/yearly_distribution.png')\n",
        "\n",
        "# Step 10: Display the plot\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r3dpXLn_jB5U"
      },
      "outputs": [],
      "source": [
        "# Step 1: Extract the month from the 'Start Date' datetime column\n",
        "df_so_cleaned_cooling['Month'] = df_so_cleaned_cooling['Start Date'].dt.month\n",
        "\n",
        "# Step 2: Count the occurrences of each month and sort them in ascending order\n",
        "month_counts = df_so_cleaned_cooling['Month'].value_counts().sort_index()\n",
        "\n",
        "# Step 3: Create a figure for the bar plot with specified dimensions (10x6 inches)\n",
        "plt.figure(figsize=(10, 6))\n",
        "\n",
        "# Step 4: Plot the month counts as a bar chart with green color\n",
        "month_counts.plot(kind='bar', color='green')\n",
        "\n",
        "# Step 5: Set the x-axis label to indicate that it represents months\n",
        "plt.xlabel('Month')\n",
        "\n",
        "# Step 6: Set the y-axis label to indicate that it represents the count of service orders\n",
        "plt.ylabel('Count of Service Orders')\n",
        "\n",
        "# Step 7: Set the title of the plot to indicate what is being displayed\n",
        "plt.title('Distribution of Service Orders for Cooling Component Across Months')\n",
        "\n",
        "# Step 8: Set the x-axis ticks to represent month labels and rotate them to 0 degrees for clarity\n",
        "plt.xticks(ticks=range(12), labels=['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'], rotation=0)\n",
        "\n",
        "# Step 9: Save the plot as a PNG file with the specified path\n",
        "plt.savefig('./figures/Frequency/cooling/monthly_distribution.png')\n",
        "\n",
        "# Step 10: Display the plot\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yOu1poSE8n4W"
      },
      "outputs": [],
      "source": [
        "# Step 1: Apply the get_season function to the 'Month' column to create a new 'Season' column.\n",
        "df_so_cleaned_cooling['Season'] = df_so_cleaned_cooling['Month'].apply(get_season)\n",
        "\n",
        "# Step 2: Count the occurrences of each season in the 'Season' column.\n",
        "season_counts = df_so_cleaned_cooling['Season'].value_counts()\n",
        "\n",
        "# Step 3: Create a figure for the pie chart with specified dimensions (8x8 inches).\n",
        "plt.figure(figsize=(8, 8))\n",
        "\n",
        "# Step 4: Plot the season counts as a pie chart with percentage labels.\n",
        "season_counts.plot(kind='pie', autopct='%1.1f%%', colors=['skyblue', 'lightgreen', 'gold', 'salmon'])\n",
        "\n",
        "# Step 5: Remove the y-label for better visual appearance.\n",
        "plt.ylabel('')\n",
        "\n",
        "# Step 6: Set the title of the pie chart to indicate what is being displayed.\n",
        "plt.title('Distribution of Service Orders for Cooling Component Across Seasons')\n",
        "\n",
        "# Step 7: Save the pie chart as a PNG file with the specified path.\n",
        "plt.savefig('./figures/Frequency/cooling/seasonal_distribution.png')\n",
        "\n",
        "# Step 8: Display the pie chart.\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyNaxN4zIBE/y56i2FPQ5R5k",
      "collapsed_sections": [
        "7595UIXVFovr",
        "sUZeu2lmr0ME",
        "xPUgLKbTr4tE",
        "utbh3w3Nr8jG",
        "-jed0F0KsC0F",
        "in6RcmHhsRMk",
        "o18HZVcFGRF4",
        "Yi9-Fr-CunVf",
        "IFLcrdADdUHG"
      ],
      "mount_file_id": "1rN_Q8-4FFdAnik6X9EUnAdGLxmI4CsoK",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}