heating_subsystem_analysis.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Notes:\n",
        "*   <mark>For confidentiality reasons</mark>, no figure is shown as a result of any block execution and the actual dataset naming and input/output addresses are replaced by aliases.\n",
        "*   The lookup tables implemented in the notebook are all taken from \"<mark>Van der Horst, S. A. M. (2019). Economically optimizing maintenance of air handling units (Master’s thesis). Technische Universiteit Eindhoven</mark>\".\n",
        "*   The ARM approach implemented in this notebook was inspired and mainly learned from: <mark>Isaiah Hull. Market Basket Analysis in Python. https://www.datacamp.com/courses/\n",
        "market-basket-analysis-in-python</mark>."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7595UIXVFovr"
      },
      "source": [
        "# Utils"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xPUgLKbTr4tE"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FzcugnMcFtfO"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import string\n",
        "import re\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "from nltk.tokenize import word_tokenize\n",
        "from mlxtend.preprocessing import TransactionEncoder\n",
        "from mlxtend.frequent_patterns import apriori\n",
        "from mlxtend.frequent_patterns import association_rules\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from wordcloud import WordCloud\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "utbh3w3Nr8jG"
      },
      "source": [
        "## Download Packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kAqZ8u70Fxwc"
      },
      "outputs": [],
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-jed0F0KsC0F"
      },
      "source": [
        "## Implement Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r8q14WZMF0rB"
      },
      "outputs": [],
      "source": [
        "def correct_types(df, columns):\n",
        "    \"\"\"\n",
        "    Preprocess the specified columns in a DataFrame by replacing NaN values with an empty string\n",
        "    and converting non-string values to string objects.\n",
        "\n",
        "    Parameters:\n",
        "    - df (pd.DataFrame): The input DataFrame.\n",
        "    - column (str): The column to be preprocessed. Default is 'Description'.\n",
        "\n",
        "    Returns:\n",
        "    - pd.DataFrame: The DataFrame with the specified column preprocessed.\n",
        "    \"\"\"\n",
        "\n",
        "    df[columns] = df[columns].fillna('')\n",
        "    df[columns] = df[columns].astype(str)\n",
        "    return df\n",
        "\n",
        "def replace_punctuation_text(text):\n",
        "    \"\"\"\n",
        "    Replace punctuation in the input text.\n",
        "\n",
        "    Parameters:\n",
        "    - text (str): The input text in which punctuation will be replaced.\n",
        "\n",
        "    Returns:\n",
        "    - str: The input text with punctuation replaced.\n",
        "    \"\"\"\n",
        "    # Replace \"'s\" with empty space.\n",
        "    text = text.replace(\"'s\", '')\n",
        "\n",
        "    # Create a translation table to replace '.' and \"'\" with empty space, and other punctuation with spaces.\n",
        "    translator = str.maketrans({'.': '', \"'\": '', **{p: ' ' for p in string.punctuation if p not in ['.', \"'\"]}})\n",
        "\n",
        "    cleaned_text = text.translate(translator)\n",
        "\n",
        "    return cleaned_text\n",
        "\n",
        "def replace_punctuation(df, columns):\n",
        "    \"\"\"\n",
        "    Replaces specific punctuation in the specified columns of a DataFrame.\n",
        "\n",
        "    This function is designed to clean textual data in the specified columns of a DataFrame by:\n",
        "    - Replacing occurrences of \"'s\" with an empty string.\n",
        "    - Replacing periods (.) and apostrophes (') with an empty string.\n",
        "    - Replacing other punctuation marks with a space.\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): The DataFrame containing the columns to be cleaned.\n",
        "        columns (list or str): The column name(s) of the DataFrame where the replacements should be applied.\n",
        "\n",
        "    Returns:\n",
        "        pd.DataFrame: The DataFrame with the specified columns cleaned of certain punctuation marks.\n",
        "    \"\"\"\n",
        "    # Replace \"'s\" with an empty string in the specified columns.\n",
        "    df[columns] = df[columns].replace(\"'s\", '', regex=True)\n",
        "\n",
        "    # Create a translation table to replace '.' and \"'\" with empty space,\n",
        "    # and other punctuation with spaces.\n",
        "    translator = str.maketrans({\n",
        "        '.': '',                # Remove periods\n",
        "        \"'\": '',                # Remove apostrophes\n",
        "        **{p: ' ' for p in string.punctuation if p not in ['.', \"'\"]}  # Replace other punctuation with space\n",
        "    })\n",
        "\n",
        "    # Apply the translation table to each element in the specified columns.\n",
        "    df[columns] = df[columns].applymap(lambda x: x.translate(translator))\n",
        "\n",
        "    return df\n",
        "\n",
        "def remove_numerical_values_text(text):\n",
        "    \"\"\"\n",
        "    Remove numbers and model-related patterns from the input text.\n",
        "\n",
        "    Parameters:\n",
        "    - text (str): The input text from which numbers and model-related patterns will be removed.\n",
        "\n",
        "    Returns:\n",
        "    - str: The input text with numbers and model-related patterns removed.\n",
        "    \"\"\"\n",
        "    # Remove standalone numbers with optional floating points; such as 154, 99.31.\n",
        "    text = re.sub(r'\\b\\d+(\\.\\d+)?\\b', '', text)\n",
        "\n",
        "    # Remove ordinal numbers; such as 1e, 2ste, 3de, 4e.\n",
        "    text = re.sub(r'\\b\\d+(e|ste|de|e)\\b', '', text)\n",
        "\n",
        "    # Remove numerical quantifiers; such as 1x OH.\n",
        "    text = re.sub(r'\\b\\d+[xX]\\b', '', text)\n",
        "\n",
        "    return text\n",
        "\n",
        "def remove_numerical_values(df, columns):\n",
        "    \"\"\"\n",
        "    Removes various forms of numerical values from the specified columns in a DataFrame.\n",
        "\n",
        "    This function cleans textual data in the specified columns by:\n",
        "    - Removing standalone numbers with optional floating points (e.g., 154, 99.31).\n",
        "    - Removing ordinal numbers commonly used in Dutch (e.g., 1e, 2ste, 3de, 4e).\n",
        "    - Removing numerical quantifiers with 'x' (e.g., 1x, 2X).\n",
        "\n",
        "    Args:\n",
        "        df (pd.DataFrame): The DataFrame containing the columns to be cleaned.\n",
        "        columns (list or str): The column name(s) of the DataFrame where the numerical values should be removed.\n",
        "\n",
        "    Returns:\n",
        "        pd.DataFrame: The DataFrame with the specified columns cleaned of numerical values.\n",
        "    \"\"\"\n",
        "    # Remove standalone numbers with optional floating points (e.g., 154, 99.31).\n",
        "    df[columns] = df[columns].applymap(lambda x: re.sub(r'\\b\\d+(\\.\\d+)?\\b', '', x))\n",
        "\n",
        "    # Remove ordinal numbers (e.g., 1e, 2ste, 3de, 4e).\n",
        "    df[columns] = df[columns].applymap(lambda x: re.sub(r'\\b\\d+(e|ste|de)\\b', '', x))\n",
        "\n",
        "    # Remove numerical quantifiers (e.g., 1x, 2X).\n",
        "    df[columns] = df[columns].applymap(lambda x: re.sub(r'\\b\\d+[xX]\\b', '', x))\n",
        "\n",
        "    return df\n",
        "\n",
        "def stemize(text):\n",
        "    \"\"\"\n",
        "    Perform stemming on the input text using the Dutch Snowball Stemmer.\n",
        "\n",
        "    Parameters:\n",
        "    - text (str): The input text to be stemmed.\n",
        "\n",
        "    Returns:\n",
        "    str: The stemmed text.\n",
        "\n",
        "    Example:\n",
        "    >>> stemize(\"This is an example text for stemming.\")\n",
        "    'thi is an exampl text for stem.'\n",
        "    \"\"\"\n",
        "    stemmer = SnowballStemmer(\"dutch\")\n",
        "    tokens = word_tokenize(text)\n",
        "    text = ' '.join([stemmer.stem(token) for token in tokens])\n",
        "    return text\n",
        "\n",
        "def normalize_lookups():\n",
        "    \"\"\"\n",
        "    Normalize and preprocess the global lookup lists.\n",
        "\n",
        "    This function performs the following operations on each global lookup list:\n",
        "    1. Orders lookup lists by length in descending order.\n",
        "    2. Initializes the Dutch Snowball Stemmer.\n",
        "    3. Replaces punctuations using the `replace_punctuation` function.\n",
        "    4. Removes numerical values using the `remove_numerical_values` function.\n",
        "    5. Applies stemming using the `stemize` function.\n",
        "    6. Converts the resulting list to a set.\n",
        "\n",
        "    Note: The original lookup lists are modified in-place.\n",
        "\n",
        "    Returns:\n",
        "    None\n",
        "    \"\"\"\n",
        "    # Get all lookup lists using regular expression.\n",
        "    lookup_lists = [var for var in globals() if re.match(r'^lookup_', var)]\n",
        "    lookup_ventilation_lists = [var for var in globals() if re.match(r'^sublookup_', var)]\n",
        "\n",
        "    for lookup_list_name in lookup_lists:\n",
        "\n",
        "        lookup_list = globals()[lookup_list_name]\n",
        "\n",
        "        # Replace punctuations, remove numerical values, and apply stemming.\n",
        "        normalized_list = [stemize(remove_numerical_values_text(replace_punctuation_text(word))) for word in lookup_list]\n",
        "\n",
        "        # Convert to set.\n",
        "        lookup_list.clear()\n",
        "        lookup_list.extend(set(normalized_list))\n",
        "\n",
        "    for lookup_list_name in lookup_ventilation_lists:\n",
        "\n",
        "        lookup_list = globals()[lookup_list_name]\n",
        "\n",
        "        # Replace punctuations, remove numerical values, and apply stemming.\n",
        "        normalized_list = [stemize(remove_numerical_values_text(replace_punctuation_text(word))) for word in lookup_list]\n",
        "\n",
        "        # Convert to set.\n",
        "        lookup_list.clear()\n",
        "        lookup_list.extend(set(normalized_list))\n",
        "\n",
        "def group_modules(description, lookup_lists):\n",
        "    \"\"\"\n",
        "    Classifies a description into a module type based on predefined lookup lists.\n",
        "\n",
        "    This function checks if the given description contains any words from a set of lookup lists\n",
        "    and assigns a module type accordingly. The lookup lists are accessed dynamically using their\n",
        "    names, and the corresponding module type is identified using a separate mapping.\n",
        "\n",
        "    Args:\n",
        "        description (str): The text description to be classified.\n",
        "        lookup_lists (list of str): A list of lookup list names to check against the description.\n",
        "\n",
        "    Returns:\n",
        "        str: The identified module type name if a match is found; otherwise, 'Other'.\n",
        "    \"\"\"\n",
        "    # Convert the description to lowercase to ensure case-insensitive matching.\n",
        "    description = description.lower()\n",
        "\n",
        "    # Iterate through each lookup list name in the provided list.\n",
        "    for lookup_list_name in lookup_lists:\n",
        "        # Access the actual lookup list using the global variable name.\n",
        "        lookup_list = globals().get(lookup_list_name)\n",
        "\n",
        "        # Check if any word in the lookup list is present in the description.\n",
        "        for word in lookup_list:\n",
        "            if word.lower() in description:\n",
        "                # Map the lookup list name to its corresponding module type.\n",
        "                type_name = module_names[lookup_list_name.split('_')[1]]\n",
        "                return type_name  # Return the matched module type.\n",
        "\n",
        "    # Return 'Other' if no matching word is found in the lookup lists.\n",
        "    return 'Other'\n",
        "\n",
        "def rules_to_coordinates(rules):\n",
        "    \"\"\"\n",
        "    Convert association rules to coordinates.\n",
        "\n",
        "    Parameters:\n",
        "    - rules (pd.DataFrame): DataFrame containing association rules with 'antecedents', 'consequents', and other columns.\n",
        "\n",
        "    Returns:\n",
        "    - pd.DataFrame: DataFrame with 'antecedent', 'consequent', and 'rule' columns representing coordinates.\n",
        "    \"\"\"\n",
        "\n",
        "    # Extract the first item from antecedents and consequents.\n",
        "    rules['antecedent'] = rules['antecedents'].apply(lambda antecedent: list(antecedent)[0])\n",
        "    rules['consequent'] = rules['consequents'].apply(lambda consequent: list(consequent)[0])\n",
        "\n",
        "    # Assign rule index to a new column 'rule'.\n",
        "    rules['rule'] = rules.index\n",
        "\n",
        "    # Select relevant columns for coordinates.\n",
        "    coords = rules[['antecedent', 'consequent', 'rule']]\n",
        "\n",
        "    return coords\n",
        "\n",
        "def contains_word_regex(sentence, target_word):\n",
        "    \"\"\"\n",
        "    Check if a given word is present in a sentence using regex.\n",
        "\n",
        "    Parameters:\n",
        "    - sentence (str): The input sentence to check.\n",
        "    - target_word (str): The word to look for in the sentence.\n",
        "\n",
        "    Returns:\n",
        "    - bool: True if the word is found, False otherwise.\n",
        "    \"\"\"\n",
        "    # Construct a regex pattern to match the whole word, case-insensitive.\n",
        "    pattern = r'\\b' + re.escape(target_word) + r'\\b'\n",
        "\n",
        "    # Use re.search to find the pattern in the sentence.\n",
        "    match = re.search(pattern, sentence, flags=re.IGNORECASE)\n",
        "\n",
        "    # Return True if a match is found, False otherwise.\n",
        "    return bool(match)\n",
        "\n",
        "def convert_str_to_set(itemset_str):\n",
        "    \"\"\"\n",
        "    Converts a comma-separated string into a set of items.\n",
        "\n",
        "    This function takes a string containing items separated by commas,\n",
        "    strips any surrounding whitespace from each item, and returns a set\n",
        "    containing the unique items. This is useful for converting textual\n",
        "    representations of itemsets into Python set objects for further analysis.\n",
        "\n",
        "    Args:\n",
        "        itemset_str (str): A string of items separated by commas.\n",
        "\n",
        "    Returns:\n",
        "        set: A set containing the stripped items from the input string.\n",
        "    \"\"\"\n",
        "    # Split the string by commas and strip whitespace from each item.\n",
        "    items = [item.strip() for item in itemset_str.split(',')]\n",
        "\n",
        "    # Convert the list of items to a set to ensure uniqueness.\n",
        "    return set(items)\n",
        "\n",
        "def get_season(month):\n",
        "    \"\"\"\n",
        "    Determine the season corresponding to a given month.\n",
        "    Args:\n",
        "        month (int): The month as an integer (1 for January, 2 for February, ..., 12 for December).\n",
        "    Returns:\n",
        "        str: The season name corresponding to the input month:\n",
        "            - 'Winter' for December (12), January (1), February (2)\n",
        "            - 'Spring' for March (3), April (4), May (5)\n",
        "            - 'Summer' for June (6), July (7), August (8)\n",
        "            - 'Fall' for September (9), October (10), November (11)\n",
        "    Raises:\n",
        "        ValueError: If the input month is not between 1 and 12.\n",
        "    \"\"\"\n",
        "    if month in [12, 1, 2]:\n",
        "        return 'Winter'\n",
        "    elif month in [3, 4, 5]:\n",
        "        return 'Spring'\n",
        "    elif month in [6, 7, 8]:\n",
        "        return 'Summer'\n",
        "    elif month in [9, 10, 11]:\n",
        "        return 'Fall'\n",
        "    else:\n",
        "        raise ValueError(\"Month must be an integer between 1 and 12.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "in6RcmHhsRMk"
      },
      "source": [
        "## Define Lookup Tables"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7sbfEVkPGB0P"
      },
      "outputs": [],
      "source": [
        "lookup_ventilation = [\n",
        "    \"lbk\",\n",
        "    \"luchtbehandeling\",\n",
        "    \"luchtbehandelen\",\n",
        "    \"luchbehandeling\",\n",
        "    \"luchtbehandeling\",\n",
        "    \"ventilatiesysteem\",\n",
        "    \"ventilatie\",\n",
        "    \"luchtklep\",\n",
        "    \"stoombevochtiger\",\n",
        "    \"stoombevochtiging\",\n",
        "    \"bevochtiger\",\n",
        "    \"toevoerventilator\",\n",
        "    \"afvoerventilator\",\n",
        "    \"ventilatormotor\",\n",
        "    \"dakventilatoren\",\n",
        "    \"Dakventilator\",\n",
        "    \"toevoer ventilator\",\n",
        "    \"dak ventilatoren\",\n",
        "    \"Afzuiventilator\",\n",
        "    \"wiel\",\n",
        "    \"afzuigvent\",\n",
        "    \"V-snaren\",\n",
        "    \"V snaren\",\n",
        "    \"filters\",\n",
        "    \"snaarbreuk\",\n",
        "    \"condensafvoer\",\n",
        "    \"condensor\",\n",
        "    \"filter\",\n",
        "    \"Luchtbeh\",\n",
        "    \"LBH\",\n",
        "    \"vorst\",\n",
        "    \"verwarmingsbatterij\",\n",
        "    \"stoomvochtiger\",\n",
        "    \"luchtdebiet\",\n",
        "    \"luchtzakken\",\n",
        "    \"fancoil\",\n",
        "    \"fan coil\",\n",
        "    \"fan-coil\"\n",
        "]\n",
        "\n",
        "lookup_cooling = [\n",
        "    \"airco\",\n",
        "    \"drogekoeler\",\n",
        "    \"koelmachine\",\n",
        "    \"koeling\",\n",
        "    \"KM\",\n",
        "    \"koelunit\",\n",
        "    \"koelinstallatie\",\n",
        "    \"topcooling\",\n",
        "    \"gkw\",\n",
        "    \"koeltoren\",\n",
        "    \"chillers\",\n",
        "    \"dry-cooler\",\n",
        "    \"drycooler\",\n",
        "    \"koelplafond\",\n",
        "    \"koelklep\",\n",
        "    \"koelwaterpomp\",\n",
        "    \"carrier\",\n",
        "    \"koel-unit\",\n",
        "    \"DX koeler\",\n",
        "    \"DX-koeler\",\n",
        "    \"draaikoeler\",\n",
        "    \"condensventilator\",\n",
        "    \"chiller\",\n",
        "    \"koelventilator\",\n",
        "    \"Condensorventilator\",\n",
        "    \"condensor\",\n",
        "    \"split unit\",\n",
        "    \"split-unit\",\n",
        "    \"splitunit\"\n",
        "]\n",
        "\n",
        "lookup_heating = [\n",
        "    \"ketel\",\n",
        "    \"CV\",\n",
        "    \"c.v.\",\n",
        "    \"c.v\",\n",
        "    \"Kachel\",\n",
        "    \"verwarming\",\n",
        "    \"radiatoren\",\n",
        "    \"radiator\",\n",
        "    \"vloerverwarming\",\n",
        "    \"rookgasventilator\"\n",
        "]\n",
        "\n",
        "lookup_fireSafety = [\n",
        "    \"brandmeld\",\n",
        "    \"rookmelder\",\n",
        "    \"branddeur\",\n",
        "    \"brandklep\",\n",
        "    \"brandweer\",\n",
        "    \"brandhaspel\",\n",
        "    \"brandblus\",\n",
        "    \"ontruiming\",\n",
        "    \"Brandventilatoren\",\n",
        "    \"trappenhuis\"\n",
        "]\n",
        "\n",
        "lookup_entrance = [\n",
        "    \"toegangspoort\",\n",
        "    \"tourniqet\",\n",
        "    \"paslezer\",\"tourniquet\",\n",
        "    \"tourniqeut\",\n",
        "    \"tourniqut\",\n",
        "    \"toegang\",\n",
        "    \"Toerniqet\",\n",
        "    \"tourniget\",\n",
        "    \"garagedeur\",\n",
        "    \"schuifhek\",\n",
        "    \"slagboom\"\n",
        "]\n",
        "\n",
        "lookup_shading = [\n",
        "    \"zonwering\",\n",
        "    \"zonneschermen\",\n",
        "    \"zonnewering\"\n",
        "]\n",
        "\n",
        "lookup_sanitary = [\n",
        "    \"toilet\",\n",
        "    \"WC\",\n",
        "    \"urinoir\",\n",
        "    \"wastafel\",\n",
        "    \"sanitair\",\n",
        "    \"wasbak\",\n",
        "    \"afvoer\"\n",
        "]\n",
        "\n",
        "lookup_heatPump = [\n",
        "    \"warmtepomp\",\n",
        "    \"warmte pomp\",\n",
        "    \"WKO\"\n",
        "]\n",
        "\n",
        "lookup_lighting = [\n",
        "    \"verlichting\",\n",
        "    \"lamp\",\n",
        "    \"Tl-buis\",\n",
        "    \"licht\",\n",
        "    \"tlarmatuur\",\n",
        "    \"tlbuizen\",\n",
        "    \"armatuur\",\n",
        "    \"armaturen\",\n",
        "    \"armanturen\"\n",
        "]\n",
        "\n",
        "lookup_elevator = [\n",
        "    \"Lift\"\n",
        "]\n",
        "\n",
        "lookup_wkk = [\n",
        "    \"WKK\"\n",
        "]\n",
        "\n",
        "lookup_bms = [\n",
        "    \"GBS\",\n",
        "    \"Data\",\n",
        "    \"logger\",\n",
        "    \"lon\",\n",
        "    \"BMC\",\n",
        "    \"priva\",\n",
        "    \"software\",\n",
        "    \"regeling\",\n",
        "    \"hardware\",\n",
        "    \"regelkast\",\n",
        "    \"RK 1\",\n",
        "    \"rk5\",\n",
        "    \"rk2\",\n",
        "    \"RK5\",\n",
        "    \"rk3\",\n",
        "    \"Rk 3\",\n",
        "    \"rk4\",\n",
        "    \"rk 4\",\n",
        "    \"rk1\",\n",
        "    \"rk2\",\n",
        "    \"rk6\",\n",
        "    \"rk 6\",\n",
        "    \"RK7\",\n",
        "    \"sensor\",\n",
        "    \"regelaar\",\n",
        "    \"opnemer\",\n",
        "    \"thermostaat\",\n",
        "    \"meting\",\n",
        "    \"onderstation\",\n",
        "    \"Kloktijden\",\n",
        "    \"kastventilatoren\",\n",
        "    \"kastventilatoren\",\n",
        "    \"kastventilator\",\n",
        "    \"Kastventilator\"\n",
        "]\n",
        "\n",
        "lookup_waterDistribution = [\n",
        "    \"regelklep\",\n",
        "    \"driewegklep\",\n",
        "    \"TSA\",\n",
        "    \"Warmtewisselaar\",\n",
        "    \"pomp\",\n",
        "    \"hydrofoor\",\n",
        "    \"expansie\",\n",
        "    \"drukvat\",\n",
        "    \"waterleiding\",\n",
        "    \"3wegklep\",\n",
        "    \"transportnet\"\n",
        "]\n",
        "\n",
        "lookup_office = [\n",
        "    \"werkvoorbereiding\",\n",
        "    \"Contractbeheerder\",\n",
        "    \"materiaal\",\n",
        "    \"Onderaanneming\",\n",
        "    \"Werkvoorbereider\",\n",
        "    \"kantoor\",\n",
        "    \"Materiaalbon\",\n",
        "    \"Urenbon\",\n",
        "    \"Contractbegeleiding\",\n",
        "    \"Contractbeheer\",\n",
        "    \"overleg\",\n",
        "    \"Meet-enregeltechniek\",\n",
        "    \"Inlenen\",\n",
        "    \"Weekplanning\",\n",
        "    \"calculatie\",\n",
        "    \"Onderaannemering\",\n",
        "    \"contractmanager\",\n",
        "    \"Contractmanagement\",\n",
        "    \"Onderaannemer\"\n",
        "]\n",
        "\n",
        "lookup_domesticWater = [\n",
        "    \"warm water\",\n",
        "    \"ww\",\n",
        "    \"w.w.\",\n",
        "    \"w.w\",\n",
        "    \"warmtapwater\",\n",
        "    \"warmwater\",\n",
        "    \"boiler\"\n",
        "]\n",
        "\n",
        "lookup_faultRedemption = [\n",
        "    \"Storingsafkoop\",\n",
        "    \"Afkoopstoringen\",\n",
        "    \"Verrekening afkoop\"\n",
        "]\n",
        "\n",
        "lookup_regularMaintenance = [\n",
        "    \"onderhoud\",\n",
        "    \"inspectie\",\n",
        "    \"OH\",\n",
        "    \"OHD\",\n",
        "    \"controle\",\n",
        "    \"Preventief\",\n",
        "    \"testen\",\n",
        "    \"Bedrijfvoering\",\n",
        "    \"Bedrijfsvoering\"\n",
        "]\n",
        "\n",
        "lookup_complaints = [\n",
        "    \"klachten\",\n",
        "    \"klacht\",\n",
        "    \"te warm\",\n",
        "    \"tekoud\",\n",
        "    \"luchtvochtigheid\",\n",
        "    \"klimaatbeheersing\",\n",
        "    \"klimaat\",\n",
        "    \"teheet\",\n",
        "    \"benauwd\",\n",
        "    \"tocht\",\n",
        "    \"R.V.\",\n",
        "    \"ergkoud\",\n",
        "    \"RV telaag\",\n",
        "    \"lekkage\",\n",
        "    \"Ruimtevochtigheid\",\n",
        "    \"erg warm\",\n",
        "    \"erg koud\"\n",
        "]\n",
        "\n",
        "module_names = {\n",
        "    \"complaints\": \"Complaints\",\n",
        "    \"regularMaintenance\": \"Regular Maintenance\",\n",
        "    \"faultRedemption\": \"Fault Redemption\",\n",
        "    \"domesticWater\": \"Domestic Water\",\n",
        "    \"office\": \"Office\",\n",
        "    \"waterDistribution\": \"Water Distribution\",\n",
        "    \"bms\": \"BMS\",\n",
        "    \"wkk\": \"WKK\",\n",
        "    \"elevator\": \"Elevator\",\n",
        "    \"lighting\": \"Lighting\",\n",
        "    \"heatPump\": \"Heat Pump\",\n",
        "    \"sanitary\": \"Sanitary\",\n",
        "    \"fireSafety\": \"Fire Safety\",\n",
        "    \"shading\": \"Shading\",\n",
        "    \"entrance\": \"Entrance\",\n",
        "    \"ventilation\": \"Ventilation\",\n",
        "    \"heating\": \"Heating\",\n",
        "    \"cooling\": \"Cooling\",\n",
        "}\n",
        "\n",
        "normalize_lookups()\n",
        "\n",
        "# Get the list of all of the lookups.\n",
        "list_lookups = [var for var in globals() if re.match(r'^lookup_', var)]\n",
        "\n",
        "# Sort the lookups descendingly based on their lengths.\n",
        "list_lookups = sorted(list_lookups, key=lambda x: len(globals()[x]), reverse=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MFYjHZpCGigl"
      },
      "source": [
        "# Data Reading"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7aqqRwUyGqOe"
      },
      "outputs": [],
      "source": [
        "# Define the path to the dataset here.\n",
        "path = 'dataset_exel_file.xlsx'\n",
        "\n",
        "df_so = pd.read_excel(path)\n",
        "df_so.reset_index(drop=True, inplace=True)\n",
        "\n",
        "# Define the columns that you want to extract from the dataset here.\n",
        "columns_to_select = ['Description', 'Problem Text', 'Causation Text', 'Solution Text', 'Start Date']\n",
        "df_so = df_so[columns_to_select]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TZRZ0oZgG3gB"
      },
      "source": [
        "# Exploratory Data Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O4Lz1Pbvl9iI"
      },
      "source": [
        "## Statistical Analysis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4mx36kk6HtxD"
      },
      "outputs": [],
      "source": [
        "df_so.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SmSykyXPHwJ1"
      },
      "outputs": [],
      "source": [
        "df_so.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QY_nF4J6aHok"
      },
      "outputs": [],
      "source": [
        "stop_words = set(stopwords.words('dutch'))\n",
        "print(stop_words)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VDJL0LMGIC6r"
      },
      "source": [
        "## Data Cleaning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vANUeU_EIGs0"
      },
      "outputs": [],
      "source": [
        "textual_columns = ['Description', 'Probleemtekst', 'Causation Text', 'Solution Text']\n",
        "\n",
        "# Correct cell value types.\n",
        "df_so_cleaned = correct_types(df_so.copy(), textual_columns)\n",
        "\n",
        "# Replace punctuations.\n",
        "df_so_cleaned = replace_punctuation(df_so_cleaned, textual_columns)\n",
        "\n",
        "# Remove numerical values.\n",
        "df_so_cleaned = remove_numerical_values(df_so_cleaned, textual_columns)\n",
        "\n",
        "# Remove Dutch stop words.\n",
        "stop_words = set(stopwords.words('dutch'))\n",
        "stop_words.add('via')\n",
        "# stop_words.add('storing')\n",
        "stop_words.remove('niet')\n",
        "df_so_cleaned[textual_columns] = df_so_cleaned[textual_columns].applymap(lambda x: ' '.join([word for word in x.split() if word.lower() not in stop_words]))\n",
        "\n",
        "# Stemize the text.\n",
        "df_so_cleaned[textual_columns] = df_so_cleaned[textual_columns].applymap(stemize)\n",
        "\n",
        "df_so_cleaned.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j9xAymHkIRnv"
      },
      "source": [
        "## Data Annotation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kjkC1P61IUjs"
      },
      "outputs": [],
      "source": [
        "# Apply the 'group_modules' function to categorize each service order (SO) description into a main subsystem (module).\n",
        "# The 'target' column will store the assigned subsystem label for each service order based on the lookup tables.\n",
        "df_so_cleaned['target'] = df_so_cleaned['Description'].apply(lambda description: group_modules(str(description), list_lookups))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "07xNgpNqLQzz"
      },
      "outputs": [],
      "source": [
        "# Calculate the counts for each target category.\n",
        "target_counts = df_so_cleaned[\"target\"].value_counts()\n",
        "\n",
        "# Calculate the total number of samples.\n",
        "total_samples = len(df_so_cleaned)\n",
        "\n",
        "# Plotting the bar chart.\n",
        "plt.figure(figsize=(10, 6))\n",
        "bars = plt.bar(target_counts.index, target_counts, color=plt.cm.viridis(np.linspace(0, 1, len(target_counts))))\n",
        "plt.title(\"Frequency of Target Categories\")\n",
        "plt.xlabel(\"Target Categories\")\n",
        "plt.ylabel(\"Frequency\")\n",
        "plt.xticks(rotation=45, ha='right')\n",
        "plt.yticks(minor=True)\n",
        "\n",
        "# Add counts on top of the bars.\n",
        "for bar, count in zip(bars, target_counts):\n",
        "    plt.text(bar.get_x() + bar.get_width() / 2, count + 1, str(count), ha='center', va='bottom')\n",
        "\n",
        "# Calculate and format percentages for legend labels.\n",
        "percentages = [(f\"{category} = {count/total_samples*100:.2f}%\") for category, count in target_counts.items()]\n",
        "\n",
        "# Add legend with custom labels.\n",
        "plt.legend(bars, percentages, loc='upper right', bbox_to_anchor=(1, 1), ncol=2, fontsize='small')\n",
        "\n",
        "plt.savefig('./figures/Annotation (Classification)/module_categories.png', bbox_inches = 'tight')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o18HZVcFGRF4"
      },
      "source": [
        "## Keep Original Description Column (Description)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1QAVZ-q3GaT_"
      },
      "outputs": [],
      "source": [
        "df_so_cleaned['original_sentence'] = df_so['Description']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yi9-Fr-CunVf"
      },
      "source": [
        "## Get Heating Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5hjbTZKluu_9"
      },
      "outputs": [],
      "source": [
        "df_so_cleaned_heating = df_so_cleaned[df_so_cleaned['target'] == 'Heating']\n",
        "\n",
        "df_so_cleaned_heating.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tg7AC71J7dxV"
      },
      "outputs": [],
      "source": [
        "df_so_cleaned_heating.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AzaIeJSxRzNO"
      },
      "source": [
        "## Visualization on Description"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xmOFCaXpR3jt"
      },
      "outputs": [],
      "source": [
        "# Extract the 'Description' (service order description) for all service orders categorized under 'Heating'.\n",
        "# Store them as a list for further processing in text analysis.\n",
        "corpus_heating = df_so_cleaned_heating[df_so_cleaned_heating['target'] == 'Heating']['Description'].tolist()\n",
        "\n",
        "# Initialize a CountVectorizer to convert the heating-related descriptions into a document-term matrix (DTM).\n",
        "# We are specifying that we only want to extract unigrams (individual words) with ngram_range=(1, 1).\n",
        "vectorizer_heating = CountVectorizer(ngram_range=(1, 1))\n",
        "\n",
        "# Fit the vectorizer to the corpus and transform it into a DTM (a matrix of word counts).\n",
        "dtm_heating = vectorizer_heating.fit_transform(corpus_heating)\n",
        "\n",
        "# Retrieve the feature names (terms) from the vectorizer (i.e., the words in the DTM).\n",
        "terms_heating = vectorizer_heating.get_feature_names_out()\n",
        "\n",
        "# Convert the DTM into a DataFrame where each column corresponds to a word and each row represents a service order.\n",
        "dtm_heating = pd.DataFrame(dtm_heating.toarray(), columns=terms_heating)\n",
        "\n",
        "# Calculate the frequency of each term by summing over the columns of the DTM.\n",
        "term_frequencies_heating = dtm_heating.sum()\n",
        "\n",
        "# Generate a word cloud based on the term frequencies for the heating-related service orders.\n",
        "wordcloud = WordCloud(width=1000, height=600, background_color='white').generate_from_frequencies(term_frequencies_heating)\n",
        "\n",
        "# Display the word cloud.\n",
        "plt.title('Heating Word Cloud')\n",
        "plt.imshow(wordcloud, interpolation='bilinear')\n",
        "plt.axis('off')\n",
        "\n",
        "# Save the generated word cloud image to the specified directory. Change this in case of need!\n",
        "plt.savefig('./figures/EDA/heating_terms_word_cloud.png')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ajp0qWDCP9YK"
      },
      "source": [
        "# ARM"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ucV1vvVYR7S"
      },
      "source": [
        "## Obtain Pruned Association Rules"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3kQ2q5acSTzj"
      },
      "outputs": [],
      "source": [
        "# Step 1: Split each service order description in the 'Description' column into individual words for the ventilation component.\n",
        "transactions_heating = df_so_cleaned_heating[df_so_cleaned_heating['target'] == 'Heating']['Description'].apply(lambda t: t.split(' '))\n",
        "\n",
        "# Convert the series of lists into a list of transactions (each transaction is a list of words).\n",
        "transactions_heating = list(transactions_heating)\n",
        "\n",
        "# Step 2: Apply one-hot encoding to the transactions using the TransactionEncoder.\n",
        "encoder_heating = TransactionEncoder().fit(transactions_heating)\n",
        "onehot_heating = encoder_heating.transform(transactions_heating)\n",
        "onehot_heating = pd.DataFrame(onehot_heating, columns=encoder_heating.columns_)\n",
        "\n",
        "# Step 3: Define the minimum support threshold for finding frequent itemsets.\n",
        "'''\n",
        "Define the minimum number of occurrences for each itemset.\n",
        "This indirectly acts as a hyperparamether, so tune it as you pefer.\n",
        "Increase the value of \"n\" if you want to find more frequent itemsets.\n",
        "Decrease the value of \"n\" if you want to find less frequent itemsets.\n",
        "'''\n",
        "n = 30\n",
        "minimum_support = n/len(onehot_heating) # Calculate the minimum support as the ratio of 'n' to the total number of transactions.\n",
        "\n",
        "# Step 4: Run the Apriori algorithm to identify frequent itemsets based on the minimum support threshold.\n",
        "frequent_itemsets_heating = apriori(onehot_heating,\n",
        "                            min_support =  minimum_support,\n",
        "                            use_colnames = True,\n",
        "                            verbose = 1)\n",
        "\n",
        "# Step 5: Generate association rules from the frequent itemsets using the 'lift' metric with a minimum threshold of 1.\n",
        "rules_heating = association_rules(frequent_itemsets_heating, metric = 'lift', min_threshold = 1)\n",
        "\n",
        "# Print summary statistics of the process.\n",
        "print(f'Number of transactions: {len(onehot_heating)}')\n",
        "print(f'Minimum number of occurrence for each itemset: {n}')\n",
        "print(f'Minimum support threshold for itemsets: {minimum_support}')\n",
        "print(f'number of frequent itemsets: {len(frequent_itemsets_heating)}')\n",
        "print(f'Number of rules: {len(rules_heating)}\\n')\n",
        "\n",
        "# Step 6: Replace frozen sets with strings for easier readability in the rules.\n",
        "rules_heating['antecedents'] = rules_heating['antecedents'].apply(lambda a: ','.join(list(a)))\n",
        "rules_heating['consequents'] = rules_heating['consequents'].apply(lambda a: ','.join(list(a)))\n",
        "\n",
        "# Display the final association rules.\n",
        "rules_heating"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0RcmlB7A0SEf"
      },
      "source": [
        "## Heatmap of support"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K0x0OQ-JkuHq"
      },
      "outputs": [],
      "source": [
        "# Step 1: Transform the association rules data into a pivot table for the heatmap.\n",
        "# The pivot table uses 'consequents' as the row index and 'antecedents' as the columns, with 'support' values as the matrix.\n",
        "pivot = rules_heating.pivot(index='consequents', columns='antecedents', values='support')\n",
        "\n",
        "# Step 2: Create a mask for the upper triangle of the heatmap to avoid duplicate values being displayed.\n",
        "mask = np.triu(np.ones_like(pivot, dtype=bool))\n",
        "\n",
        "# Step 3: Set the font scale for the heatmap.\n",
        "sns.set(font_scale=0.9)\n",
        "\n",
        "# Step 4: Create the heatmap figure.\n",
        "plt.figure(figsize=(15, 8))\n",
        "\n",
        "# Step 5: Generate the heatmap using seaborn.\n",
        "heatmap = sns.heatmap(pivot,  # The pivot table containing support values.\n",
        "                      cmap='coolwarm',  # Use the 'coolwarm' color palette.\n",
        "                      mask=mask,  # Apply the mask to hide the upper triangle of the heatmap.\n",
        "                      cbar=True,  # Show the color bar indicating the support values.\n",
        "                      linewidths=0.5,  # Set the width of the grid lines.\n",
        "                      linecolor='black',  # Set the color of the grid lines.\n",
        "                      annot=True,  # Annotate each cell with the support value.\n",
        "                      fmt='.2f')  # Format the annotations to two decimal places.\n",
        "\n",
        "# Step 6: Customize the heatmap.\n",
        "plt.xticks(rotation=45)  # Rotate the x-axis labels 45 degrees for better readability.\n",
        "plt.title(f'Heating Rules Heatmap: {len(onehot_heating)} transactions, occurrence min-value of {n}, support threshold of {minimum_support:.2f}')  # Set the heatmap title.\n",
        "plt.yticks(rotation=0)  # Keep the y-axis labels horizontal.\n",
        "\n",
        "# Step 7: Save the heatmap as an image file. Change the output path in case of need!\n",
        "plt.savefig('./figures/Association Rule Mining/heating/support_heatmap.png')\n",
        "\n",
        "# Step 8: Display the heatmap.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "estESIcU0byi"
      },
      "source": [
        "## Heatmap of leverage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q-C8mH070h5Y"
      },
      "outputs": [],
      "source": [
        "# Step 1: Transform the association rules data into a pivot table for the heatmap.\n",
        "# The pivot table uses 'consequents' as the row index and 'antecedents' as the columns, with 'leverage' values as the matrix.\n",
        "pivot = rules_heating.pivot(index='consequents', columns='antecedents', values='leverage')\n",
        "\n",
        "# Step 2: Create a mask for the upper triangle of the heatmap to avoid duplicate values being displayed.\n",
        "mask = np.triu(np.ones_like(pivot, dtype=bool))\n",
        "\n",
        "# Step 3: Set the font scale for the heatmap.\n",
        "sns.set(font_scale=0.9)\n",
        "\n",
        "# Step 4: Create the heatmap figure.\n",
        "plt.figure(figsize=(15, 8))\n",
        "\n",
        "# Step 5: Generate the heatmap using seaborn.\n",
        "heatmap = sns.heatmap(pivot,  # The pivot table containing support values.\n",
        "                      cmap='coolwarm',  # Use the 'coolwarm' color palette.\n",
        "                      mask=mask,  # Apply the mask to hide the upper triangle of the heatmap.\n",
        "                      cbar=True,  # Show the color bar indicating the support values.\n",
        "                      linewidths=0.5,  # Set the width of the grid lines.\n",
        "                      linecolor='black',  # Set the color of the grid lines.\n",
        "                      annot=True,  # Annotate each cell with the support value.\n",
        "                      fmt='.2f')  # Format the annotations to two decimal places.\n",
        "\n",
        "# Step 6: Customize the heatmap.\n",
        "plt.xticks(rotation=45) # Rotate the x-axis labels 45 degrees for better readability.\n",
        "plt.title(f'Heating Rules Heatmap: {len(onehot_heating)} transactions, occurrence min-value of {n}, support threshold of {minimum_support:.2f}') # Set the heatmap title.\n",
        "plt.yticks(rotation=0) # Keep the y-axis labels horizontal.\n",
        "\n",
        "# Step 7: Save the heatmap as an image file. Change the output path in case of need!\n",
        "plt.savefig('./figures/Association Rule Mining/heating/leverage_heatmap.png')\n",
        "\n",
        "# Step 8: Display the heatmap.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sEJi49ya0zFE"
      },
      "source": [
        "## Heatmap of conviction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RmW1dUaT04PH"
      },
      "outputs": [],
      "source": [
        "# Step 1: Transform the association rules data into a pivot table for the heatmap.\n",
        "# The pivot table uses 'consequents' as the row index and 'antecedents' as the columns, with 'conviction' values as the matrix.\n",
        "pivot = rules_heating.pivot(index='consequents', columns='antecedents', values='conviction')\n",
        "\n",
        "# Step 2: Create a mask for the upper triangle of the heatmap to avoid duplicate values being displayed.\n",
        "mask = np.triu(np.ones_like(pivot, dtype=bool))\n",
        "\n",
        "# Step 3: Set the font scale for the heatmap.\n",
        "sns.set(font_scale=0.9)\n",
        "\n",
        "# Step 4: Create the heatmap figure.\n",
        "plt.figure(figsize=(15, 8))\n",
        "\n",
        "# Step 5: Generate the heatmap using seaborn.\n",
        "heatmap = sns.heatmap(pivot,  # The pivot table containing support values.\n",
        "                      cmap='coolwarm',  # Use the 'coolwarm' color palette.\n",
        "                      mask=mask,  # Apply the mask to hide the upper triangle of the heatmap.\n",
        "                      cbar=True,  # Show the color bar indicating the support values.\n",
        "                      linewidths=0.5,  # Set the width of the grid lines.\n",
        "                      linecolor='black',  # Set the color of the grid lines.\n",
        "                      annot=True,  # Annotate each cell with the support value.\n",
        "                      fmt='.2f')  # Format the annotations to two decimal places.\n",
        "\n",
        "# Step 6: Customize the heatmap.\n",
        "plt.xticks(rotation=45) # Rotate the x-axis labels 45 degrees for better readability.\n",
        "plt.title(f'Heating Rules Heatmap: {len(onehot_heating)} transactions, occurrence min-value of {n}, support threshold of {minimum_support:.2f}') # Set the heatmap title.\n",
        "plt.yticks(rotation=0) # Keep the y-axis labels horizontal.\n",
        "\n",
        "# Step 7: Save the heatmap as an image file. Change the output path in case of need!\n",
        "plt.savefig('./figures/Association Rule Mining/heating/conviction_heatmap.png')\n",
        "\n",
        "# Step 8: Display the heatmap.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ObBCjFgh1TWG"
      },
      "source": [
        "## Frequency table"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z_Szvvds7Tbb"
      },
      "source": [
        "### Prepare items"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "01pvJ5SO7X-4"
      },
      "outputs": [],
      "source": [
        "# Step 1: Combine the 'antecedents' and 'consequents' of each rule into a single 'itemset' string.\n",
        "rules_heating['itemset'] = rules_heating.apply(lambda row: f\"{row['antecedents']},{row['consequents']}\", axis=1)\n",
        "\n",
        "# Step 2: Calculate the count of each rule's occurrence based on its support value and the total number of transactions.\n",
        "# Multiply the support by the total number of transactions (len(onehot_heating)) to get the absolute count.\n",
        "rules_heating['count'] = rules_heating.apply(lambda row: int(row['support'] * len(onehot_heating)), axis=1)\n",
        "\n",
        "# Step 3: Convert the 'itemset' string to a Python set to facilitate itemset manipulation.\n",
        "rules_heating['itemset'] = rules_heating['itemset'].apply(convert_str_to_set)\n",
        "\n",
        "# Step 4: Format the support values to three decimal places and store them in a new column 'rounded_support'.\n",
        "rules_heating['rounded_support'] = rules_heating.apply(lambda row: f\"{row['support']:.3f}\", axis=1)\n",
        "\n",
        "# Step 5: Calculate the percentage occurrence of each itemset and format it with two decimal places, adding the '%' symbol.\n",
        "rules_heating['percentage'] = rules_heating.apply(lambda row: f\"{row['count']*100/len(onehot_heating):.2f}%\", axis=1)\n",
        "\n",
        "# Step 6: Round the conviction values to three decimal places and store them in a new column 'rounded_conviction'.\n",
        "rules_heating['rounded_conviction'] = rules_heating.apply(lambda row: f\"{row['conviction']:.3f}\", axis=1)\n",
        "\n",
        "# Step 7: Round the lift values to three decimal places and store them in a new column 'rounded_lift'.\n",
        "rules_heating['rounded_lift'] = rules_heating.apply(lambda row: f\"{row['lift']:.3f}\", axis=1)\n",
        "\n",
        "# Step 8: Sort the rules based on conviction values in descending order to prioritize rules with higher conviction.\n",
        "rules_heating.sort_values(by='conviction', ascending=False, inplace=True)\n",
        "\n",
        "# Step 9: Drop duplicate rules based on 'itemset', keeping only the first occurrence.\n",
        "rules_heating.drop_duplicates(subset=['itemset'], keep='first', inplace=True)\n",
        "\n",
        "# Step 10: Reset the index of the DataFrame to maintain consistency after sorting and dropping duplicates.\n",
        "rules_heating.reset_index(inplace=True)\n",
        "\n",
        "# Output the final processed DataFrame.\n",
        "rules_heating"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LQ3y9Zbabahx"
      },
      "source": [
        "### Show Heating Rules"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PhqSvjCacYtb"
      },
      "outputs": [],
      "source": [
        "# Sort values by 'support' column.\n",
        "rules_heating.sort_values(by='support', ascending=False, inplace=True)\n",
        "\n",
        "# Add the index as a column.\n",
        "rules_heating.reset_index(inplace=True)\n",
        "rules_heating.rename(columns={'index': 'index'}, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SwWidvBIThER"
      },
      "outputs": [],
      "source": [
        "# Step 1: Create a matplotlib subplot with a figure size of 15x6 inches to accommodate the table.\n",
        "fig, ax = plt.subplots(figsize=(15, 6))\n",
        "\n",
        "# Step 2: Turn off the axis display to hide axis lines and labels.\n",
        "ax.axis('tight')\n",
        "ax.axis('off')\n",
        "\n",
        "# Step 3: Define the columns to include in the table. These columns will be used to extract data for table rows.\n",
        "columns_to_include = ['index', 'itemset', 'rounded_support', 'count', 'percentage', 'rounded_lift', 'rounded_conviction']\n",
        "\n",
        "# Step 4: Create a table using the sorted 'rules_heating' DataFrame.\n",
        "# 'cellText' contains the table data for the specified columns.\n",
        "# 'colLabels' provides the actual column headers for the table.\n",
        "table = ax.table(cellText=rules_heating[columns_to_include].values,\n",
        "                 colLabels=rules_heating[['index', 'itemset', 'support', 'count', 'percentage', 'lift','conviction']].columns,\n",
        "                 cellLoc='center', loc='center')  # Center-align both the cell text and the headers.\n",
        "\n",
        "# Step 5: Disable the automatic adjustment of font size and set a fixed font size for the table.\n",
        "table.auto_set_font_size(False)\n",
        "table.set_fontsize(10)\n",
        "\n",
        "# Step 6: Set the title of the table plot, indicating that the rules are sorted by support.\n",
        "plt.title('Heating Rules - Sorted by Support')\n",
        "\n",
        "# Step 7: Adjust the layout to prevent any overlap and ensure the table fits within the figure dimensions.\n",
        "plt.tight_layout()\n",
        "\n",
        "# Step 8: Save the table as an image file in the specified path with bounding box adjustments to avoid cutting off elements.\n",
        "# Change the saving path in case of need!\n",
        "plt.savefig('./figures/Association Rule Mining/heating/itemsets_frequency_table.png', bbox_inches='tight')\n",
        "\n",
        "# Step 9: Display the table plot.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a4Nrt2Xx7mVq"
      },
      "outputs": [],
      "source": [
        "# Step 1: Sort the 'rules_heating' DataFrame by the 'conviction' column in descending order.\n",
        "rules_heating.sort_values(by='conviction', ascending=False, inplace=True)\n",
        "\n",
        "# Step 2: Create a matplotlib subplot with a figure size of 15x6 inches to display the table.\n",
        "fig, ax = plt.subplots(figsize=(15, 6))\n",
        "\n",
        "# Step 3: Set the axis to 'tight' to avoid excessive whitespace and turn off the axis display.\n",
        "ax.axis('tight')\n",
        "ax.axis('off')\n",
        "\n",
        "# Step 4: Create a table using the sorted 'rules_heating' DataFrame.\n",
        "# 'cellText' contains the data for the specified columns to be displayed in the table.\n",
        "# 'colLabels' defines the column headers for the table.\n",
        "table = ax.table(cellText=rules_heating[['itemset', 'rounded_support', 'count', 'percentage', 'rounded_conviction']].values,\n",
        "                 colLabels=rules_heating[['itemset', 'support', 'count', 'percentage', 'conviction']].columns,\n",
        "                 cellLoc='center', loc='center', fontsize=15)  # Center-align cell text and set font size.\n",
        "\n",
        "# Step 5: Set the title for the table plot indicating it displays heating rules sorted by conviction.\n",
        "plt.title('Heating Rules - Sorted by Conviction')\n",
        "\n",
        "# Step 6: Adjust the layout to prevent any overlap and ensure that the table fits well within the figure.\n",
        "plt.tight_layout()\n",
        "\n",
        "# Step 7: Display the table plot.\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cPwfQBL9eh6o"
      },
      "source": [
        "# Frequency Analysis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kHdGqdsOeo5W"
      },
      "outputs": [],
      "source": [
        "# Extract the year from the 'Start Date' column and create a new 'Year' column.\n",
        "df_so_cleaned_heating['Year'] = df_so_cleaned_heating['Start Date'].dt.year\n",
        "\n",
        "# Count the occurrences of each year and sort the results in ascending order.\n",
        "year_counts = df_so_cleaned_heating['Year'].value_counts().sort_index()\n",
        "\n",
        "# Create a bar plot to visualize the distribution of service orders by year.\n",
        "plt.figure(figsize=(10, 6))  # Set the figure size for the plot.\n",
        "year_counts.plot(kind='bar', color='green')  # Plot the year counts as a bar graph.\n",
        "plt.xlabel('Year')  # Label the x-axis.\n",
        "plt.ylabel('Count of Service Orders')  # Label the y-axis.\n",
        "plt.title('Distribution of Service Orders for Heating Component Across Years')  # Set the title of the plot.\n",
        "plt.xticks(rotation=0)  # Rotate x-axis labels to 0 degrees for better readability.\n",
        "plt.savefig('./figures/Frequency/heating/yearly_distribution.png')  # Save the figure as a PNG file.\n",
        "plt.show()  # Display the plot."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sX6XsYlqepUI"
      },
      "outputs": [],
      "source": [
        "# Preprocessing: Extract the month from the 'Start Date' datetime column and create a new 'Month' column.\n",
        "df_so_cleaned_heating['Month'] = df_so_cleaned_heating['Start Date'].dt.month\n",
        "\n",
        "# Count the occurrences of each month and sort the results in ascending order.\n",
        "month_counts = df_so_cleaned_heating['Month'].value_counts().sort_index()\n",
        "\n",
        "# Create a bar plot to visualize the distribution of service orders by month.\n",
        "plt.figure(figsize=(10, 6))  # Set the figure size for the plot.\n",
        "month_counts.plot(kind='bar', color='green')  # Plot the month counts as a bar graph.\n",
        "plt.xlabel('Month')  # Label the x-axis.\n",
        "plt.ylabel('Count of Service Orders')  # Label the y-axis.\n",
        "plt.title('Distribution of Service Orders for Heating Component Across Months')  # Set the title of the plot.\n",
        "plt.xticks(ticks=range(12), labels=['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'], rotation=0)  # Set the x-tick labels to months.\n",
        "plt.savefig('./figures/Frequency/heating/monthly_distribution.png')  # Save the figure as a PNG file.\n",
        "plt.show()  # Display the plot."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f_emGZzaf1bS"
      },
      "outputs": [],
      "source": [
        "# Apply the get_season function to create a new 'Season' column based on the 'Month' column.\n",
        "df_so_cleaned_heating['Season'] = df_so_cleaned_heating['Month'].apply(get_season)\n",
        "\n",
        "# Count the occurrences of each season in the 'Season' column.\n",
        "season_counts = df_so_cleaned_heating['Season'].value_counts()\n",
        "\n",
        "# Create a pie chart to visualize the distribution of service orders by season.\n",
        "plt.figure(figsize=(8, 8))  # Set the figure size for the pie chart.\n",
        "season_counts.plot(kind='pie', autopct='%1.1f%%', colors=['skyblue', 'lightgreen', 'gold', 'salmon'])  # Plot the season counts as a pie chart with percentage annotations.\n",
        "plt.ylabel('')  # Hide the y-label for aesthetic purposes.\n",
        "plt.title('Distribution of Service Orders for Heating Component Across Seasons')  # Set the title of the pie chart.\n",
        "plt.savefig('./figures/Frequency/heating/seasonal_distribution.png')  # Save the figure as a PNG file.\n",
        "plt.show()  # Display the pie chart."
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyMilTNCTJlUeqoYJjMI4GSl",
      "collapsed_sections": [
        "7595UIXVFovr",
        "MFYjHZpCGigl",
        "O4Lz1Pbvl9iI",
        "VDJL0LMGIC6r",
        "j9xAymHkIRnv",
        "o18HZVcFGRF4",
        "Yi9-Fr-CunVf",
        "AzaIeJSxRzNO",
        "0RcmlB7A0SEf",
        "estESIcU0byi",
        "sEJi49ya0zFE"
      ],
      "mount_file_id": "1Ly7hHQs69jI9p6bok77mVVBNRwUuwMff",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}