DataAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import the relevant libraries\n",
    "import requests\n",
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import datetime\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read the properties csv using the pandas read_csv() method\n",
    "properties_raw_data = pd.read_csv('properties.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "﻿id             int64\n",
       "building        int64\n",
       "date_sale      object\n",
       "type           object\n",
       "property#       int64\n",
       "area          float64\n",
       "price          object\n",
       "status         object\n",
       "customerid     object\n",
       "dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here are the data types\n",
    "properties_raw_data.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Customers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read the properties csv using the pandas read_csv() method\n",
    "customers_raw_data = pd.read_csv('customers.csv', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "﻿customerid          object\n",
       "entity               object\n",
       "name                 object\n",
       "surname              object\n",
       "birth_date           object\n",
       "sex                  object\n",
       "country              object\n",
       "state                object\n",
       "purpose              object\n",
       "deal_satisfaction     int64\n",
       "mortgage             object\n",
       "source               object\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customers_raw_data.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning and Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a new variable that replicates the original data is crucial. \n",
    "# This allows us to rerun the code from this point if required.\n",
    "\n",
    "properties = properties_raw_data.copy()\n",
    "\n",
    "# This is effectively a natural 'checkpoint' in our data processing.\n",
    "# Moving forward, we'll manipulate the 'properties' variable and refrain from touching 'properties_raw_data'.\n",
    "# This ensures that we always have access to the pristine data if we need to revert our changes.\n",
    "# Often, developers refer to such variables as 'properties_backup' or similar, \n",
    "# indicating that it holds a safeguarded copy of the original data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>﻿id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>price</th>\n",
       "      <th>status</th>\n",
       "      <th>customerid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>267.000000</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>266</td>\n",
       "      <td>2</td>\n",
       "      <td>162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>#NUM!</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$460,001.26</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72</td>\n",
       "      <td>259</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>195</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2975.142322</td>\n",
       "      <td>2.947566</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.580524</td>\n",
       "      <td>936.221311</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1392.197474</td>\n",
       "      <td>1.391794</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.361437</td>\n",
       "      <td>284.894858</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1002.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>410.710000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2015.500000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.500000</td>\n",
       "      <td>756.210000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3024.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>798.280000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4031.500000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>1121.950000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5052.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>1942.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                ﻿id    building date_sale       type   property#         area  \\\n",
       "count    267.000000  267.000000       267        267  267.000000   267.000000   \n",
       "unique          NaN         NaN        44          2         NaN          NaN   \n",
       "top             NaN         NaN     #NUM!  Apartment         NaN          NaN   \n",
       "freq            NaN         NaN        72        259         NaN          NaN   \n",
       "mean    2975.142322    2.947566       NaN        NaN   27.580524   936.221311   \n",
       "std     1392.197474    1.391794       NaN        NaN   15.361437   284.894858   \n",
       "min     1002.000000    1.000000       NaN        NaN    1.000000   410.710000   \n",
       "25%     2015.500000    2.000000       NaN        NaN   14.500000   756.210000   \n",
       "50%     3024.000000    3.000000       NaN        NaN   28.000000   798.280000   \n",
       "75%     4031.500000    4.000000       NaN        NaN   40.000000  1121.950000   \n",
       "max     5052.000000    5.000000       NaN        NaN   59.000000  1942.500000   \n",
       "\n",
       "               price  status customerid  \n",
       "count            267     267        195  \n",
       "unique           266       2        162  \n",
       "top     $460,001.26    Sold      C0174   \n",
       "freq               2     195          9  \n",
       "mean             NaN     NaN        NaN  \n",
       "std              NaN     NaN        NaN  \n",
       "min              NaN     NaN        NaN  \n",
       "25%              NaN     NaN        NaN  \n",
       "50%              NaN     NaN        NaN  \n",
       "75%              NaN     NaN        NaN  \n",
       "max              NaN     NaN        NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# properties.describe() method allows us to examine statistical summaries of the data.\n",
    "# However, by default, it only includes numeric columns. \n",
    "# To include all variables, we specify 'include=all' in the parameter. \n",
    "\n",
    "properties.describe(include='all')\n",
    "\n",
    "# At first glance, the 'ID' column seems to be well-populated with no missing values.\n",
    "# However, the nature of IDs often renders them insignificant in analysis as they serve merely as identifiers.\n",
    "# The absence of 'unique', 'top', and 'freq' values suggests that 'ID' may be numeric.\n",
    "# The presence of mean and standard deviation further supports this assumption, \n",
    "# since these are common statistical measures for numerical columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "﻿id             int64\n",
       "building        int64\n",
       "date_sale      object\n",
       "type           object\n",
       "property#       int64\n",
       "area          float64\n",
       "price          object\n",
       "status         object\n",
       "customerid     object\n",
       "dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is a good time to check the data types\n",
    "\n",
    "properties.dtypes\n",
    "\n",
    "# Confirming our earlier hypothesis ID seems to be of integer type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # having ID as an integer is sometimes confusing, sometimes dangerous\n",
    "# # we will never treat an ID as a numeric variable, so let's change the data type \n",
    "# # we will cast it as string so no numerical operations can be performed on it\n",
    "#properties['id'] = properties['id'].astype(str)\n",
    "\n",
    "# # unfortunately we get an error when we execute the code above\n",
    "# # pandas does not recognize the column name 'id'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<PandasArray>\n",
       "[  '\\ufeffid',   'building',  'date_sale',       'type',  'property#',\n",
       "       'area',      'price',     'status', 'customerid']\n",
       "Length: 9, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To better understand the issue, let's take a closer look at the column values. \n",
    "# Although we can access these values through the 'values' attribute, it's not recommended with newer versions of pandas.\n",
    "\n",
    "# Instead, we'll use the 'array' attribute to inspect the column values.\n",
    "properties.columns.array\n",
    "\n",
    "# It appears we've stumbled upon an encoding issue.\n",
    "# During data collection, we've unintentionally fetched the byte order mark (or BOM) along with the actual data.\n",
    "# This is essentially metadata that doesn't hold any value for us and indeed, isn't visible when we display the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>price</th>\n",
       "      <th>status</th>\n",
       "      <th>customerid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>11/1/2005</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>$246,172.68</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>10/1/2005</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>$246,331.90</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>7/1/2007</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>$209,280.91</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>12/1/2007</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>$452,667.01</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>11/1/2004</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>$467,083.31</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0014</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id  building  date_sale       type  property#     area         price  \\\n",
       "0  1030         1  11/1/2005  Apartment         30   743.09  $246,172.68    \n",
       "1  1029         1  10/1/2005  Apartment         29   756.21  $246,331.90    \n",
       "2  2002         2   7/1/2007  Apartment          2   587.28  $209,280.91    \n",
       "3  2031         2  12/1/2007  Apartment         31  1604.75  $452,667.01    \n",
       "4  1049         1  11/1/2004  Apartment         49  1375.45  $467,083.31    \n",
       "\n",
       "   status customerid  \n",
       "0   Sold      C0028   \n",
       "1   Sold      C0027   \n",
       "2   Sold      C0112   \n",
       "3   Sold      C0160   \n",
       "4   Sold      C0014   "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To address this issue, we have a couple of options: we could revisit the data collection phase or simply rename the column. \n",
    "# There are various ways to rename a column, but one straightforward method is to use the 'rename' function. \n",
    "# This function accepts a dictionary as an argument, mapping the original column names (keys) to their new names (values).\n",
    "properties = properties.rename(columns= {'\\ufeffid':'id'})\n",
    "# Let's preview our dataset to confirm the changes.\n",
    "properties.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      1030\n",
       "1      1029\n",
       "2      2002\n",
       "3      2031\n",
       "4      1049\n",
       "       ... \n",
       "262    5044\n",
       "263    5047\n",
       "264    5048\n",
       "265    5050\n",
       "266    5051\n",
       "Name: id, Length: 267, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Excellent, with the column name issue resolved, we can now proceed to convert 'id' to a string.\n",
    "\n",
    "# As discussed earlier, maintaining 'ID' as an integer can lead to confusion or unintended operations. \n",
    "# Since 'ID' serves as an identifier rather than a numeric variable, it's best to change its data type.\n",
    "# By converting it to a string, we ensure no numerical operations can be inadvertently performed on it. \n",
    "properties['id'] = properties['id'].astype(str)\n",
    "# Let's examine the 'id' column to confirm our changes.\n",
    "properties['id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             object\n",
       "building       object\n",
       "date_sale      object\n",
       "type           object\n",
       "property#      object\n",
       "area          float64\n",
       "price          object\n",
       "status         object\n",
       "customerid     object\n",
       "dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The same applies for the 'building' and 'property#' variables\n",
    "properties['building'] = properties['building'].astype(str)\n",
    "properties['property#'] = properties['property#'].astype(str)\n",
    "\n",
    "# let's check the new data types\n",
    "properties.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>price</th>\n",
       "      <th>status</th>\n",
       "      <th>customerid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>240</td>\n",
       "      <td>5</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "      <td>59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>266</td>\n",
       "      <td>2</td>\n",
       "      <td>162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>5041</td>\n",
       "      <td>3</td>\n",
       "      <td>#NUM!</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$460,001.26</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>2</td>\n",
       "      <td>63</td>\n",
       "      <td>72</td>\n",
       "      <td>259</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>195</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>936.221311</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>284.894858</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>410.710000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>756.210000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>798.280000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1121.950000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1942.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id building date_sale       type property#         area  \\\n",
       "count    267      267       267        267       267   267.000000   \n",
       "unique   240        5        44          2        59          NaN   \n",
       "top     5041        3     #NUM!  Apartment        34          NaN   \n",
       "freq       2       63        72        259         7          NaN   \n",
       "mean     NaN      NaN       NaN        NaN       NaN   936.221311   \n",
       "std      NaN      NaN       NaN        NaN       NaN   284.894858   \n",
       "min      NaN      NaN       NaN        NaN       NaN   410.710000   \n",
       "25%      NaN      NaN       NaN        NaN       NaN   756.210000   \n",
       "50%      NaN      NaN       NaN        NaN       NaN   798.280000   \n",
       "75%      NaN      NaN       NaN        NaN       NaN  1121.950000   \n",
       "max      NaN      NaN       NaN        NaN       NaN  1942.500000   \n",
       "\n",
       "               price  status customerid  \n",
       "count            267     267        195  \n",
       "unique           266       2        162  \n",
       "top     $460,001.26    Sold      C0174   \n",
       "freq               2     195          9  \n",
       "mean             NaN     NaN        NaN  \n",
       "std              NaN     NaN        NaN  \n",
       "min              NaN     NaN        NaN  \n",
       "25%              NaN     NaN        NaN  \n",
       "50%              NaN     NaN        NaN  \n",
       "75%              NaN     NaN        NaN  \n",
       "max              NaN     NaN        NaN  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "properties.describe(include='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['11/1/2005', '10/1/2005', '7/1/2007', '12/1/2007', '11/1/2004',\n",
       "       '9/1/2007', '1/1/2008', '6/1/2006', '3/1/2006', '10/1/2004',\n",
       "       '8/1/2006', '10/1/2007', '11/1/2006', '4/1/2007', '10/1/2006',\n",
       "       '12/1/2006', '3/1/2005', '3/1/2007', '9/1/2006', '1/1/2007',\n",
       "       '5/1/2007', '7/1/2006', '5/1/2008', '11/1/2007', '1/1/2005',\n",
       "       '8/1/2007', '6/1/2005', '2/1/2007', '6/1/2007', '2/1/2005',\n",
       "       '8/1/2004', '12/1/2008', '5/1/2010', '4/1/2006', '8/1/2005',\n",
       "       '6/1/2004', '12/1/2005', '3/1/2004', '2/1/2006', '4/1/2005',\n",
       "       '5/1/2006', '#NUM!', '9/1/2005', '7/1/2005'], dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Observing that all variables, except 'area', are of the 'object' data type might raise some concerns.\n",
    "# We previously encountered columns like 'id', 'building', and 'property#' that were initially interpreted as integers \n",
    "# but required conversion to strings.\n",
    "# This invites the question whether there might be other variables incorrectly classified as 'object' type.\n",
    "\n",
    "# To investigate this, let's proceed methodically.\n",
    "# Focusing on 'date of sale', we notice several issues:\n",
    "# Among 267 entries, there are 44 unique ones, which is unusual and warrants a closer look.\n",
    "\n",
    "properties['date_sale'].unique()\n",
    "\n",
    "# Our investigation reveals multiple problems:\n",
    "\n",
    "# 1. Some entries are '#NUM!', a placeholder for missing or erroneous values, likely carried over from an Excel file.\n",
    "\n",
    "# 2. Every date is listed as the first of the month, regardless of the actual month. \n",
    "# This suggests that our data may only include information about the month of the transaction, rather than the precise date.\n",
    "\n",
    "# 3. The 'date_sale' Series is currently classified as an 'object' type, \n",
    "# meaning the dates are treated as strings instead of actual dates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['11/1/2005', '10/1/2005', '7/1/2007', '12/1/2007', '11/1/2004',\n",
       "       '9/1/2007', '1/1/2008', '6/1/2006', '3/1/2006', '10/1/2004',\n",
       "       '8/1/2006', '10/1/2007', '11/1/2006', '4/1/2007', '10/1/2006',\n",
       "       '12/1/2006', '3/1/2005', '3/1/2007', '9/1/2006', '1/1/2007',\n",
       "       '5/1/2007', '7/1/2006', '5/1/2008', '11/1/2007', '1/1/2005',\n",
       "       '8/1/2007', '6/1/2005', '2/1/2007', '6/1/2007', '2/1/2005',\n",
       "       '8/1/2004', '12/1/2008', '5/1/2010', '4/1/2006', '8/1/2005',\n",
       "       '6/1/2004', '12/1/2005', '3/1/2004', '2/1/2006', '4/1/2005',\n",
       "       '5/1/2006', <NA>, '9/1/2005', '7/1/2005'], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's address these issues step by step.\n",
    "# The first task is to replace the Excel-specific missing value indicator '#NUM!' with a format \n",
    "# that pandas can understand as a null value, i.e., 'pd.NA'.\n",
    "# A convenient way to implement this substitution is using the 'np.where' function from the NumPy library.\n",
    "properties['date_sale'] = np.where(properties['date_sale']=='#NUM!', pd.NA, properties['date_sale'])\n",
    "\n",
    "# Please note, 'pd.NA' appears as '<NA>' when displayed.\n",
    "\n",
    "# Upon rechecking the unique entries, we can observe that the '#NUM!' placeholders have been successfully replaced with 'pd.NA'.\n",
    "properties['date_sale'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     2005-11-01\n",
       "1     2005-10-01\n",
       "2     2007-07-01\n",
       "3     2007-12-01\n",
       "4     2004-11-01\n",
       "         ...    \n",
       "262          NaT\n",
       "263          NaT\n",
       "264          NaT\n",
       "265          NaT\n",
       "266          NaT\n",
       "Name: date_sale, Length: 267, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The next step is to convert the 'date_sale' column to a date type. \n",
    "# This makes subsequent manipulations and analysis easier.\n",
    "properties['date_sale'] = pd.to_datetime(properties['date_sale'])\n",
    "# Let's review the changes.\n",
    "properties['date_sale']\n",
    "\n",
    "# Note that the default precision of datetime is nanoseconds, represented as '[ns]'.\n",
    "# Also, since we're using the dedicated pandas null type, 'pd.NA', \n",
    "# the datetime function knows to convert these into 'NaT' (not a timestamp)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>price</th>\n",
       "      <th>status</th>\n",
       "      <th>customerid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>195</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>240</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>266</td>\n",
       "      <td>2</td>\n",
       "      <td>162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>5041</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>$460,001.26</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>2</td>\n",
       "      <td>63</td>\n",
       "      <td>NaN</td>\n",
       "      <td>259</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>195</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2006-11-08 14:38:46.153846272</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>936.221311</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2004-03-01 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>410.710000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2006-04-16 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>756.210000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2007-03-01 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>798.280000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2007-09-01 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1121.950000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2010-05-01 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1942.500000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>284.894858</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id building                      date_sale       type property#  \\\n",
       "count    267      267                            195        267       267   \n",
       "unique   240        5                            NaN          2        59   \n",
       "top     5041        3                            NaN  Apartment        34   \n",
       "freq       2       63                            NaN        259         7   \n",
       "mean     NaN      NaN  2006-11-08 14:38:46.153846272        NaN       NaN   \n",
       "min      NaN      NaN            2004-03-01 00:00:00        NaN       NaN   \n",
       "25%      NaN      NaN            2006-04-16 00:00:00        NaN       NaN   \n",
       "50%      NaN      NaN            2007-03-01 00:00:00        NaN       NaN   \n",
       "75%      NaN      NaN            2007-09-01 00:00:00        NaN       NaN   \n",
       "max      NaN      NaN            2010-05-01 00:00:00        NaN       NaN   \n",
       "std      NaN      NaN                            NaN        NaN       NaN   \n",
       "\n",
       "               area         price  status customerid  \n",
       "count    267.000000           267     267        195  \n",
       "unique          NaN           266       2        162  \n",
       "top             NaN  $460,001.26    Sold      C0174   \n",
       "freq            NaN             2     195          9  \n",
       "mean     936.221311           NaN     NaN        NaN  \n",
       "min      410.710000           NaN     NaN        NaN  \n",
       "25%      756.210000           NaN     NaN        NaN  \n",
       "50%      798.280000           NaN     NaN        NaN  \n",
       "75%     1121.950000           NaN     NaN        NaN  \n",
       "max     1942.500000           NaN     NaN        NaN  \n",
       "std      284.894858           NaN     NaN        NaN  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# After the conversion, we see that the total count of 'date_sale' entries is now 195, while unique values count dropped to 43.\n",
    "# This change is due to the replacement of empty strings with 'NaT', \n",
    "# resulting in one less unique value and a total of 72 (267-195) missing values.\n",
    "\n",
    "properties.describe(include=\"all\", datetime_is_numeric=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Apartment', 'Office'], dtype=object)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "properties['type'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, let's turn our attention to the 'type' column. \n",
    "# Although it appears to be well-formatted, we'll standardize it further by converting all its entries to lowercase.\n",
    "# This can be achieved using the .lower() method applied on the string representation of the column.\n",
    "\n",
    "properties['type'] = properties['type'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>price</th>\n",
       "      <th>status</th>\n",
       "      <th>customerid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>$246,172.68</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>$246,331.90</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>$209,280.91</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>$452,667.01</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>$467,083.31</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0014</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id building  date_sale       type property#     area         price  \\\n",
       "0  1030        1 2005-11-01  apartment        30   743.09  $246,172.68    \n",
       "1  1029        1 2005-10-01  apartment        29   756.21  $246,331.90    \n",
       "2  2002        2 2007-07-01  apartment         2   587.28  $209,280.91    \n",
       "3  2031        2 2007-12-01  apartment        31  1604.75  $452,667.01    \n",
       "4  1049        1 2004-11-01  apartment        49  1375.45  $467,083.31    \n",
       "\n",
       "   status customerid  \n",
       "0   Sold      C0028   \n",
       "1   Sold      C0027   \n",
       "2   Sold      C0112   \n",
       "3   Sold      C0160   \n",
       "4   Sold      C0014   "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "properties.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                    object\n",
       "building              object\n",
       "date_sale     datetime64[ns]\n",
       "type                  object\n",
       "property#             object\n",
       "area                 float64\n",
       "price                 object\n",
       "status                object\n",
       "customerid            object\n",
       "dtype: object"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "properties.dtypes\n",
    "\n",
    "# interestingly, price is also of object type and we definitely want it to be a float"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>price</th>\n",
       "      <th>status</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>$246,172.68</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246,172.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>$246,331.90</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246,331.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>$209,280.91</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209,280.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>$452,667.01</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452,667.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>$467,083.31</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467,083.31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id building  date_sale       type property#     area         price  \\\n",
       "0  1030        1 2005-11-01  apartment        30   743.09  $246,172.68    \n",
       "1  1029        1 2005-10-01  apartment        29   756.21  $246,331.90    \n",
       "2  2002        2 2007-07-01  apartment         2   587.28  $209,280.91    \n",
       "3  2031        2 2007-12-01  apartment        31  1604.75  $452,667.01    \n",
       "4  1049        1 2004-11-01  apartment        49  1375.45  $467,083.31    \n",
       "\n",
       "   status customerid       price$  \n",
       "0   Sold      C0028   246,172.68   \n",
       "1   Sold      C0027   246,331.90   \n",
       "2   Sold      C0112   209,280.91   \n",
       "3   Sold      C0160   452,667.01   \n",
       "4   Sold      C0014   467,083.31   "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the reason for this is the $ sign, so let's strip it \n",
    "# instead of overwriting the price, let's create a new Series 'price$'\n",
    "properties['price$'] = properties['price'].str.strip('$')\n",
    "properties.head()\n",
    "\n",
    "# The '$' symbol in the 'price' column is causing it to be treated as a string. \n",
    "# To fix this, we'll strip the '$' sign and create a new Series 'price$'.\n",
    "properties['price$'] = properties['price'].str.strip('$')\n",
    "# Let's take a peek at the dataset to see the changes.\n",
    "properties.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Next, we intend to convert the 'price$' column to a float data type.\n",
    "#properties['price$'] = properties['price$'].astype(float)\n",
    "\n",
    "# Unfortunately, we encounter an error because of the thousands separator ',' still present in the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>price</th>\n",
       "      <th>status</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>$246,172.68</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>$246,331.90</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>$209,280.91</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>$452,667.01</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>$467,083.31</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id building  date_sale       type property#     area         price  \\\n",
       "0  1030        1 2005-11-01  apartment        30   743.09  $246,172.68    \n",
       "1  1029        1 2005-10-01  apartment        29   756.21  $246,331.90    \n",
       "2  2002        2 2007-07-01  apartment         2   587.28  $209,280.91    \n",
       "3  2031        2 2007-12-01  apartment        31  1604.75  $452,667.01    \n",
       "4  1049        1 2004-11-01  apartment        49  1375.45  $467,083.31    \n",
       "\n",
       "   status customerid      price$  \n",
       "0   Sold      C0028   246172.68   \n",
       "1   Sold      C0027   246331.90   \n",
       "2   Sold      C0112   209280.91   \n",
       "3   Sold      C0160   452667.01   \n",
       "4   Sold      C0014   467083.31   "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Since the .strip() method only removes characters from the beginning and end of a string, \n",
    "# we need a different approach to remove symbols located in the middle.\n",
    "# The .replace() method is apt for this task, replacing all instances of ',' with an empty string.\n",
    "# It's important to set regex=True here, as the comma has several interpretations in Python.\n",
    "properties['price$']= properties['price$'].replace(\",\",\"\",regex=True) \n",
    "# Let's preview the data once more to check our progress.\n",
    "properties.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now we can carry on with casting price into a float\n",
    "properties['price$'] = properties['price$'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>status</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>Sold</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id building  date_sale       type property#     area  status customerid  \\\n",
       "0  1030        1 2005-11-01  apartment        30   743.09   Sold      C0028    \n",
       "1  1029        1 2005-10-01  apartment        29   756.21   Sold      C0027    \n",
       "2  2002        2 2007-07-01  apartment         2   587.28   Sold      C0112    \n",
       "3  2031        2 2007-12-01  apartment        31  1604.75   Sold      C0160    \n",
       "4  1049        1 2004-11-01  apartment        49  1375.45   Sold      C0014    \n",
       "\n",
       "      price$  \n",
       "0  246172.68  \n",
       "1  246331.90  \n",
       "2  209280.91  \n",
       "3  452667.01  \n",
       "4  467083.31  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# I believe we can now safely drop price from our dataframe\n",
    "properties = properties.drop(['price'],axis=1)\n",
    "properties.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                    object\n",
       "building              object\n",
       "date_sale     datetime64[ns]\n",
       "type                  object\n",
       "property#             object\n",
       "area                 float64\n",
       "status                object\n",
       "customerid            object\n",
       "price$               float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We can also check the data types at this point\n",
    "properties.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([' Sold ', '-'], dtype=object)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The next column for review is 'status', which is critical to our analysis.\n",
    "# Let's examine the unique values in this column. \n",
    "\n",
    "properties['status'].unique()\n",
    "\n",
    "# Good news: there don't appear to be any missing values.\n",
    "# However, we notice some inconsistencies: the word 'sold' is surrounded by spaces and is capitalized, \n",
    "# which may not be ideal for data uniformity and analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['sold', '-'], dtype=object)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To ensure uniformity and ease of analysis, we'll remove the surrounding spaces using the .strip() method \n",
    "# and convert the text to lowercase.\n",
    "properties['status'] = properties['status'].str.strip()\n",
    "properties['status'] = properties['status'].str.lower()\n",
    "\n",
    "# Let's examine the unique values again to confirm our changes.\n",
    "properties['status'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# regarding the alternative: \"-\" it most definitely means 'not sold yet'\n",
    "\n",
    "# Now, there are two statuses - 'Sold' or '-'\n",
    "# The dash is a rather strange symbol to have, though\n",
    "\n",
    "# It make sense to me to change the name of \"status\" to \"sold\"\n",
    "# and then change the values to 1s and 0s so they could be Boolean or at least binary\n",
    "\n",
    "properties = properties.rename(columns= {'status':'sold'})\n",
    "\n",
    "# The symbol '-' likely represents properties not yet sold.\n",
    "\n",
    "# Currently, we have two statuses: 'sold' and '-'. However, using a dash to represent unsold properties is somewhat unusual.\n",
    "\n",
    "# A more intuitive approach might be to rename the 'status' column as 'sold' and change its values to 1s (for sold properties)\n",
    "# and 0s (for unsold properties), essentially creating a binary classification.\n",
    "\n",
    "properties = properties.rename(columns= {'status':'sold'})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0], dtype=int64)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To implement this, we can use the .map() method. This method is often used in data science to provide a numeric representation\n",
    "# of categorical data, allowing mathematical operations.Like .rename(), .map() is applied to a Series and takes a dictionary \n",
    "# as an argument, where the keys represent the original values and the dictionary values are the new values.\n",
    "# Unlike '.rename()', however, we need to specify all distinct options as keys. \n",
    "# Any options not explicitly mapped will be transformed into NaN.\n",
    "\n",
    "properties['sold'] = properties['sold'].map({'sold':1,'-':0})\n",
    "# Let's view the unique values in 'sold' to confirm our transformations.\n",
    "properties['sold'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                    object\n",
       "building              object\n",
       "date_sale     datetime64[ns]\n",
       "type                  object\n",
       "property#             object\n",
       "area                 float64\n",
       "sold                   int64\n",
       "customerid            object\n",
       "price$               float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "properties.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "building       0\n",
       "date_sale     72\n",
       "type           0\n",
       "property#      0\n",
       "area           0\n",
       "sold           0\n",
       "customerid    72\n",
       "price$         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finally we can check for missing values\n",
    "properties.isna().sum()\n",
    "\n",
    "# it seems there are none (apart from date of sale)\n",
    "# to check if that's true, you can investigate each column with the .unique() method\n",
    "\n",
    "# It's time to check for missing values in our cleaned dataset.\n",
    "properties.isna().sum()\n",
    "\n",
    "# From the initial glance, it seems there are no missing values, apart from the 'date_sale' column. \n",
    "# To be certain, you can verify this by inspecting each column with the .unique() method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>1</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>1</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>1</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>1</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>1</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>5044</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>44</td>\n",
       "      <td>1238.58</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>322610.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>5047</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>47</td>\n",
       "      <td>794.52</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>279191.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>5048</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>48</td>\n",
       "      <td>1013.27</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287996.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>5050</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>50</td>\n",
       "      <td>1074.71</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>365868.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>5051</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>51</td>\n",
       "      <td>789.25</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>199216.40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>267 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id building  date_sale       type property#     area  sold customerid  \\\n",
       "0    1030        1 2005-11-01  apartment        30   743.09     1     C0028    \n",
       "1    1029        1 2005-10-01  apartment        29   756.21     1     C0027    \n",
       "2    2002        2 2007-07-01  apartment         2   587.28     1     C0112    \n",
       "3    2031        2 2007-12-01  apartment        31  1604.75     1     C0160    \n",
       "4    1049        1 2004-11-01  apartment        49  1375.45     1     C0014    \n",
       "..    ...      ...        ...        ...       ...      ...   ...        ...   \n",
       "262  5044        5        NaT  apartment        44  1238.58     0        NaN   \n",
       "263  5047        5        NaT  apartment        47   794.52     0        NaN   \n",
       "264  5048        5        NaT  apartment        48  1013.27     0        NaN   \n",
       "265  5050        5        NaT  apartment        50  1074.71     0        NaN   \n",
       "266  5051        5        NaT  apartment        51   789.25     0        NaN   \n",
       "\n",
       "        price$  \n",
       "0    246172.68  \n",
       "1    246331.90  \n",
       "2    209280.91  \n",
       "3    452667.01  \n",
       "4    467083.31  \n",
       "..         ...  \n",
       "262  322610.74  \n",
       "263  279191.26  \n",
       "264  287996.53  \n",
       "265  365868.78  \n",
       "266  199216.40  \n",
       "\n",
       "[267 rows x 9 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To wrap up the cleaning process, let's take a look at our final, tidied dataset.\n",
    "properties"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Customers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For convenience and to avoid having to scroll up every time, let's create a copy of the raw customer data.\n",
    "customers = customers_raw_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<PandasArray>\n",
       "[ '\\ufeffcustomerid',            'entity',              'name',\n",
       "           'surname',        'birth_date',               'sex',\n",
       "           'country',             'state',           'purpose',\n",
       " 'deal_satisfaction',          'mortgage',            'source']\n",
       "Length: 12, dtype: object"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's peek into the structure of the dataset by inspecting the column names. \n",
    "# While we could use the .values attribute to do this, newer versions of pandas recommend using the .array attribute instead.\n",
    "customers.columns.array\n",
    "\n",
    "# Without further examination of the file, we'll commence by scrutinizing the column structure.\n",
    "# We already suspect that pandas might have interpreted the JSON format in an unusual manner.\n",
    "#customers.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>entity</th>\n",
       "      <th>name</th>\n",
       "      <th>surname</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Kareem</td>\n",
       "      <td>Liu</td>\n",
       "      <td>5/11/1968</td>\n",
       "      <td>F</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Trystan</td>\n",
       "      <td>Oconnor</td>\n",
       "      <td>11/26/1962</td>\n",
       "      <td>M</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Kale</td>\n",
       "      <td>Gay</td>\n",
       "      <td>4/7/1959</td>\n",
       "      <td>M</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Agency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Russell</td>\n",
       "      <td>Gross</td>\n",
       "      <td>11/25/1959</td>\n",
       "      <td>M</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>5</td>\n",
       "      <td>No</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>Company</td>\n",
       "      <td>Marleez</td>\n",
       "      <td>Co</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Investment</td>\n",
       "      <td>5</td>\n",
       "      <td>No</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid      entity     name  surname  birth_date  sex country  \\\n",
       "0      C0110  Individual   Kareem      Liu   5/11/1968    F     USA   \n",
       "1      C0010  Individual  Trystan  Oconnor  11/26/1962    M     USA   \n",
       "2      C0132  Individual     Kale      Gay    4/7/1959    M     USA   \n",
       "3      C0137  Individual  Russell    Gross  11/25/1959    M     USA   \n",
       "4      C0174     Company  Marleez       Co         NaN  NaN    USA    \n",
       "\n",
       "        state     purpose  deal_satisfaction mortgage   source  \n",
       "0  California        Home                  4      Yes  Website  \n",
       "1  California        Home                  1       No  Website  \n",
       "2  California        Home                  4      Yes   Agency  \n",
       "3  California        Home                  5       No  Website  \n",
       "4  California  Investment                  5       No  Website  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rename customerid in an appropriate way\n",
    "customers = customers.rename(columns= {'\\ufeffcustomerid':'customerid'})\n",
    "customers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>entity</th>\n",
       "      <th>name</th>\n",
       "      <th>surname</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>162</td>\n",
       "      <td>162</td>\n",
       "      <td>162</td>\n",
       "      <td>162</td>\n",
       "      <td>155</td>\n",
       "      <td>155</td>\n",
       "      <td>162</td>\n",
       "      <td>154</td>\n",
       "      <td>162</td>\n",
       "      <td>162.000000</td>\n",
       "      <td>162</td>\n",
       "      <td>162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>162</td>\n",
       "      <td>2</td>\n",
       "      <td>159</td>\n",
       "      <td>156</td>\n",
       "      <td>151</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>C0110</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Jack</td>\n",
       "      <td>Co</td>\n",
       "      <td>9/14/1966</td>\n",
       "      <td>M</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>155</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>87</td>\n",
       "      <td>146</td>\n",
       "      <td>100</td>\n",
       "      <td>112</td>\n",
       "      <td>NaN</td>\n",
       "      <td>102</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.456790</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.333276</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       customerid      entity  name surname birth_date  sex country  \\\n",
       "count         162         162   162     162        155  155     162   \n",
       "unique        162           2   159     156        151    2       9   \n",
       "top         C0110  Individual  Jack      Co  9/14/1966    M     USA   \n",
       "freq            1         155     2       7          2   87     146   \n",
       "mean          NaN         NaN   NaN     NaN        NaN  NaN     NaN   \n",
       "std           NaN         NaN   NaN     NaN        NaN  NaN     NaN   \n",
       "min           NaN         NaN   NaN     NaN        NaN  NaN     NaN   \n",
       "25%           NaN         NaN   NaN     NaN        NaN  NaN     NaN   \n",
       "50%           NaN         NaN   NaN     NaN        NaN  NaN     NaN   \n",
       "75%           NaN         NaN   NaN     NaN        NaN  NaN     NaN   \n",
       "max           NaN         NaN   NaN     NaN        NaN  NaN     NaN   \n",
       "\n",
       "             state purpose  deal_satisfaction mortgage   source  \n",
       "count          154     162         162.000000      162      162  \n",
       "unique           9       2                NaN        2        3  \n",
       "top     California    Home                NaN       No  Website  \n",
       "freq           100     112                NaN      102       93  \n",
       "mean           NaN     NaN           3.456790      NaN      NaN  \n",
       "std            NaN     NaN           1.333276      NaN      NaN  \n",
       "min            NaN     NaN           1.000000      NaN      NaN  \n",
       "25%            NaN     NaN           3.000000      NaN      NaN  \n",
       "50%            NaN     NaN           4.000000      NaN      NaN  \n",
       "75%            NaN     NaN           5.000000      NaN      NaN  \n",
       "max            NaN     NaN           5.000000      NaN      NaN  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's get an overview of our data. First, we'll examine the descriptive statistics of all variables.\n",
    "customers.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "customerid           object\n",
       "entity               object\n",
       "name                 object\n",
       "surname              object\n",
       "birth_date           object\n",
       "sex                  object\n",
       "country              object\n",
       "state                object\n",
       "purpose              object\n",
       "deal_satisfaction     int64\n",
       "mortgage             object\n",
       "source               object\n",
       "dtype: object"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Next, let's check the data types of our variables.\n",
    "customers.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "customerid           0\n",
       "entity               0\n",
       "name                 0\n",
       "surname              0\n",
       "birth_date           7\n",
       "sex                  7\n",
       "country              0\n",
       "state                8\n",
       "purpose              0\n",
       "deal_satisfaction    0\n",
       "mortgage             0\n",
       "source               0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A glance at the top of the table suggests the presence of missing values, so we need to verify this.\n",
    "customers.isna().sum()\n",
    "\n",
    "# Surprisingly, it appears there are no missing values. This might be due to all values being stored as strings.\n",
    "# Hence, we'll reserve this verification step for a later stage in our analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>individual</th>\n",
       "      <th>name</th>\n",
       "      <th>surname</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Kareem</td>\n",
       "      <td>Liu</td>\n",
       "      <td>5/11/1968</td>\n",
       "      <td>F</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Trystan</td>\n",
       "      <td>Oconnor</td>\n",
       "      <td>11/26/1962</td>\n",
       "      <td>M</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Kale</td>\n",
       "      <td>Gay</td>\n",
       "      <td>4/7/1959</td>\n",
       "      <td>M</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Agency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Russell</td>\n",
       "      <td>Gross</td>\n",
       "      <td>11/25/1959</td>\n",
       "      <td>M</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Home</td>\n",
       "      <td>5</td>\n",
       "      <td>No</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>Company</td>\n",
       "      <td>Marleez</td>\n",
       "      <td>Co</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>Investment</td>\n",
       "      <td>5</td>\n",
       "      <td>No</td>\n",
       "      <td>Website</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid  individual     name  surname  birth_date  sex country  \\\n",
       "0      C0110  Individual   Kareem      Liu   5/11/1968    F     USA   \n",
       "1      C0010  Individual  Trystan  Oconnor  11/26/1962    M     USA   \n",
       "2      C0132  Individual     Kale      Gay    4/7/1959    M     USA   \n",
       "3      C0137  Individual  Russell    Gross  11/25/1959    M     USA   \n",
       "4      C0174     Company  Marleez       Co         NaN  NaN    USA    \n",
       "\n",
       "        state     purpose  deal_satisfaction mortgage   source  \n",
       "0  California        Home                  4      Yes  Website  \n",
       "1  California        Home                  1       No  Website  \n",
       "2  California        Home                  4      Yes   Agency  \n",
       "3  California        Home                  5       No  Website  \n",
       "4  California  Investment                  5       No  Website  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We'll start by renaming 'entity' and 'purpose' to names that could potentially represent an indicator variable.\n",
    "customers = customers.rename(columns= {'entity':'individual'})\n",
    "customers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Individual', 'Company'], dtype=object)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Next, let's focus on the 'individual' variable and observe its unique values.\n",
    "customers['individual'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To make 'individual' a Boolean-like variable (at least conceptually), let's map 'Individual' to 1 and 'Company' to 0.\n",
    "customers['individual'] = customers['individual'].map({'Individual':1,'Company':0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['F', 'M', nan], dtype=object)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now, let's examine the 'sex' variable.\n",
    "customers['sex'].unique()\n",
    "\n",
    "# It appears there are three options: 'F', 'M', and an empty string. \n",
    "# We should map the empty string to NaN to indicate missing data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We want to keep 'sex' as an 'object' variable, so we'll map 'F' and 'M' to '1' and '0' respectively. \n",
    "# We'll use string format, not integers.\n",
    "customers['sex'] = customers['sex'].map({'F':'1','M':'0', '':pd.NA})\n",
    "\n",
    "# Please note, if we used the np.where method here, it would have modified our data in an undesirable way.\n",
    "# All missing values would fall into the 'other' category, which in this case is '0'.\n",
    "# customers['sex'] = np.where(customers['sex']=='F','1','0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "customerid           0\n",
       "individual           0\n",
       "name                 0\n",
       "surname              0\n",
       "birth_date           7\n",
       "sex                  7\n",
       "country              0\n",
       "state                8\n",
       "purpose              0\n",
       "deal_satisfaction    0\n",
       "mortgage             0\n",
       "source               0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's verify the missing data.\n",
    "customers.isna().sum()\n",
    "\n",
    "# Now, the missing values are easy to identify. Remember, np.where() is a more aggressive approach \n",
    "# that could potentially lead to inappropriate mapping of missing values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>individual</th>\n",
       "      <th>name</th>\n",
       "      <th>surname</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>1</td>\n",
       "      <td>Kareem</td>\n",
       "      <td>Liu</td>\n",
       "      <td>5/11/1968</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>1</td>\n",
       "      <td>Trystan</td>\n",
       "      <td>Oconnor</td>\n",
       "      <td>11/26/1962</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>1</td>\n",
       "      <td>Kale</td>\n",
       "      <td>Gay</td>\n",
       "      <td>4/7/1959</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>agency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>1</td>\n",
       "      <td>Russell</td>\n",
       "      <td>Gross</td>\n",
       "      <td>11/25/1959</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5</td>\n",
       "      <td>No</td>\n",
       "      <td>website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>0</td>\n",
       "      <td>Marleez</td>\n",
       "      <td>Co</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>5</td>\n",
       "      <td>No</td>\n",
       "      <td>website</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid  individual     name  surname  birth_date  sex country  \\\n",
       "0      C0110           1   Kareem      Liu   5/11/1968    1     USA   \n",
       "1      C0010           1  Trystan  Oconnor  11/26/1962    0     USA   \n",
       "2      C0132           1     Kale      Gay    4/7/1959    0     USA   \n",
       "3      C0137           1  Russell    Gross  11/25/1959    0     USA   \n",
       "4      C0174           0  Marleez       Co         NaN  NaN    USA    \n",
       "\n",
       "        state     purpose  deal_satisfaction mortgage   source  \n",
       "0  California        home                  4      Yes  website  \n",
       "1  California        home                  1       No  website  \n",
       "2  California        home                  4      Yes   agency  \n",
       "3  California        home                  5       No  website  \n",
       "4  California  investment                  5       No  website  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# It seems 'state' and 'purpose' are in good shape, but it would be useful to convert their values to lowercase.\n",
    "# This can be done using the .lower() method.\n",
    "\n",
    "# We'll apply this to the 'purpose' and 'source' columns.\n",
    "customers['purpose'] = customers['purpose'].str.lower()\n",
    "customers['source'] = customers['source'].str.lower()\n",
    "\n",
    "# Let's have a look at the changes.\n",
    "customers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>individual</th>\n",
       "      <th>name</th>\n",
       "      <th>surname</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>1</td>\n",
       "      <td>Kareem</td>\n",
       "      <td>Liu</td>\n",
       "      <td>5/11/1968</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>1</td>\n",
       "      <td>Trystan</td>\n",
       "      <td>Oconnor</td>\n",
       "      <td>11/26/1962</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>1</td>\n",
       "      <td>Kale</td>\n",
       "      <td>Gay</td>\n",
       "      <td>4/7/1959</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>agency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>1</td>\n",
       "      <td>Russell</td>\n",
       "      <td>Gross</td>\n",
       "      <td>11/25/1959</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>0</td>\n",
       "      <td>Marleez</td>\n",
       "      <td>Co</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid  individual     name  surname  birth_date  sex country  \\\n",
       "0      C0110           1   Kareem      Liu   5/11/1968    1     USA   \n",
       "1      C0010           1  Trystan  Oconnor  11/26/1962    0     USA   \n",
       "2      C0132           1     Kale      Gay    4/7/1959    0     USA   \n",
       "3      C0137           1  Russell    Gross  11/25/1959    0     USA   \n",
       "4      C0174           0  Marleez       Co         NaN  NaN    USA    \n",
       "\n",
       "        state     purpose  deal_satisfaction  mortgage   source  \n",
       "0  California        home                  4         1  website  \n",
       "1  California        home                  1         0  website  \n",
       "2  California        home                  4         1   agency  \n",
       "3  California        home                  5         0  website  \n",
       "4  California  investment                  5         0  website  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now, let's move on to 'mortgage'. We'll map 'No' to 0 and 'Yes' to 1.\n",
    "customers['mortgage'] = customers['mortgage'].map({'No':0,'Yes':1})\n",
    "\n",
    "# Let's see how the dataframe looks now.\n",
    "customers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "customerid           0\n",
       "individual           0\n",
       "name                 0\n",
       "surname              0\n",
       "birth_date           7\n",
       "sex                  7\n",
       "country              0\n",
       "state                8\n",
       "purpose              0\n",
       "deal_satisfaction    0\n",
       "mortgage             0\n",
       "source               0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lastly, let's check for any missing values.\n",
    "customers.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>individual</th>\n",
       "      <th>name</th>\n",
       "      <th>surname</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>1</td>\n",
       "      <td>Kareem</td>\n",
       "      <td>Liu</td>\n",
       "      <td>5/11/1968</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>website</td>\n",
       "      <td>Kareem Liu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>1</td>\n",
       "      <td>Trystan</td>\n",
       "      <td>Oconnor</td>\n",
       "      <td>11/26/1962</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Trystan Oconnor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>1</td>\n",
       "      <td>Kale</td>\n",
       "      <td>Gay</td>\n",
       "      <td>4/7/1959</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>agency</td>\n",
       "      <td>Kale Gay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>1</td>\n",
       "      <td>Russell</td>\n",
       "      <td>Gross</td>\n",
       "      <td>11/25/1959</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Russell Gross</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>0</td>\n",
       "      <td>Marleez</td>\n",
       "      <td>Co</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Marleez Co</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid  individual     name  surname  birth_date  sex country  \\\n",
       "0      C0110           1   Kareem      Liu   5/11/1968    1     USA   \n",
       "1      C0010           1  Trystan  Oconnor  11/26/1962    0     USA   \n",
       "2      C0132           1     Kale      Gay    4/7/1959    0     USA   \n",
       "3      C0137           1  Russell    Gross  11/25/1959    0     USA   \n",
       "4      C0174           0  Marleez       Co         NaN  NaN    USA    \n",
       "\n",
       "        state     purpose  deal_satisfaction  mortgage   source  \\\n",
       "0  California        home                  4         1  website   \n",
       "1  California        home                  1         0  website   \n",
       "2  California        home                  4         1   agency   \n",
       "3  California        home                  5         0  website   \n",
       "4  California  investment                  5         0  website   \n",
       "\n",
       "         full_name  \n",
       "0       Kareem Liu  \n",
       "1  Trystan Oconnor  \n",
       "2         Kale Gay  \n",
       "3    Russell Gross  \n",
       "4       Marleez Co  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We want to create a new column 'full_name' that combines the 'name' and 'surname' of each customer.\n",
    "# This can be achieved by simply adding the 'name' and 'surname' columns, with a space in between.\n",
    "customers['full_name'] = customers['name'] + \" \" + customers['surname']\n",
    "\n",
    "# Now, let's check how our dataframe looks.\n",
    "customers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>1</td>\n",
       "      <td>5/11/1968</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>website</td>\n",
       "      <td>Kareem Liu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>1</td>\n",
       "      <td>11/26/1962</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Trystan Oconnor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>1</td>\n",
       "      <td>4/7/1959</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>agency</td>\n",
       "      <td>Kale Gay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>1</td>\n",
       "      <td>11/25/1959</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Russell Gross</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Marleez Co</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid  individual  birth_date  sex country       state     purpose  \\\n",
       "0      C0110           1   5/11/1968    1     USA  California        home   \n",
       "1      C0010           1  11/26/1962    0     USA  California        home   \n",
       "2      C0132           1    4/7/1959    0     USA  California        home   \n",
       "3      C0137           1  11/25/1959    0     USA  California        home   \n",
       "4      C0174           0         NaN  NaN    USA   California  investment   \n",
       "\n",
       "   deal_satisfaction  mortgage   source        full_name  \n",
       "0                  4         1  website       Kareem Liu  \n",
       "1                  1         0  website  Trystan Oconnor  \n",
       "2                  4         1   agency         Kale Gay  \n",
       "3                  5         0  website    Russell Gross  \n",
       "4                  5         0  website       Marleez Co  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# After creating the 'full_name' column, we no longer need the individual 'name' and 'surname' columns.\n",
    "# Let's drop these columns from the dataframe.\n",
    "customers = customers.drop(['name', 'surname'], axis=1)\n",
    "\n",
    "# Let's take a look at the dataframe now.\n",
    "customers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>1</td>\n",
       "      <td>1968-05-11</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>website</td>\n",
       "      <td>Kareem Liu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>1</td>\n",
       "      <td>1962-11-26</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Trystan Oconnor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>1</td>\n",
       "      <td>1959-04-07</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>agency</td>\n",
       "      <td>Kale Gay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>1</td>\n",
       "      <td>1959-11-25</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Russell Gross</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>0</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Marleez Co</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid  individual birth_date  sex country       state     purpose  \\\n",
       "0      C0110           1 1968-05-11    1     USA  California        home   \n",
       "1      C0010           1 1962-11-26    0     USA  California        home   \n",
       "2      C0132           1 1959-04-07    0     USA  California        home   \n",
       "3      C0137           1 1959-11-25    0     USA  California        home   \n",
       "4      C0174           0        NaT  NaN    USA   California  investment   \n",
       "\n",
       "   deal_satisfaction  mortgage   source        full_name  \n",
       "0                  4         1  website       Kareem Liu  \n",
       "1                  1         0  website  Trystan Oconnor  \n",
       "2                  4         1   agency         Kale Gay  \n",
       "3                  5         0  website    Russell Gross  \n",
       "4                  5         0  website       Marleez Co  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Next, we want to convert 'birth_date' to a datetime format.\n",
    "# This will make it easier for us to work with this data in the future.\n",
    "customers['birth_date'] = pd.to_datetime(customers['birth_date'])\n",
    "\n",
    "# Let's check the dataframe one more time.\n",
    "customers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "customerid                   object\n",
       "individual                    int64\n",
       "birth_date           datetime64[ns]\n",
       "sex                          object\n",
       "country                      object\n",
       "state                        object\n",
       "purpose                      object\n",
       "deal_satisfaction             int64\n",
       "mortgage                      int64\n",
       "source                       object\n",
       "full_name                    object\n",
       "dtype: object"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Finally, let's take a look at the datatypes of our columns to confirm the changes.\n",
    "customers.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "customerid           0\n",
       "individual           0\n",
       "birth_date           7\n",
       "sex                  7\n",
       "country              0\n",
       "state                8\n",
       "purpose              0\n",
       "deal_satisfaction    0\n",
       "mortgage             0\n",
       "source               0\n",
       "full_name            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's review the status of missing values post-preprocessing.\n",
    "customers.isna().sum()\n",
    "\n",
    "# The preprocessing steps resolved many of the data issues, but there may still be unidentified missing values.\n",
    "# These could be entries in our dataset that are technically filled, but represent missing data in a disguised form.\n",
    "\n",
    "# The .unique() method is a powerful tool to identify these hidden missing values.\n",
    "# Remember to take appropriate actions based on what you discover."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>1</td>\n",
       "      <td>1968-05-11</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>website</td>\n",
       "      <td>Kareem Liu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>1</td>\n",
       "      <td>1962-11-26</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Trystan Oconnor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>1</td>\n",
       "      <td>1959-04-07</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>agency</td>\n",
       "      <td>Kale Gay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>1</td>\n",
       "      <td>1959-11-25</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Russell Gross</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>0</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Marleez Co</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid  individual birth_date  sex country       state     purpose  \\\n",
       "0      C0110           1 1968-05-11    1     USA  California        home   \n",
       "1      C0010           1 1962-11-26    0     USA  California        home   \n",
       "2      C0132           1 1959-04-07    0     USA  California        home   \n",
       "3      C0137           1 1959-11-25    0     USA  California        home   \n",
       "4      C0174           0        NaT  NaN    USA   California  investment   \n",
       "\n",
       "   deal_satisfaction  mortgage   source        full_name  \n",
       "0                  4         1  website       Kareem Liu  \n",
       "1                  1         0  website  Trystan Oconnor  \n",
       "2                  4         1   agency         Kale Gay  \n",
       "3                  5         0  website    Russell Gross  \n",
       "4                  5         0  website       Marleez Co  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customers.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine the two tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>1</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>1</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>1</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>1</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>1</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id building  date_sale       type property#     area  sold customerid  \\\n",
       "0  1030        1 2005-11-01  apartment        30   743.09     1     C0028    \n",
       "1  1029        1 2005-10-01  apartment        29   756.21     1     C0027    \n",
       "2  2002        2 2007-07-01  apartment         2   587.28     1     C0112    \n",
       "3  2031        2 2007-12-01  apartment        31  1604.75     1     C0160    \n",
       "4  1049        1 2004-11-01  apartment        49  1375.45     1     C0014    \n",
       "\n",
       "      price$  \n",
       "0  246172.68  \n",
       "1  246331.90  \n",
       "2  209280.91  \n",
       "3  452667.01  \n",
       "4  467083.31  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "properties.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerid</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0110</td>\n",
       "      <td>1</td>\n",
       "      <td>1968-05-11</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>website</td>\n",
       "      <td>Kareem Liu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0010</td>\n",
       "      <td>1</td>\n",
       "      <td>1962-11-26</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Trystan Oconnor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0132</td>\n",
       "      <td>1</td>\n",
       "      <td>1959-04-07</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>agency</td>\n",
       "      <td>Kale Gay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0137</td>\n",
       "      <td>1</td>\n",
       "      <td>1959-11-25</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Russell Gross</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0174</td>\n",
       "      <td>0</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>website</td>\n",
       "      <td>Marleez Co</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  customerid  individual birth_date  sex country       state     purpose  \\\n",
       "0      C0110           1 1968-05-11    1     USA  California        home   \n",
       "1      C0010           1 1962-11-26    0     USA  California        home   \n",
       "2      C0132           1 1959-04-07    0     USA  California        home   \n",
       "3      C0137           1 1959-11-25    0     USA  California        home   \n",
       "4      C0174           0        NaT  NaN    USA   California  investment   \n",
       "\n",
       "   deal_satisfaction  mortgage   source        full_name  \n",
       "0                  4         1  website       Kareem Liu  \n",
       "1                  1         0  website  Trystan Oconnor  \n",
       "2                  4         1   agency         Kale Gay  \n",
       "3                  5         0  website    Russell Gross  \n",
       "4                  5         0  website       Marleez Co  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>1</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>1</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>1</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>1</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>1</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>5044</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>44</td>\n",
       "      <td>1238.58</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>322610.74</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>5047</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>47</td>\n",
       "      <td>794.52</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>279191.26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>5048</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>48</td>\n",
       "      <td>1013.27</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287996.53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>5050</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>50</td>\n",
       "      <td>1074.71</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>365868.78</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>5051</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>51</td>\n",
       "      <td>789.25</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>199216.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>267 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id building  date_sale       type property#     area  sold customerid  \\\n",
       "0    1030        1 2005-11-01  apartment        30   743.09     1     C0028    \n",
       "1    1029        1 2005-10-01  apartment        29   756.21     1     C0027    \n",
       "2    2002        2 2007-07-01  apartment         2   587.28     1     C0112    \n",
       "3    2031        2 2007-12-01  apartment        31  1604.75     1     C0160    \n",
       "4    1049        1 2004-11-01  apartment        49  1375.45     1     C0014    \n",
       "..    ...      ...        ...        ...       ...      ...   ...        ...   \n",
       "262  5044        5        NaT  apartment        44  1238.58     0        NaN   \n",
       "263  5047        5        NaT  apartment        47   794.52     0        NaN   \n",
       "264  5048        5        NaT  apartment        48  1013.27     0        NaN   \n",
       "265  5050        5        NaT  apartment        50  1074.71     0        NaN   \n",
       "266  5051        5        NaT  apartment        51   789.25     0        NaN   \n",
       "\n",
       "        price$  individual birth_date  sex country state purpose  \\\n",
       "0    246172.68         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "1    246331.90         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "2    209280.91         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "3    452667.01         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "4    467083.31         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "..         ...         ...        ...  ...     ...   ...     ...   \n",
       "262  322610.74         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "263  279191.26         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "264  287996.53         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "265  365868.78         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "266  199216.40         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "\n",
       "     deal_satisfaction  mortgage source full_name  \n",
       "0                  NaN       NaN    NaN       NaN  \n",
       "1                  NaN       NaN    NaN       NaN  \n",
       "2                  NaN       NaN    NaN       NaN  \n",
       "3                  NaN       NaN    NaN       NaN  \n",
       "4                  NaN       NaN    NaN       NaN  \n",
       "..                 ...       ...    ...       ...  \n",
       "262                NaN       NaN    NaN       NaN  \n",
       "263                NaN       NaN    NaN       NaN  \n",
       "264                NaN       NaN    NaN       NaN  \n",
       "265                NaN       NaN    NaN       NaN  \n",
       "266                NaN       NaN    NaN       NaN  \n",
       "\n",
       "[267 rows x 19 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Before performing extensive checks, we'll attempt to merge the two datasets \n",
    "# and only later we will investigate any potential corrections that might be required.\n",
    "\n",
    "# We will use the 'merge' function in pandas to combine the two dataframes.\n",
    "# The 'customerid' column is designated as the common key on which the dataframes will be merged. \n",
    "\n",
    "# We opt for a left join, retaining all rows from the 'properties' dataframe and appending matching rows from the 'customers' dataframe. \n",
    "# If there's no match, the resultant dataframe will have 'NA' for the corresponding 'customers' dataframe columns.\n",
    "\n",
    "# It's assumed that a customer entry will only be present in the 'customers' dataframe if they have made a property purchase.\n",
    "pd.merge(properties, customers, on='customerid', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([' C0028 ', ' C0027 ', ' C0112 ', ' C0160 ', ' C0014 ', ' C0125 ',\n",
       "       ' C0166 ', ' C0034 ', ' C0170 ', ' C0009 ', ' C0041 ', ' C0057 ',\n",
       "       ' C0061 ', ' C0089 ', ' C0159 ', ' C0171 ', ' C0042 ', ' C0093 ',\n",
       "       ' C0051 ', ' C0064 ', ' C0128 ', ' C0019 ', ' C0037 ', ' C0127 ',\n",
       "       ' C0018 ', ' C0040 ', ' C0080 ', ' C0083 ', ' C0085 ', ' C0091 ',\n",
       "       ' C0007 ', ' C0048 ', ' C0065 ', ' C0096 ', ' C0164 ', ' C0038 ',\n",
       "       ' C0087 ', ' C0142 ', ' C0015 ', ' C0122 ', ' C0017 ', ' C0020 ',\n",
       "       ' C0086 ', ' C0150 ', ' C0156 ', ' C0169 ', ' C0022 ', ' C0072 ',\n",
       "       ' C0136 ', ' C0149 ', ' C0011 ', ' C0073 ', ' C0110 ', ' C0111 ',\n",
       "       ' C0123 ', ' C0070 ', ' C0075 ', ' C0076 ', ' C0105 ', ' C0135 ',\n",
       "       ' C0153 ', ' C0047 ', ' C0060 ', ' C0066 ', ' C0068 ', ' C0090 ',\n",
       "       ' C0095 ', ' C0151 ', ' C0162 ', ' C0010 ', ' C0054 ', ' C0056 ',\n",
       "       ' C0081 ', ' C0084 ', ' C0016 ', ' C0035 ', ' C0062 ', ' C0099 ',\n",
       "       ' C0114 ', ' C0006 ', ' C0053 ', ' C0069 ', ' C0077 ', ' C0168 ',\n",
       "       ' C0058 ', ' C0107 ', ' C0131 ', ' C0144 ', ' C0098 ', ' C0130 ',\n",
       "       ' C0141 ', ' C0067 ', ' C0082 ', ' C0113 ', ' C0132 ', ' C0137 ',\n",
       "       ' C0155 ', ' C0163 ', ' C0039 ', ' C0071 ', ' C0101 ', ' C0146 ',\n",
       "       ' C0052 ', ' C0063 ', ' C0088 ', ' C0094 ', ' C0165 ', ' C0044 ',\n",
       "       ' C0157 ', ' C0118 ', ' C0119 ', ' C0033 ', ' C0100 ', ' C0133 ',\n",
       "       ' C0175 ', ' C0008 ', ' C0023 ', ' C0108 ', ' C0109 ', ' C0145 ',\n",
       "       ' C0003 ', ' C0004 ', ' C0024 ', ' C0030 ', ' C0147 ', ' C0002 ',\n",
       "       ' C0031 ', ' C0059 ', ' C0140 ', ' C0043 ', ' C0078 ', ' C0124 ',\n",
       "       ' C0049 ', ' C0079 ', ' C0013 ', ' C0139 ', ' C0154 ', ' C0045 ',\n",
       "       ' C0134 ', ' C0138 ', ' C0158 ', ' C0036 ', ' C0074 ', ' C0120 ',\n",
       "       ' C0005 ', ' C0032 ', ' C0126 ', ' C0106 ', ' C0115 ', ' C0129 ',\n",
       "       ' C0103 ', ' C0021 ', ' C0148 ', ' C0172 ', ' C0104 ', nan,\n",
       "       ' C0001 ', ' C0012 ', ' C0025 ', ' C0029 ', ' C0055 ', ' C0121 ',\n",
       "       ' C0174 '], dtype=object)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The merge did not work as intended.\n",
    "# There probably is a problem with the key, so we will investigate further.\n",
    "properties.customerid.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['C0110', 'C0010', 'C0132', 'C0137', 'C0174', 'C0088', 'C0079',\n",
       "       'C0149', 'C0064', 'C0068', 'C0006', 'C0055', 'C0016', 'C0121',\n",
       "       'C0074', 'C0032', 'C0093', 'C0076', 'C0165', 'C0153', 'C0129',\n",
       "       'C0162', 'C0071', 'C0069', 'C0005', 'C0144', 'C0020', 'C0101',\n",
       "       'C0128', 'C0033', 'C0171', 'C0095', 'C0123', 'C0015', 'C0084',\n",
       "       'C0019', 'C0053', 'C0080', 'C0070', 'C0051', 'C0022', 'C0127',\n",
       "       'C0081', 'C0104', 'C0073', 'C0168', 'C0169', 'C0039', 'C0086',\n",
       "       'C0036', 'C0045', 'C0105', 'C0119', 'C0012', 'C0065', 'C0054',\n",
       "       'C0172', 'C0029', 'C0098', 'C0142', 'C0135', 'C0037', 'C0134',\n",
       "       'C0091', 'C0018', 'C0085', 'C0061', 'C0066', 'C0156', 'C0043',\n",
       "       'C0112', 'C0145', 'C0047', 'C0082', 'C0007', 'C0052', 'C0041',\n",
       "       'C0004', 'C0118', 'C0030', 'C0154', 'C0125', 'C0115', 'C0078',\n",
       "       'C0151', 'C0111', 'C0014', 'C0103', 'C0024', 'C0166', 'C0067',\n",
       "       'C0122', 'C0107', 'C0003', 'C0099', 'C0160', 'C0013', 'C0059',\n",
       "       'C0120', 'C0150', 'C0140', 'C0002', 'C0096', 'C0159', 'C0163',\n",
       "       'C0100', 'C0155', 'C0049', 'C0148', 'C0146', 'C0034', 'C0141',\n",
       "       'C0008', 'C0131', 'C0089', 'C0157', 'C0114', 'C0124', 'C0158',\n",
       "       'C0175', 'C0023', 'C0164', 'C0058', 'C0062', 'C0042', 'C0048',\n",
       "       'C0083', 'C0025', 'C0087', 'C0094', 'C0038', 'C0011', 'C0139',\n",
       "       'C0021', 'C0106', 'C0040', 'C0035', 'C0126', 'C0136', 'C0109',\n",
       "       'C0028', 'C0057', 'C0130', 'C0044', 'C0056', 'C0077', 'C0031',\n",
       "       'C0072', 'C0063', 'C0017', 'C0009', 'C0075', 'C0027', 'C0060',\n",
       "       'C0001', 'C0090', 'C0108', 'C0170', 'C0138', 'C0113', 'C0147',\n",
       "       'C0133'], dtype=object)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# It's apparent that the two datasets differ - there are unexpected and unnecessary spaces.\n",
    "# This issue is quite common when handling CSV files.\n",
    "# A more thorough cleaning process could have prevented this issue.\n",
    "customers.customerid.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's trim the unnecessary white space from 'customerid' in the properties dataframe.\n",
    "properties['customerid'] = properties['customerid'].str.strip()\n",
    "\n",
    "# Although there's no noticeable issue with the customers dataframe, we'll trim the 'customerid' column to maintain consistency.\n",
    "customers['customerid'] = customers['customerid'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "customerid           162\n",
       "individual           162\n",
       "birth_date           155\n",
       "sex                  155\n",
       "country              162\n",
       "state                154\n",
       "purpose              162\n",
       "deal_satisfaction    162\n",
       "mortgage             162\n",
       "source               162\n",
       "full_name            162\n",
       "dtype: int64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To ensure a successful merge, we must ensure that 'customerid' in the customers table has only unique values.\n",
    "# Here's the total count of entries in the customers dataframe.\n",
    "customers.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "162"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Here's the count of unique entries in the 'customerid' column of the properties dataframe.\n",
    "properties['customerid'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We've observed that there's an empty space in 'customerid' in the 'properties' dataframe.\n",
    "# Let's replace it with 'NA'. This corresponds to properties that are not yet purchased.\n",
    "properties['customerid'] = np.where(properties['customerid']=='', pd.NA, properties['customerid'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's rename our merged dataframe to 'real_estate_data' for clarity.\n",
    "real_estate_data = pd.merge(properties, customers, on='customerid', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>1</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1986-06-21</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Madalyn Mercer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>1</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1983-02-24</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Lara Carrillo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>1</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1985-12-27</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>client</td>\n",
       "      <td>Donavan Flowers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>1</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1985-12-27</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Darien Dorsey</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>1</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1979-05-15</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>agency</td>\n",
       "      <td>Alessandra Perry</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id building  date_sale       type property#     area  sold customerid  \\\n",
       "0  1030        1 2005-11-01  apartment        30   743.09     1      C0028   \n",
       "1  1029        1 2005-10-01  apartment        29   756.21     1      C0027   \n",
       "2  2002        2 2007-07-01  apartment         2   587.28     1      C0112   \n",
       "3  2031        2 2007-12-01  apartment        31  1604.75     1      C0160   \n",
       "4  1049        1 2004-11-01  apartment        49  1375.45     1      C0014   \n",
       "\n",
       "      price$  individual birth_date sex country       state     purpose  \\\n",
       "0  246172.68         1.0 1986-06-21   1     USA  California        home   \n",
       "1  246331.90         1.0 1983-02-24   1     USA  California        home   \n",
       "2  209280.91         1.0 1985-12-27   0     USA  California        home   \n",
       "3  452667.01         1.0 1985-12-27   0     USA  California  investment   \n",
       "4  467083.31         1.0 1979-05-15   1     USA  California        home   \n",
       "\n",
       "   deal_satisfaction  mortgage   source         full_name  \n",
       "0                5.0       0.0  website    Madalyn Mercer  \n",
       "1                5.0       0.0  website     Lara Carrillo  \n",
       "2                1.0       1.0   client   Donavan Flowers  \n",
       "3                3.0       1.0  website     Darien Dorsey  \n",
       "4                4.0       0.0   agency  Alessandra Perry  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Our data preprocessing step has been successful. Let's take a look at the head of our processed dataframe.\n",
    "real_estate_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>5044</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>44</td>\n",
       "      <td>1238.58</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>322610.74</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>5047</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>47</td>\n",
       "      <td>794.52</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>279191.26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>5048</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>48</td>\n",
       "      <td>1013.27</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287996.53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>5050</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>50</td>\n",
       "      <td>1074.71</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>365868.78</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>5051</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>51</td>\n",
       "      <td>789.25</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>199216.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id building date_sale       type property#     area  sold customerid  \\\n",
       "262  5044        5       NaT  apartment        44  1238.58     0        NaN   \n",
       "263  5047        5       NaT  apartment        47   794.52     0        NaN   \n",
       "264  5048        5       NaT  apartment        48  1013.27     0        NaN   \n",
       "265  5050        5       NaT  apartment        50  1074.71     0        NaN   \n",
       "266  5051        5       NaT  apartment        51   789.25     0        NaN   \n",
       "\n",
       "        price$  individual birth_date  sex country state purpose  \\\n",
       "262  322610.74         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "263  279191.26         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "264  287996.53         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "265  365868.78         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "266  199216.40         NaN        NaT  NaN     NaN   NaN     NaN   \n",
       "\n",
       "     deal_satisfaction  mortgage source full_name  \n",
       "262                NaN       NaN    NaN       NaN  \n",
       "263                NaN       NaN    NaN       NaN  \n",
       "264                NaN       NaN    NaN       NaN  \n",
       "265                NaN       NaN    NaN       NaN  \n",
       "266                NaN       NaN    NaN       NaN  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_estate_data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>1</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1986-06-21</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Madalyn Mercer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>1</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1983-02-24</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Lara Carrillo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>1</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1985-12-27</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>client</td>\n",
       "      <td>Donavan Flowers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>1</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1985-12-27</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Darien Dorsey</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>1</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1979-05-15</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>agency</td>\n",
       "      <td>Alessandra Perry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>5044</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>44</td>\n",
       "      <td>1238.58</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>322610.74</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>5047</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>47</td>\n",
       "      <td>794.52</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>279191.26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>5048</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>48</td>\n",
       "      <td>1013.27</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287996.53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>5050</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>50</td>\n",
       "      <td>1074.71</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>365868.78</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>5051</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>51</td>\n",
       "      <td>789.25</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>199216.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>267 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id building  date_sale       type property#     area  sold customerid  \\\n",
       "0    1030        1 2005-11-01  apartment        30   743.09     1      C0028   \n",
       "1    1029        1 2005-10-01  apartment        29   756.21     1      C0027   \n",
       "2    2002        2 2007-07-01  apartment         2   587.28     1      C0112   \n",
       "3    2031        2 2007-12-01  apartment        31  1604.75     1      C0160   \n",
       "4    1049        1 2004-11-01  apartment        49  1375.45     1      C0014   \n",
       "..    ...      ...        ...        ...       ...      ...   ...        ...   \n",
       "262  5044        5        NaT  apartment        44  1238.58     0        NaN   \n",
       "263  5047        5        NaT  apartment        47   794.52     0        NaN   \n",
       "264  5048        5        NaT  apartment        48  1013.27     0        NaN   \n",
       "265  5050        5        NaT  apartment        50  1074.71     0        NaN   \n",
       "266  5051        5        NaT  apartment        51   789.25     0        NaN   \n",
       "\n",
       "        price$ individual birth_date  sex country       state     purpose  \\\n",
       "0    246172.68        1.0 1986-06-21    1     USA  California        home   \n",
       "1    246331.90        1.0 1983-02-24    1     USA  California        home   \n",
       "2    209280.91        1.0 1985-12-27    0     USA  California        home   \n",
       "3    452667.01        1.0 1985-12-27    0     USA  California  investment   \n",
       "4    467083.31        1.0 1979-05-15    1     USA  California        home   \n",
       "..         ...        ...        ...  ...     ...         ...         ...   \n",
       "262  322610.74        NaN        NaT  NaN     NaN         NaN         NaN   \n",
       "263  279191.26        NaN        NaT  NaN     NaN         NaN         NaN   \n",
       "264  287996.53        NaN        NaT  NaN     NaN         NaN         NaN   \n",
       "265  365868.78        NaN        NaT  NaN     NaN         NaN         NaN   \n",
       "266  199216.40        NaN        NaT  NaN     NaN         NaN         NaN   \n",
       "\n",
       "    deal_satisfaction mortgage   source         full_name  \n",
       "0                 5.0      0.0  website    Madalyn Mercer  \n",
       "1                 5.0      0.0  website     Lara Carrillo  \n",
       "2                 1.0      1.0   client   Donavan Flowers  \n",
       "3                 3.0      1.0  website     Darien Dorsey  \n",
       "4                 4.0      0.0   agency  Alessandra Perry  \n",
       "..                ...      ...      ...               ...  \n",
       "262               NaN      NaN      NaN               NaN  \n",
       "263               NaN      NaN      NaN               NaN  \n",
       "264               NaN      NaN      NaN               NaN  \n",
       "265               NaN      NaN      NaN               NaN  \n",
       "266               NaN      NaN      NaN               NaN  \n",
       "\n",
       "[267 rows x 19 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Given the numerous NaN values in our dataset, \n",
    "# it would be beneficial to replace them with pandas' native representation for missing values.\n",
    "real_estate_data.fillna(pd.NA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                           object\n",
       "building                     object\n",
       "date_sale            datetime64[ns]\n",
       "type                         object\n",
       "property#                    object\n",
       "area                        float64\n",
       "sold                          int64\n",
       "customerid                   object\n",
       "price$                      float64\n",
       "individual                  float64\n",
       "birth_date           datetime64[ns]\n",
       "sex                          object\n",
       "country                      object\n",
       "state                        object\n",
       "purpose                      object\n",
       "deal_satisfaction           float64\n",
       "mortgage                    float64\n",
       "source                       object\n",
       "full_name                    object\n",
       "dtype: object"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_estate_data.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After finishing with cleaning and preprocessing, it's advisable to create a checkpoint. \n",
    "# We'll assign the cleaned and preprocessed data to a new variable 'data'.\n",
    "data = real_estate_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>1</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1986-06-21</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Madalyn Mercer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>1</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1983-02-24</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Lara Carrillo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>1</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1985-12-27</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>client</td>\n",
       "      <td>Donavan Flowers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>1</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1985-12-27</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Darien Dorsey</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>1</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1979-05-15</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>agency</td>\n",
       "      <td>Alessandra Perry</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id building  date_sale       type property#     area  sold customerid  \\\n",
       "0  1030        1 2005-11-01  apartment        30   743.09     1      C0028   \n",
       "1  1029        1 2005-10-01  apartment        29   756.21     1      C0027   \n",
       "2  2002        2 2007-07-01  apartment         2   587.28     1      C0112   \n",
       "3  2031        2 2007-12-01  apartment        31  1604.75     1      C0160   \n",
       "4  1049        1 2004-11-01  apartment        49  1375.45     1      C0014   \n",
       "\n",
       "      price$  individual birth_date sex country       state     purpose  \\\n",
       "0  246172.68         1.0 1986-06-21   1     USA  California        home   \n",
       "1  246331.90         1.0 1983-02-24   1     USA  California        home   \n",
       "2  209280.91         1.0 1985-12-27   0     USA  California        home   \n",
       "3  452667.01         1.0 1985-12-27   0     USA  California  investment   \n",
       "4  467083.31         1.0 1979-05-15   1     USA  California        home   \n",
       "\n",
       "   deal_satisfaction  mortgage   source         full_name  \n",
       "0                5.0       0.0  website    Madalyn Mercer  \n",
       "1                5.0       0.0  website     Lara Carrillo  \n",
       "2                1.0       1.0   client   Donavan Flowers  \n",
       "3                3.0       1.0  website     Darien Dorsey  \n",
       "4                4.0       0.0   agency  Alessandra Perry  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's inspect the first few rows of our processed data.\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>267.000000</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>195.000000</td>\n",
       "      <td>195.000000</td>\n",
       "      <td>195.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>936.221311</td>\n",
       "      <td>0.730337</td>\n",
       "      <td>281171.901386</td>\n",
       "      <td>0.912821</td>\n",
       "      <td>3.600000</td>\n",
       "      <td>0.317949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>284.894858</td>\n",
       "      <td>0.444618</td>\n",
       "      <td>89119.121005</td>\n",
       "      <td>0.282824</td>\n",
       "      <td>1.340872</td>\n",
       "      <td>0.466878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>410.710000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117564.070000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>756.210000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>217553.055000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>798.280000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>249075.660000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1121.950000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>326964.855000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1942.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>538271.740000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              area        sold         price$  individual  deal_satisfaction  \\\n",
       "count   267.000000  267.000000     267.000000  195.000000         195.000000   \n",
       "mean    936.221311    0.730337  281171.901386    0.912821           3.600000   \n",
       "std     284.894858    0.444618   89119.121005    0.282824           1.340872   \n",
       "min     410.710000    0.000000  117564.070000    0.000000           1.000000   \n",
       "25%     756.210000    0.000000  217553.055000    1.000000           3.000000   \n",
       "50%     798.280000    1.000000  249075.660000    1.000000           4.000000   \n",
       "75%    1121.950000    1.000000  326964.855000    1.000000           5.000000   \n",
       "max    1942.500000    1.000000  538271.740000    1.000000           5.000000   \n",
       "\n",
       "         mortgage  \n",
       "count  195.000000  \n",
       "mean     0.317949  \n",
       "std      0.466878  \n",
       "min      0.000000  \n",
       "25%      0.000000  \n",
       "50%      0.000000  \n",
       "75%      1.000000  \n",
       "max      1.000000  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# As we're transitioning into the statistical analysis phase, running descriptive statistics would be particularly helpful.\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>195</td>\n",
       "      <td>267</td>\n",
       "      <td>267</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>195</td>\n",
       "      <td>267.000000</td>\n",
       "      <td>195.000000</td>\n",
       "      <td>178</td>\n",
       "      <td>178</td>\n",
       "      <td>195</td>\n",
       "      <td>181</td>\n",
       "      <td>195</td>\n",
       "      <td>195.000000</td>\n",
       "      <td>195.000000</td>\n",
       "      <td>195</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>240</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>162</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>5041</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>apartment</td>\n",
       "      <td>34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C0174</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>website</td>\n",
       "      <td>Marleez Co</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>2</td>\n",
       "      <td>63</td>\n",
       "      <td>NaN</td>\n",
       "      <td>259</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>108</td>\n",
       "      <td>165</td>\n",
       "      <td>120</td>\n",
       "      <td>119</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>119</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2006-11-08 14:38:46.153846272</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>936.221311</td>\n",
       "      <td>0.730337</td>\n",
       "      <td>NaN</td>\n",
       "      <td>281171.901386</td>\n",
       "      <td>0.912821</td>\n",
       "      <td>1961-01-08 11:35:43.820224704</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.600000</td>\n",
       "      <td>0.317949</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2004-03-01 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>410.710000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>117564.070000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1931-02-13 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2006-04-16 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>756.210000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>217553.055000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1951-05-04 06:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2007-03-01 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>798.280000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>249075.660000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1962-09-23 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2007-09-01 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1121.950000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>326964.855000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1970-07-09 06:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2010-05-01 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1942.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>538271.740000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1986-06-21 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>284.894858</td>\n",
       "      <td>0.444618</td>\n",
       "      <td>NaN</td>\n",
       "      <td>89119.121005</td>\n",
       "      <td>0.282824</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.340872</td>\n",
       "      <td>0.466878</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id building                      date_sale       type property#  \\\n",
       "count    267      267                            195        267       267   \n",
       "unique   240        5                            NaN          2        59   \n",
       "top     5041        3                            NaN  apartment        34   \n",
       "freq       2       63                            NaN        259         7   \n",
       "mean     NaN      NaN  2006-11-08 14:38:46.153846272        NaN       NaN   \n",
       "min      NaN      NaN            2004-03-01 00:00:00        NaN       NaN   \n",
       "25%      NaN      NaN            2006-04-16 00:00:00        NaN       NaN   \n",
       "50%      NaN      NaN            2007-03-01 00:00:00        NaN       NaN   \n",
       "75%      NaN      NaN            2007-09-01 00:00:00        NaN       NaN   \n",
       "max      NaN      NaN            2010-05-01 00:00:00        NaN       NaN   \n",
       "std      NaN      NaN                            NaN        NaN       NaN   \n",
       "\n",
       "               area        sold customerid         price$  individual  \\\n",
       "count    267.000000  267.000000        195     267.000000  195.000000   \n",
       "unique          NaN         NaN        162            NaN         NaN   \n",
       "top             NaN         NaN      C0174            NaN         NaN   \n",
       "freq            NaN         NaN          9            NaN         NaN   \n",
       "mean     936.221311    0.730337        NaN  281171.901386    0.912821   \n",
       "min      410.710000    0.000000        NaN  117564.070000    0.000000   \n",
       "25%      756.210000    0.000000        NaN  217553.055000    1.000000   \n",
       "50%      798.280000    1.000000        NaN  249075.660000    1.000000   \n",
       "75%     1121.950000    1.000000        NaN  326964.855000    1.000000   \n",
       "max     1942.500000    1.000000        NaN  538271.740000    1.000000   \n",
       "std      284.894858    0.444618        NaN   89119.121005    0.282824   \n",
       "\n",
       "                           birth_date  sex country       state purpose  \\\n",
       "count                             178  178     195         181     195   \n",
       "unique                            NaN    2       9           9       2   \n",
       "top                               NaN    0     USA  California    home   \n",
       "freq                              NaN  108     165         120     119   \n",
       "mean    1961-01-08 11:35:43.820224704  NaN     NaN         NaN     NaN   \n",
       "min               1931-02-13 00:00:00  NaN     NaN         NaN     NaN   \n",
       "25%               1951-05-04 06:00:00  NaN     NaN         NaN     NaN   \n",
       "50%               1962-09-23 00:00:00  NaN     NaN         NaN     NaN   \n",
       "75%               1970-07-09 06:00:00  NaN     NaN         NaN     NaN   \n",
       "max               1986-06-21 00:00:00  NaN     NaN         NaN     NaN   \n",
       "std                               NaN  NaN     NaN         NaN     NaN   \n",
       "\n",
       "        deal_satisfaction    mortgage   source   full_name  \n",
       "count          195.000000  195.000000      195         195  \n",
       "unique                NaN         NaN        3         162  \n",
       "top                   NaN         NaN  website  Marleez Co  \n",
       "freq                  NaN         NaN      119           9  \n",
       "mean             3.600000    0.317949      NaN         NaN  \n",
       "min              1.000000    0.000000      NaN         NaN  \n",
       "25%              3.000000    0.000000      NaN         NaN  \n",
       "50%              4.000000    0.000000      NaN         NaN  \n",
       "75%              5.000000    1.000000      NaN         NaN  \n",
       "max              5.000000    1.000000      NaN         NaN  \n",
       "std              1.340872    0.466878      NaN         NaN  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Given that we've now entered the statistical analysis phase, \n",
    "# obtaining a comprehensive set of descriptive statistics will be immensely beneficial.\n",
    "data.describe(include=\"all\", datetime_is_numeric = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Breakdowns by building"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can examine:\n",
    "1. Breakdown of totals by building (frequency distribution by building)\n",
    "2. Breakdown of averages by building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['1', '2', '3', '4', '5'], dtype=object)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's identify the unique buildings in our dataset.\n",
    "data['building'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.generic.DataFrameGroupBy object at 0x000001314A5A4CC8>"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A groupby operation typically involves a combination of splitting the object, \n",
    "# applying a function, and then combining the results.\n",
    "\n",
    "# Let's experiment with grouping the data by 'building'.\n",
    "data.groupby('building')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>building</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48258.02</td>\n",
       "      <td>46</td>\n",
       "      <td>14307448.61</td>\n",
       "      <td>31.0</td>\n",
       "      <td>167.0</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>53801.84</td>\n",
       "      <td>54</td>\n",
       "      <td>16339725.35</td>\n",
       "      <td>52.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>58454.70</td>\n",
       "      <td>53</td>\n",
       "      <td>17668429.10</td>\n",
       "      <td>53.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41913.00</td>\n",
       "      <td>23</td>\n",
       "      <td>12480299.17</td>\n",
       "      <td>23.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>47543.53</td>\n",
       "      <td>19</td>\n",
       "      <td>14276995.44</td>\n",
       "      <td>19.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              area  sold       price$  individual  deal_satisfaction  mortgage\n",
       "building                                                                      \n",
       "1         48258.02    46  14307448.61        31.0              167.0      14.0\n",
       "2         53801.84    54  16339725.35        52.0              190.0      18.0\n",
       "3         58454.70    53  17668429.10        53.0              189.0      15.0\n",
       "4         41913.00    23  12480299.17        23.0               89.0       9.0\n",
       "5         47543.53    19  14276995.44        19.0               67.0       6.0"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# It appears that this operation results in a special object type. As indicated in the documentation, \n",
    "# this object functions best when used with an aggregation method.\n",
    "# For example, we could use aggregation methods such as count, sum, mean, median, etc.\n",
    "\n",
    "data.groupby('building').sum()\n",
    "\n",
    "# However, there seems to be excessive information here, and the sum of certain variables,\n",
    "# such as 'area', does not provide meaningful insights when aggregated in this manner.\n",
    "# Consequently, it would be more sensible to select a subset of 'data' to aggregate."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Breakdown of totals by building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "      <th>mortgage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>building</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>46</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>54</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>19</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          sold  mortgage\n",
       "building                \n",
       "1           46      14.0\n",
       "2           54      18.0\n",
       "3           53      15.0\n",
       "4           23       9.0\n",
       "5           19       6.0"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For a more structured approach to this problem, let's create a variable to hold the columns of interest.\n",
    "# This variable will include the column we want to group by and the columns we wish to aggregate.\n",
    "# It would be informative to observe the total number of properties sold \n",
    "# and those under mortgage per building - these would represent total counts.\n",
    "# Alternatively, we could examine the average area, price, and deal satisfaction by building.\n",
    "\n",
    "# Let's commence with deriving total counts per building.\n",
    "columns_of_interest = ['building', 'sold','mortgage']\n",
    "\n",
    "# Now we will create a new variable to hold the total counts grouped by building.\n",
    "totals_by_building = data[columns_of_interest].groupby(\"building\").sum()\n",
    "totals_by_building \n",
    "\n",
    "# Note that 'mortgage' here is a float, a side effect of the 'merge' operation we performed earlier.\n",
    "# While this doesn't affect our results, it's an interesting observation. You may recast it to int64 if you prefer."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Breakdown of averages by building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>area</th>\n",
       "      <th>price$</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>building</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>928.038846</td>\n",
       "      <td>275143.242500</td>\n",
       "      <td>3.630435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>943.891930</td>\n",
       "      <td>286661.848246</td>\n",
       "      <td>3.518519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>927.852381</td>\n",
       "      <td>280451.255556</td>\n",
       "      <td>3.566038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>974.720930</td>\n",
       "      <td>290239.515581</td>\n",
       "      <td>3.869565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>914.298654</td>\n",
       "      <td>274557.604615</td>\n",
       "      <td>3.526316</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                area         price$  deal_satisfaction\n",
       "building                                              \n",
       "1         928.038846  275143.242500           3.630435\n",
       "2         943.891930  286661.848246           3.518519\n",
       "3         927.852381  280451.255556           3.566038\n",
       "4         974.720930  290239.515581           3.869565\n",
       "5         914.298654  274557.604615           3.526316"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now let's focus on deriving average values. Here is the relevant list of columns.\n",
    "# Notice that we are overwriting the 'columns_of_interest' variable.\n",
    "# There are different ways to tackle this issue. \n",
    "# I've chosen to use this variable as a temporary holder of the list of columns of interest\n",
    "# for the given groupby operation. This is one of the simplest approaches.\n",
    "# If you feel uneasy about it, you can skip using this variable and directly list the relevant columns\n",
    "# in the .groupby() operation line itself.\n",
    "columns_of_interest = ['building', 'area', 'price$', 'deal_satisfaction']\n",
    "\n",
    "# Let's now create a new variable to hold the average values grouped by building.\n",
    "averages_by_building = data[columns_of_interest].groupby(\"building\").mean()\n",
    "averages_by_building"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Breakdowns by country and state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Country\n",
    "1. Breakdown of totals by country (frequency distribution by country)\n",
    "2. Breakdown of averages by country\n",
    "\n",
    "State\n",
    "3. Frequency distribution by state\n",
    "4. Relative frequency by state\n",
    "5. Cumulative frequency by state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Breakdown of totals by country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "      <th>mortgage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Belgium</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Canada</th>\n",
       "      <td>7</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Denmark</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Germany</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mexico</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Russia</th>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UK</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>USA</th>\n",
       "      <td>165</td>\n",
       "      <td>61.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>USA</th>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sold  mortgage\n",
       "country                \n",
       "Belgium     2       0.0\n",
       "Canada      7       0.0\n",
       "Denmark     1       0.0\n",
       "Germany     1       0.0\n",
       "Mexico      1       0.0\n",
       "Russia      4       1.0\n",
       "UK          2       0.0\n",
       "USA       165      61.0\n",
       "USA        12       0.0"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# While attempting to break down the data by country, we notice a duplication of the USA entries.\n",
    "# It's logical to use the same 'columns_of_interest' as before for the totals.\n",
    "# This is because we've already reasoned out which totals are meaningful, regardless of what we aggregate on.\n",
    "# The only adjustment required is to substitute 'building' with 'country'.\n",
    "\n",
    "columns_of_interest = ['country', 'sold','mortgage']\n",
    "totals_by_country = data[columns_of_interest].groupby(\"country\").sum()\n",
    "totals_by_country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['USA', 'UK', 'USA ', 'Belgium', 'Russia', 'Denmark', 'Germany',\n",
       "       'Mexico', 'Canada', nan], dtype=object)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ensure your data has been thoroughly cleaned.\n",
    "# It's possible we didn't scrutinize all columns as we should have.\n",
    "# If you see two entries for USA, you've likely overlooked a step in the data cleaning process.\n",
    "\n",
    "# Let's inspect the unique values in the 'country' column.\n",
    "data['country'].unique()\n",
    "\n",
    "# Clearly, there are duplicate entries for USA,\n",
    "# one of them has a trailing space, which needs to be addressed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'building', 'type', 'property#', 'customerid', 'sex', 'country',\n",
       "       'state', 'purpose', 'source', 'full_name'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# As a matter of fact, we can perform the cleanup operation on all columns at once.\n",
    "# There's a possibility that we overlooked similar issues in other columns.\n",
    "# Therefore, it's prudent to apply this cleanup to all 'object' columns in one go.\n",
    "# The following is a clever way to accomplish this.\n",
    "object_columns = data.select_dtypes(['object']).columns\n",
    "object_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Using this, we select only the data from these columns and\n",
    "# use the .apply() method to strip all white spaces from them simultaneously. \n",
    "data[object_columns] = data[object_columns].apply(lambda x: x.str.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "      <th>mortgage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Belgium</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Canada</th>\n",
       "      <td>7</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Denmark</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Germany</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mexico</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Russia</th>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UK</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>USA</th>\n",
       "      <td>177</td>\n",
       "      <td>61.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sold  mortgage\n",
       "country                \n",
       "Belgium     2       0.0\n",
       "Canada      7       0.0\n",
       "Denmark     1       0.0\n",
       "Germany     1       0.0\n",
       "Mexico      1       0.0\n",
       "Russia      4       1.0\n",
       "UK          2       0.0\n",
       "USA       177      61.0"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we've achieved a much cleaner data set!\n",
    "totals_by_country = data[columns_of_interest].groupby(\"country\").sum()\n",
    "totals_by_country"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Breakdown of averages by country\n",
    "Based on what you have seen before for the breakdowns by bulding and by state, please find the breakdown by country of the columns 'area', 'deal_satisfaction', and 'price$'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>area</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Belgium</th>\n",
       "      <td>852.730000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>229075.470000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Canada</th>\n",
       "      <td>917.382857</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>274069.384286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Denmark</th>\n",
       "      <td>785.480000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>257183.480000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Germany</th>\n",
       "      <td>743.410000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>205098.210000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mexico</th>\n",
       "      <td>1283.450000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>338181.180000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Russia</th>\n",
       "      <td>903.757500</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>278828.835000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UK</th>\n",
       "      <td>739.480000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>220142.680000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>USA</th>\n",
       "      <td>900.794463</td>\n",
       "      <td>3.581921</td>\n",
       "      <td>270096.266554</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                area  deal_satisfaction         price$\n",
       "country                                               \n",
       "Belgium   852.730000           3.000000  229075.470000\n",
       "Canada    917.382857           5.000000  274069.384286\n",
       "Denmark   785.480000           1.000000  257183.480000\n",
       "Germany   743.410000           5.000000  205098.210000\n",
       "Mexico   1283.450000           3.000000  338181.180000\n",
       "Russia    903.757500           2.500000  278828.835000\n",
       "UK        739.480000           4.000000  220142.680000\n",
       "USA       900.794463           3.581921  270096.266554"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For the average computations, we use the following list of columns.\n",
    "columns_of_interest = ['country', 'area', 'deal_satisfaction','price$']\n",
    "\n",
    "# We create a new variable where we will store the data relevant to our calculations.\n",
    "averages_by_country = data[columns_of_interest].groupby(\"country\").mean()\n",
    "averages_by_country"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Frequency distribution by state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "      <th>mortgage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Arizona</th>\n",
       "      <td>11</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California</th>\n",
       "      <td>120</td>\n",
       "      <td>41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colorado</th>\n",
       "      <td>11</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kansas</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nevada</th>\n",
       "      <td>17</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oregon</th>\n",
       "      <td>11</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Utah</th>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Virginia</th>\n",
       "      <td>4</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wyoming</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            sold  mortgage\n",
       "state                     \n",
       "Arizona       11       3.0\n",
       "California   120      41.0\n",
       "Colorado      11       5.0\n",
       "Kansas         1       0.0\n",
       "Nevada        17       8.0\n",
       "Oregon        11       1.0\n",
       "Utah           5       1.0\n",
       "Virginia       4       2.0\n",
       "Wyoming        1       0.0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns_of_interest = ['state', 'sold','mortgage']\n",
    "totals_by_state = data[columns_of_interest].groupby(\"state\").sum()\n",
    "totals_by_state\n",
    "\n",
    "# Please be aware that not all entries have a 'state' value.\n",
    "# This could cause complications if we didn't previously map the empty strings to NA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "181"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "totals_by_state.sold.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "      <th>mortgage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Belgium</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Canada</th>\n",
       "      <td>7</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Denmark</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Germany</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mexico</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Russia</th>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UK</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>USA</th>\n",
       "      <td>177</td>\n",
       "      <td>61.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sold  mortgage\n",
       "country                \n",
       "Belgium     2       0.0\n",
       "Canada      7       0.0\n",
       "Denmark     1       0.0\n",
       "Germany     1       0.0\n",
       "Mexico      1       0.0\n",
       "Russia      4       1.0\n",
       "UK          2       0.0\n",
       "USA       177      61.0"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "totals_by_country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['state'] = np.where(data['state']=='', pd.NA, data['state'])\n",
    "data['state'] = np.where(data['country']!='USA', pd.NA, data['state'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "      <th>mortgage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Arizona</th>\n",
       "      <td>9</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California</th>\n",
       "      <td>118</td>\n",
       "      <td>41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colorado</th>\n",
       "      <td>11</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kansas</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nevada</th>\n",
       "      <td>17</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oregon</th>\n",
       "      <td>11</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Utah</th>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Virginia</th>\n",
       "      <td>4</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wyoming</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            sold  mortgage\n",
       "state                     \n",
       "Arizona        9       3.0\n",
       "California   118      41.0\n",
       "Colorado      11       5.0\n",
       "Kansas         1       0.0\n",
       "Nevada        17       8.0\n",
       "Oregon        11       1.0\n",
       "Utah           5       1.0\n",
       "Virginia       4       2.0\n",
       "Wyoming        1       0.0"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here's the new result\n",
    "totals_by_state = data[columns_of_interest].groupby(\"state\").sum()\n",
    "totals_by_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "177"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "totals_by_state.sold.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Arizona</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>California</th>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colorado</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kansas</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nevada</th>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oregon</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Utah</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Virginia</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wyoming</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            sold\n",
       "state           \n",
       "Arizona        9\n",
       "California   118\n",
       "Colorado      11\n",
       "Kansas         1\n",
       "Nevada        17\n",
       "Oregon        11\n",
       "Utah           5\n",
       "Virginia       4\n",
       "Wyoming        1"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To focus on the relative and cumulative frequency of sales, we can refine our table by state.\n",
    "# We will start by excluding 'mortgage' from our columns of interest.\n",
    "columns_of_interest = ['state', 'sold']\n",
    "\n",
    "# We will store this data in a new variable called 'sold_by_state'.\n",
    "sold_by_state = data[columns_of_interest].groupby(\"state\").sum()\n",
    "sold_by_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>California</th>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nevada</th>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colorado</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oregon</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Arizona</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Utah</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Virginia</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kansas</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wyoming</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            sold\n",
       "state           \n",
       "California   118\n",
       "Nevada        17\n",
       "Colorado      11\n",
       "Oregon        11\n",
       "Arizona        9\n",
       "Utah           5\n",
       "Virginia       4\n",
       "Kansas         1\n",
       "Wyoming        1"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To get a clearer picture, we can sort the values in descending order. This will place the states with the highest sales on top.\n",
    "sold_by_state = sold_by_state.sort_values('sold', ascending=False)\n",
    "sold_by_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frequency</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>California</th>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nevada</th>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colorado</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oregon</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Arizona</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Utah</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Virginia</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kansas</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wyoming</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            frequency\n",
       "state                \n",
       "California        118\n",
       "Nevada             17\n",
       "Colorado           11\n",
       "Oregon             11\n",
       "Arizona             9\n",
       "Utah                5\n",
       "Virginia            4\n",
       "Kansas              1\n",
       "Wyoming             1"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The term 'sold' may not be the best to describe frequency, so we can rename this column.\n",
    "sold_by_state = sold_by_state.rename(columns={'sold':'frequency'})\n",
    "sold_by_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Relative frequency distribution by state\n",
    "1. Add a new column to the 'sold_by_state' data frame, called 'relative_frequency' which contains the relative frequency of the different states.\n",
    "2. Add a new column to the 'sold_by_state' data frame, called 'cumulative_frequency' which contains the cumulative frequency of the different states. You can use your own tools to achieve this, or look up the .cumsum() method here: https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.cumsum.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frequency</th>\n",
       "      <th>relative_frequency</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>California</th>\n",
       "      <td>118</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nevada</th>\n",
       "      <td>17</td>\n",
       "      <td>0.096045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colorado</th>\n",
       "      <td>11</td>\n",
       "      <td>0.062147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oregon</th>\n",
       "      <td>11</td>\n",
       "      <td>0.062147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Arizona</th>\n",
       "      <td>9</td>\n",
       "      <td>0.050847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Utah</th>\n",
       "      <td>5</td>\n",
       "      <td>0.028249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Virginia</th>\n",
       "      <td>4</td>\n",
       "      <td>0.022599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kansas</th>\n",
       "      <td>1</td>\n",
       "      <td>0.005650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wyoming</th>\n",
       "      <td>1</td>\n",
       "      <td>0.005650</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            frequency  relative_frequency\n",
       "state                                    \n",
       "California        118            0.666667\n",
       "Nevada             17            0.096045\n",
       "Colorado           11            0.062147\n",
       "Oregon             11            0.062147\n",
       "Arizona             9            0.050847\n",
       "Utah                5            0.028249\n",
       "Virginia            4            0.022599\n",
       "Kansas              1            0.005650\n",
       "Wyoming             1            0.005650"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The relative frequency can be computed by dividing the frequency of each state by the total frequency.\n",
    "sold_by_state['relative_frequency'] = sold_by_state['frequency']/sold_by_state['frequency'].sum()\n",
    "sold_by_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frequency</th>\n",
       "      <th>relative_frequency</th>\n",
       "      <th>cumulative_frequency</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>California</th>\n",
       "      <td>118</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nevada</th>\n",
       "      <td>17</td>\n",
       "      <td>0.096045</td>\n",
       "      <td>0.762712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Colorado</th>\n",
       "      <td>11</td>\n",
       "      <td>0.062147</td>\n",
       "      <td>0.824859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oregon</th>\n",
       "      <td>11</td>\n",
       "      <td>0.062147</td>\n",
       "      <td>0.887006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Arizona</th>\n",
       "      <td>9</td>\n",
       "      <td>0.050847</td>\n",
       "      <td>0.937853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Utah</th>\n",
       "      <td>5</td>\n",
       "      <td>0.028249</td>\n",
       "      <td>0.966102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Virginia</th>\n",
       "      <td>4</td>\n",
       "      <td>0.022599</td>\n",
       "      <td>0.988701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kansas</th>\n",
       "      <td>1</td>\n",
       "      <td>0.005650</td>\n",
       "      <td>0.994350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wyoming</th>\n",
       "      <td>1</td>\n",
       "      <td>0.005650</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            frequency  relative_frequency  cumulative_frequency\n",
       "state                                                          \n",
       "California        118            0.666667              0.666667\n",
       "Nevada             17            0.096045              0.762712\n",
       "Colorado           11            0.062147              0.824859\n",
       "Oregon             11            0.062147              0.887006\n",
       "Arizona             9            0.050847              0.937853\n",
       "Utah                5            0.028249              0.966102\n",
       "Virginia            4            0.022599              0.988701\n",
       "Kansas              1            0.005650              0.994350\n",
       "Wyoming             1            0.005650              1.000000"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cumulative frequency can be obtained using the 'cumsum()' function in pandas.\n",
    "# This function calculates the cumulative sum of values in a Series.\n",
    "# Applying this on our relative frequency column provides us with the cumulative frequency.\n",
    "sold_by_state['cumulative_frequency'] = sold_by_state['relative_frequency'].cumsum()\n",
    "sold_by_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Customers Age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>birth_date</th>\n",
       "      <th>sex</th>\n",
       "      <th>country</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>1</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1986-06-21</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Madalyn Mercer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>1</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1983-02-24</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Lara Carrillo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>1</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1985-12-27</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>client</td>\n",
       "      <td>Donavan Flowers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>1</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1985-12-27</td>\n",
       "      <td>0</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Darien Dorsey</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>1</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1979-05-15</td>\n",
       "      <td>1</td>\n",
       "      <td>USA</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>agency</td>\n",
       "      <td>Alessandra Perry</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id building  date_sale       type property#     area  sold customerid  \\\n",
       "0  1030        1 2005-11-01  apartment        30   743.09     1      C0028   \n",
       "1  1029        1 2005-10-01  apartment        29   756.21     1      C0027   \n",
       "2  2002        2 2007-07-01  apartment         2   587.28     1      C0112   \n",
       "3  2031        2 2007-12-01  apartment        31  1604.75     1      C0160   \n",
       "4  1049        1 2004-11-01  apartment        49  1375.45     1      C0014   \n",
       "\n",
       "      price$  individual birth_date sex country       state     purpose  \\\n",
       "0  246172.68         1.0 1986-06-21   1     USA  California        home   \n",
       "1  246331.90         1.0 1983-02-24   1     USA  California        home   \n",
       "2  209280.91         1.0 1985-12-27   0     USA  California        home   \n",
       "3  452667.01         1.0 1985-12-27   0     USA  California  investment   \n",
       "4  467083.31         1.0 1979-05-15   1     USA  California        home   \n",
       "\n",
       "   deal_satisfaction  mortgage   source         full_name  \n",
       "0                5.0       0.0  website    Madalyn Mercer  \n",
       "1                5.0       0.0  website     Lara Carrillo  \n",
       "2                1.0       1.0   client   Donavan Flowers  \n",
       "3                3.0       1.0  website     Darien Dorsey  \n",
       "4                4.0       0.0   agency  Alessandra Perry  "
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     7073 days\n",
       "1     8255 days\n",
       "2     7856 days\n",
       "3     8009 days\n",
       "4     9302 days\n",
       "         ...   \n",
       "262         NaT\n",
       "263         NaT\n",
       "264         NaT\n",
       "265         NaT\n",
       "266         NaT\n",
       "Name: age_at_purchase, Length: 267, dtype: timedelta64[ns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's calculate the age of our buyers at the time of purchase.\n",
    "# This can be done by subtracting the birth date from the sale date.\n",
    "data['age_at_purchase'] = data['date_sale'] - data['birth_date']\n",
    "data['age_at_purchase']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "int"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# timedelta has different attributes than normal dates, which you can explore in the docs\n",
    "# for our purposes, we want the age in integers, so we can work with it\n",
    "# to access the number of days stored inside, we can use the attribute 'days'\n",
    "\n",
    "#data['age_at_purchase'][0].days\n",
    "type(data['age_at_purchase'][0].days)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      7073.0\n",
       "1      8255.0\n",
       "2      7856.0\n",
       "3      8009.0\n",
       "4      9302.0\n",
       "        ...  \n",
       "262       NaN\n",
       "263       NaN\n",
       "264       NaN\n",
       "265       NaN\n",
       "266       NaN\n",
       "Name: age_at_purchase, Length: 267, dtype: float64"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# It's important to note that the 'age_at_purchase' field we've just created is of timedelta datatype.\n",
    "# A timedelta object represents a duration, the difference between two dates or times.\n",
    "# For our analysis, we need the age in integers or floats for better manipulation.\n",
    "# To convert it, we use the 'days' attribute to extract the number of days, as timedelta measures duration in days.\n",
    "\n",
    "data['age_at_purchase'] = data['age_at_purchase'].apply(lambda x: x.days)\n",
    "data['age_at_purchase']\n",
    "\n",
    "# Notice that the datatype is float64, which may be due to how pandas internally manages timedelta objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      19.378082\n",
       "1      22.616438\n",
       "2      21.523288\n",
       "3      21.942466\n",
       "4      25.484932\n",
       "         ...    \n",
       "262          NaN\n",
       "263          NaN\n",
       "264          NaN\n",
       "265          NaN\n",
       "266          NaN\n",
       "Name: age_at_purchase, Length: 267, dtype: float64"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To obtain the age in years at the time of purchase, we can divide the 'age_at_purchase' field by 365.\n",
    "# This may not be perfectly accurate due to leap years, but it should be close enough for our analysis.\n",
    "data['age_at_purchase'] = data['age_at_purchase']/365\n",
    "data['age_at_purchase']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      19.0\n",
       "1      22.0\n",
       "2      21.0\n",
       "3      21.0\n",
       "4      25.0\n",
       "       ... \n",
       "262     NaN\n",
       "263     NaN\n",
       "264     NaN\n",
       "265     NaN\n",
       "266     NaN\n",
       "Name: age_at_purchase_rounded, Length: 267, dtype: float64"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Since age is typically represented in whole numbers, we can round down the values using np.floor().\n",
    "data['age_at_purchase_rounded'] = data['age_at_purchase'].apply(lambda x: np.floor(x))\n",
    "data['age_at_purchase_rounded']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create age intervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      (19.0, 25.0]\n",
       "1      (19.0, 25.0]\n",
       "2      (19.0, 25.0]\n",
       "3      (19.0, 25.0]\n",
       "4      (25.0, 31.0]\n",
       "           ...     \n",
       "262             NaN\n",
       "263             NaN\n",
       "264             NaN\n",
       "265             NaN\n",
       "266             NaN\n",
       "Name: age_interval, Length: 267, dtype: category\n",
       "Categories (10, interval[float64, right]): [(19.0, 25.0] < (25.0, 31.0] < (31.0, 36.0] < (36.0, 42.0] ... (54.0, 59.0] < (59.0, 65.0] < (65.0, 71.0] < (71.0, 76.0]]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note that we retain the original values when calculating statistics, as they are more precise.\n",
    "# To divide age into different intervals, we can use the 'cut' method from pandas.\n",
    "# This function segments and sorts the data values into bins we specify.\n",
    "# Here, we're dividing the 'age_at_purchase' into 10 bins and setting the decimal precision to 0.\n",
    "data['age_interval'] = pd.cut(data['age_at_purchase'], bins = 10, precision = 0)\n",
    "data['age_interval']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Breakdown by age interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age_interval</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(19.0, 25.0]</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(25.0, 31.0]</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(31.0, 36.0]</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(36.0, 42.0]</th>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(42.0, 48.0]</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(48.0, 54.0]</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(54.0, 59.0]</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(59.0, 65.0]</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(65.0, 71.0]</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(71.0, 76.0]</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              sold\n",
       "age_interval      \n",
       "(19.0, 25.0]     4\n",
       "(25.0, 31.0]    16\n",
       "(31.0, 36.0]    26\n",
       "(36.0, 42.0]    33\n",
       "(42.0, 48.0]    22\n",
       "(48.0, 54.0]    22\n",
       "(54.0, 59.0]    22\n",
       "(59.0, 65.0]    11\n",
       "(65.0, 71.0]    16\n",
       "(71.0, 76.0]     6"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns_of_interest = ['age_interval', 'sold']\n",
    "sold_by_age = data[columns_of_interest].groupby(\"age_interval\").sum()\n",
    "sold_by_age"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis of the price of properties"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Price interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      (243776.371, 285847.138]\n",
       "1      (243776.371, 285847.138]\n",
       "2      (201705.604, 243776.371]\n",
       "3      (412059.439, 454130.206]\n",
       "4      (454130.206, 496200.973]\n",
       "                 ...           \n",
       "262    (285847.138, 327917.905]\n",
       "263    (243776.371, 285847.138]\n",
       "264    (285847.138, 327917.905]\n",
       "265    (327917.905, 369988.672]\n",
       "266    (159634.837, 201705.604]\n",
       "Name: price_interval, Length: 267, dtype: category\n",
       "Categories (10, interval[float64, right]): [(117143.362, 159634.837] < (159634.837, 201705.604] < (201705.604, 243776.371] < (243776.371, 285847.138] ... (369988.672, 412059.439] < (412059.439, 454130.206] < (454130.206, 496200.973] < (496200.973, 538271.74]]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To create 'price_interval', we adopt a similar process to that of 'age_interval'.\n",
    "# We segment and sort the 'price$' values into 10 bins.\n",
    "data['price_interval'] = pd.cut(data['price$'], bins=10)\n",
    "data['price_interval']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total number of properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price_interval</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(117143.362, 159634.837]</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(159634.837, 201705.604]</th>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(201705.604, 243776.371]</th>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(243776.371, 285847.138]</th>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(285847.138, 327917.905]</th>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(327917.905, 369988.672]</th>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(369988.672, 412059.439]</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(412059.439, 454130.206]</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(454130.206, 496200.973]</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(496200.973, 538271.74]</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          count\n",
       "price_interval                 \n",
       "(117143.362, 159634.837]      3\n",
       "(159634.837, 201705.604]     32\n",
       "(201705.604, 243776.371]     88\n",
       "(243776.371, 285847.138]     47\n",
       "(285847.138, 327917.905]     31\n",
       "(327917.905, 369988.672]     18\n",
       "(369988.672, 412059.439]     21\n",
       "(412059.439, 454130.206]      7\n",
       "(454130.206, 496200.973]     11\n",
       "(496200.973, 538271.74]       9"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now, we want to count the total number of properties for each price interval.\n",
    "columns_of_interest = ['price_interval', 'sold']\n",
    "all_properties_by_price = data[columns_of_interest].groupby(\"price_interval\").count()\n",
    "\n",
    "# Here, 'sold' simply represents a count, so it's more fitting to rename the column as 'count'.\n",
    "all_properties_by_price = all_properties_by_price.rename(columns={'sold':'count'})\n",
    "all_properties_by_price"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total number of sold properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sold</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price_interval</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(117143.362, 159634.837]</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(159634.837, 201705.604]</th>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(201705.604, 243776.371]</th>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(243776.371, 285847.138]</th>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(285847.138, 327917.905]</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(327917.905, 369988.672]</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(369988.672, 412059.439]</th>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(412059.439, 454130.206]</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(454130.206, 496200.973]</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(496200.973, 538271.74]</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          sold\n",
       "price_interval                \n",
       "(117143.362, 159634.837]     2\n",
       "(159634.837, 201705.604]    28\n",
       "(201705.604, 243776.371]    68\n",
       "(243776.371, 285847.138]    34\n",
       "(285847.138, 327917.905]    24\n",
       "(327917.905, 369988.672]    12\n",
       "(369988.672, 412059.439]    13\n",
       "(412059.439, 454130.206]     4\n",
       "(454130.206, 496200.973]     7\n",
       "(496200.973, 538271.74]      3"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For the sold properties, we will count the number of properties sold for each price interval.\n",
    "# Here, we use the 'sum()' function with 'groupby' as 'sold' here indeed represents properties sold.\n",
    "columns_of_interest = ['price_interval', 'sold']\n",
    "sold_properties_by_price = data[columns_of_interest].groupby(\"price_interval\").sum()\n",
    "sold_properties_by_price"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total number of not sold properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>not_sold</th>\n",
       "      <th>sold</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price_interval</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(117143.362, 159634.837]</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(159634.837, 201705.604]</th>\n",
       "      <td>32</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(201705.604, 243776.371]</th>\n",
       "      <td>88</td>\n",
       "      <td>20</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(243776.371, 285847.138]</th>\n",
       "      <td>47</td>\n",
       "      <td>13</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(285847.138, 327917.905]</th>\n",
       "      <td>31</td>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(327917.905, 369988.672]</th>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(369988.672, 412059.439]</th>\n",
       "      <td>21</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(412059.439, 454130.206]</th>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(454130.206, 496200.973]</th>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(496200.973, 538271.74]</th>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          count  not_sold  sold\n",
       "price_interval                                 \n",
       "(117143.362, 159634.837]      3         1     2\n",
       "(159634.837, 201705.604]     32         4    28\n",
       "(201705.604, 243776.371]     88        20    68\n",
       "(243776.371, 285847.138]     47        13    34\n",
       "(285847.138, 327917.905]     31         7    24\n",
       "(327917.905, 369988.672]     18         6    12\n",
       "(369988.672, 412059.439]     21         8    13\n",
       "(412059.439, 454130.206]      7         3     4\n",
       "(454130.206, 496200.973]     11         4     7\n",
       "(496200.973, 538271.74]       9         6     3"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To identify properties that remain unsold, we can subtract the sold properties from the total count.\n",
    "# We store this in a new column named 'not_sold' in the original dataframe.\n",
    "all_properties_by_price['not_sold'] = all_properties_by_price['count'] - sold_properties_by_price['sold']\n",
    "all_properties_by_price['sold'] = sold_properties_by_price['sold']\n",
    "all_properties_by_price"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Relationship between age and price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>building</th>\n",
       "      <th>date_sale</th>\n",
       "      <th>type</th>\n",
       "      <th>property#</th>\n",
       "      <th>area</th>\n",
       "      <th>sold</th>\n",
       "      <th>customerid</th>\n",
       "      <th>price$</th>\n",
       "      <th>individual</th>\n",
       "      <th>...</th>\n",
       "      <th>state</th>\n",
       "      <th>purpose</th>\n",
       "      <th>deal_satisfaction</th>\n",
       "      <th>mortgage</th>\n",
       "      <th>source</th>\n",
       "      <th>full_name</th>\n",
       "      <th>age_at_purchase</th>\n",
       "      <th>age_at_purchase_rounded</th>\n",
       "      <th>age_interval</th>\n",
       "      <th>price_interval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1030</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>30</td>\n",
       "      <td>743.09</td>\n",
       "      <td>1</td>\n",
       "      <td>C0028</td>\n",
       "      <td>246172.68</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Madalyn Mercer</td>\n",
       "      <td>19.378082</td>\n",
       "      <td>19.0</td>\n",
       "      <td>(19.0, 25.0]</td>\n",
       "      <td>(243776.371, 285847.138]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1029</td>\n",
       "      <td>1</td>\n",
       "      <td>2005-10-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>29</td>\n",
       "      <td>756.21</td>\n",
       "      <td>1</td>\n",
       "      <td>C0027</td>\n",
       "      <td>246331.90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Lara Carrillo</td>\n",
       "      <td>22.616438</td>\n",
       "      <td>22.0</td>\n",
       "      <td>(19.0, 25.0]</td>\n",
       "      <td>(243776.371, 285847.138]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2002</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-07-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>2</td>\n",
       "      <td>587.28</td>\n",
       "      <td>1</td>\n",
       "      <td>C0112</td>\n",
       "      <td>209280.91</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>client</td>\n",
       "      <td>Donavan Flowers</td>\n",
       "      <td>21.523288</td>\n",
       "      <td>21.0</td>\n",
       "      <td>(19.0, 25.0]</td>\n",
       "      <td>(201705.604, 243776.371]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2031</td>\n",
       "      <td>2</td>\n",
       "      <td>2007-12-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>31</td>\n",
       "      <td>1604.75</td>\n",
       "      <td>1</td>\n",
       "      <td>C0160</td>\n",
       "      <td>452667.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>California</td>\n",
       "      <td>investment</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>website</td>\n",
       "      <td>Darien Dorsey</td>\n",
       "      <td>21.942466</td>\n",
       "      <td>21.0</td>\n",
       "      <td>(19.0, 25.0]</td>\n",
       "      <td>(412059.439, 454130.206]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1049</td>\n",
       "      <td>1</td>\n",
       "      <td>2004-11-01</td>\n",
       "      <td>apartment</td>\n",
       "      <td>49</td>\n",
       "      <td>1375.45</td>\n",
       "      <td>1</td>\n",
       "      <td>C0014</td>\n",
       "      <td>467083.31</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>California</td>\n",
       "      <td>home</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>agency</td>\n",
       "      <td>Alessandra Perry</td>\n",
       "      <td>25.484932</td>\n",
       "      <td>25.0</td>\n",
       "      <td>(25.0, 31.0]</td>\n",
       "      <td>(454130.206, 496200.973]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>5044</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>44</td>\n",
       "      <td>1238.58</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>322610.74</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(285847.138, 327917.905]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>5047</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>47</td>\n",
       "      <td>794.52</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>279191.26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(243776.371, 285847.138]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>5048</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>48</td>\n",
       "      <td>1013.27</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287996.53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(285847.138, 327917.905]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>5050</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>50</td>\n",
       "      <td>1074.71</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>365868.78</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(327917.905, 369988.672]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>5051</td>\n",
       "      <td>5</td>\n",
       "      <td>NaT</td>\n",
       "      <td>apartment</td>\n",
       "      <td>51</td>\n",
       "      <td>789.25</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>199216.40</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(159634.837, 201705.604]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>267 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id building  date_sale       type property#     area  sold customerid  \\\n",
       "0    1030        1 2005-11-01  apartment        30   743.09     1      C0028   \n",
       "1    1029        1 2005-10-01  apartment        29   756.21     1      C0027   \n",
       "2    2002        2 2007-07-01  apartment         2   587.28     1      C0112   \n",
       "3    2031        2 2007-12-01  apartment        31  1604.75     1      C0160   \n",
       "4    1049        1 2004-11-01  apartment        49  1375.45     1      C0014   \n",
       "..    ...      ...        ...        ...       ...      ...   ...        ...   \n",
       "262  5044        5        NaT  apartment        44  1238.58     0        NaN   \n",
       "263  5047        5        NaT  apartment        47   794.52     0        NaN   \n",
       "264  5048        5        NaT  apartment        48  1013.27     0        NaN   \n",
       "265  5050        5        NaT  apartment        50  1074.71     0        NaN   \n",
       "266  5051        5        NaT  apartment        51   789.25     0        NaN   \n",
       "\n",
       "        price$  individual  ...       state     purpose deal_satisfaction  \\\n",
       "0    246172.68         1.0  ...  California        home               5.0   \n",
       "1    246331.90         1.0  ...  California        home               5.0   \n",
       "2    209280.91         1.0  ...  California        home               1.0   \n",
       "3    452667.01         1.0  ...  California  investment               3.0   \n",
       "4    467083.31         1.0  ...  California        home               4.0   \n",
       "..         ...         ...  ...         ...         ...               ...   \n",
       "262  322610.74         NaN  ...         NaN         NaN               NaN   \n",
       "263  279191.26         NaN  ...         NaN         NaN               NaN   \n",
       "264  287996.53         NaN  ...         NaN         NaN               NaN   \n",
       "265  365868.78         NaN  ...         NaN         NaN               NaN   \n",
       "266  199216.40         NaN  ...         NaN         NaN               NaN   \n",
       "\n",
       "    mortgage   source         full_name  age_at_purchase  \\\n",
       "0        0.0  website    Madalyn Mercer        19.378082   \n",
       "1        0.0  website     Lara Carrillo        22.616438   \n",
       "2        1.0   client   Donavan Flowers        21.523288   \n",
       "3        1.0  website     Darien Dorsey        21.942466   \n",
       "4        0.0   agency  Alessandra Perry        25.484932   \n",
       "..       ...      ...               ...              ...   \n",
       "262      NaN      NaN               NaN              NaN   \n",
       "263      NaN      NaN               NaN              NaN   \n",
       "264      NaN      NaN               NaN              NaN   \n",
       "265      NaN      NaN               NaN              NaN   \n",
       "266      NaN      NaN               NaN              NaN   \n",
       "\n",
       "    age_at_purchase_rounded  age_interval            price_interval  \n",
       "0                      19.0  (19.0, 25.0]  (243776.371, 285847.138]  \n",
       "1                      22.0  (19.0, 25.0]  (243776.371, 285847.138]  \n",
       "2                      21.0  (19.0, 25.0]  (201705.604, 243776.371]  \n",
       "3                      21.0  (19.0, 25.0]  (412059.439, 454130.206]  \n",
       "4                      25.0  (25.0, 31.0]  (454130.206, 496200.973]  \n",
       "..                      ...           ...                       ...  \n",
       "262                     NaN           NaN  (285847.138, 327917.905]  \n",
       "263                     NaN           NaN  (243776.371, 285847.138]  \n",
       "264                     NaN           NaN  (285847.138, 327917.905]  \n",
       "265                     NaN           NaN  (327917.905, 369988.672]  \n",
       "266                     NaN           NaN  (159634.837, 201705.604]  \n",
       "\n",
       "[267 rows x 23 columns]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter out only the sold apartments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Next, let's filter out the properties that have been sold.\n",
    "# This will be done in a new variable by filtering the 'sold' column for the value of 1.\n",
    "data_sold = data[data['sold']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Further, let's exclude any company data, focusing only on individual sales.\n",
    "data_sold = data_sold[data_sold['individual']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_at_purchase</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19.378082</td>\n",
       "      <td>246172.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22.616438</td>\n",
       "      <td>246331.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>21.523288</td>\n",
       "      <td>209280.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21.942466</td>\n",
       "      <td>452667.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>25.484932</td>\n",
       "      <td>467083.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>70.846575</td>\n",
       "      <td>204292.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>72.849315</td>\n",
       "      <td>261579.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>73.038356</td>\n",
       "      <td>222867.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>72.778082</td>\n",
       "      <td>291494.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>76.347945</td>\n",
       "      <td>296483.14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>178 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age_at_purchase     price$\n",
       "0          19.378082  246172.68\n",
       "1          22.616438  246331.90\n",
       "2          21.523288  209280.91\n",
       "3          21.942466  452667.01\n",
       "4          25.484932  467083.31\n",
       "..               ...        ...\n",
       "173        70.846575  204292.49\n",
       "174        72.849315  261579.89\n",
       "175        73.038356  222867.42\n",
       "176        72.778082  291494.36\n",
       "177        76.347945  296483.14\n",
       "\n",
       "[178 rows x 2 columns]"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now, let's narrow our focus to the 'age_at_purchase' and 'price$' columns.\n",
    "data_sold[['age_at_purchase','price$']]\n",
    "\n",
    "# Be aware that some observations may not have an 'age' value as the data comes from companies."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Covariance of age and price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.68344293e+02, -1.77726142e+05],\n",
       "       [-1.77726142e+05,  6.16619957e+09]])"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's calculate the covariance of 'age_at_purchase' and 'price$'.\n",
    "# Covariance doesn't provide an intuitively understandable value, however, it's useful in statistical analysis.\n",
    "np.cov(data_sold['age_at_purchase'], data_sold['price$'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlation of age and price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.        , -0.17443889],\n",
       "       [-0.17443889,  1.        ]])"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A more interpretable measure is correlation, which can be calculated as follows.\n",
    "np.corrcoef(data_sold['age_at_purchase'], data_sold['price$'])\n",
    "# Note: NumPy doesn't handle missing values effectively, so ensure your data is clean before using these functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.        , -0.19574621],\n",
       "       [-0.19574621,  1.        ]])"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For comprehensive results, let's drop any rows with missing values and store this in a new variable 'data_sold_no_na'.\n",
    "data_sold_no_na = data_sold.dropna()\n",
    "np.corrcoef(data_sold_no_na['age_at_purchase'], data_sold_no_na['price$'])#, ddof=1)\n",
    "\n",
    "# Observe that using degrees of freedom (ddof) to indicate it's a sample, numpy has deprecated it.\n",
    "# The reason behind this is unclear, hence, we'll comply with the recommendations and exclude it from the function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_at_purchase</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>age_at_purchase</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.195746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price$</th>\n",
       "      <td>-0.195746</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 age_at_purchase    price$\n",
       "age_at_purchase         1.000000 -0.195746\n",
       "price$                 -0.195746  1.000000"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# As an alternative, we can use the correlation method provided by pandas, which gives the same result.\n",
    "data_sold_no_na[['age_at_purchase','price$']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_at_purchase</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>age_at_purchase</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.174439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price$</th>\n",
       "      <td>-0.174439</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 age_at_purchase    price$\n",
       "age_at_purchase         1.000000 -0.174439\n",
       "price$                 -0.174439  1.000000"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Interestingly, pandas also handles NA values in its calculations, resulting in different values compared to our cleaned dataset. \n",
    "# This discrepancy arises because pandas employs distinct strategies for NA handling, rather than outright removal (as in dropna()).\n",
    "data_sold[['age_at_purchase','price$']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_at_purchase</th>\n",
       "      <th>price$</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>age_at_purchase</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.174439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>price$</th>\n",
       "      <td>-0.174439</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 age_at_purchase    price$\n",
       "age_at_purchase         1.000000 -0.174439\n",
       "price$                 -0.174439  1.000000"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note: Using the original data (which includes NA values) for correlation calculation would give the same result as the 'data_sold' dataset.\n",
    "data[['age_at_purchase','price$']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}