notebooks/jl-data-cleaning-v1.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Connected to .DATA-PROJ-VENV (Python 3.12.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e7647ace-4e46-40d6-8f8d-91f167649454",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary libraries\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "03fd5b27-2a7f-49ad-9f29-1ebb818b466b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read file from root .\\data\\raw\\\n",
    "script_dir = os.getcwd()\n",
    "listings_file_path = os.path.join(script_dir, \"../data/raw/listings.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5273b497-8085-4b7e-b638-8b3d02fef590",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the CSV file into a DataFrame\n",
    "df_listings = pd.read_csv(listings_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "60bb059f-5adc-443d-86b0-346653a24960",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>host_id</th>\n",
       "      <th>host_name</th>\n",
       "      <th>neighbourhood_group</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>room_type</th>\n",
       "      <th>price</th>\n",
       "      <th>minimum_nights</th>\n",
       "      <th>number_of_reviews</th>\n",
       "      <th>last_review</th>\n",
       "      <th>reviews_per_month</th>\n",
       "      <th>calculated_host_listings_count</th>\n",
       "      <th>availability_365</th>\n",
       "      <th>number_of_reviews_ltm</th>\n",
       "      <th>license</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>197677</td>\n",
       "      <td>Oshiage Holiday Apartment                     ...</td>\n",
       "      <td>964081</td>\n",
       "      <td>Yoshimi &amp; Marek                               ...</td>\n",
       "      <td></td>\n",
       "      <td>Sumida Ku</td>\n",
       "      <td>35.717070</td>\n",
       "      <td>139.826080</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>12000</td>\n",
       "      <td>3</td>\n",
       "      <td>176</td>\n",
       "      <td>2024-05-15</td>\n",
       "      <td>1.13</td>\n",
       "      <td>1</td>\n",
       "      <td>223</td>\n",
       "      <td>4</td>\n",
       "      <td>M130003350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>776070</td>\n",
       "      <td>Kero-kero house room 1                        ...</td>\n",
       "      <td>801494</td>\n",
       "      <td>Kei                                           ...</td>\n",
       "      <td></td>\n",
       "      <td>Kita Ku</td>\n",
       "      <td>35.738440</td>\n",
       "      <td>139.769170</td>\n",
       "      <td>Private room</td>\n",
       "      <td>9652</td>\n",
       "      <td>3</td>\n",
       "      <td>256</td>\n",
       "      <td>2024-06-03</td>\n",
       "      <td>1.81</td>\n",
       "      <td>1</td>\n",
       "      <td>173</td>\n",
       "      <td>13</td>\n",
       "      <td>M130000243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>905944</td>\n",
       "      <td>4F - Near Shinjuku &amp; Shibuya                  ...</td>\n",
       "      <td>4847803</td>\n",
       "      <td>Best Stay In Tokyo!                           ...</td>\n",
       "      <td></td>\n",
       "      <td>Shibuya Ku</td>\n",
       "      <td>35.678780</td>\n",
       "      <td>139.678470</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>25738</td>\n",
       "      <td>3</td>\n",
       "      <td>219</td>\n",
       "      <td>2024-06-19</td>\n",
       "      <td>1.60</td>\n",
       "      <td>6</td>\n",
       "      <td>115</td>\n",
       "      <td>33</td>\n",
       "      <td>Hotels and Inns Business Act | 渋谷区保健所長 | 31渋健生...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1016831</td>\n",
       "      <td>5 mins Shibuya Cat modern sunny  Shimokita    ...</td>\n",
       "      <td>5596383</td>\n",
       "      <td>Wakana                                        ...</td>\n",
       "      <td></td>\n",
       "      <td>Setagaya Ku</td>\n",
       "      <td>35.658000</td>\n",
       "      <td>139.671340</td>\n",
       "      <td>Private room</td>\n",
       "      <td>23286</td>\n",
       "      <td>10</td>\n",
       "      <td>268</td>\n",
       "      <td>2024-06-26</td>\n",
       "      <td>1.96</td>\n",
       "      <td>2</td>\n",
       "      <td>129</td>\n",
       "      <td>24</td>\n",
       "      <td>M130001107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1196177</td>\n",
       "      <td>Stay with host Cozy private room Senju area   ...</td>\n",
       "      <td>5686404</td>\n",
       "      <td>Yukiko                                        ...</td>\n",
       "      <td></td>\n",
       "      <td>Adachi Ku</td>\n",
       "      <td>35.744731</td>\n",
       "      <td>139.797384</td>\n",
       "      <td>Private room</td>\n",
       "      <td>7500</td>\n",
       "      <td>2</td>\n",
       "      <td>120</td>\n",
       "      <td>2024-06-15</td>\n",
       "      <td>0.91</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>26</td>\n",
       "      <td>M130007760</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                    \\\n",
       "0                197677   \n",
       "1                776070   \n",
       "2                905944   \n",
       "3               1016831   \n",
       "4               1196177   \n",
       "\n",
       "  name                                                                                                            \\\n",
       "0  Oshiage Holiday Apartment                     ...                                                               \n",
       "1  Kero-kero house room 1                        ...                                                               \n",
       "2  4F - Near Shinjuku & Shibuya                  ...                                                               \n",
       "3  5 mins Shibuya Cat modern sunny  Shimokita    ...                                                               \n",
       "4  Stay with host Cozy private room Senju area   ...                                                               \n",
       "\n",
       "   host_id    host_name                                               \\\n",
       "0      964081  Yoshimi & Marek                               ...       \n",
       "1      801494  Kei                                           ...       \n",
       "2     4847803  Best Stay In Tokyo!                           ...       \n",
       "3     5596383  Wakana                                        ...       \n",
       "4     5686404  Yukiko                                        ...       \n",
       "\n",
       "   neighbourhood_group   neighbourhood         latitude             \\\n",
       "0                        Sumida Ku                       35.717070   \n",
       "1                        Kita Ku                         35.738440   \n",
       "2                        Shibuya Ku                      35.678780   \n",
       "3                        Setagaya Ku                     35.658000   \n",
       "4                        Adachi Ku                       35.744731   \n",
       "\n",
       "   longitude            room_type         price    minimum_nights   \\\n",
       "0           139.826080  Entire home/apt    12000                 3   \n",
       "1           139.769170  Private room        9652                 3   \n",
       "2           139.678470  Entire home/apt    25738                 3   \n",
       "3           139.671340  Private room       23286                10   \n",
       "4           139.797384  Private room        7500                 2   \n",
       "\n",
       "   number_of_reviews   last_review   reviews_per_month   \\\n",
       "0                 176  2024-05-15                 1.13    \n",
       "1                 256  2024-06-03                 1.81    \n",
       "2                 219  2024-06-19                 1.60    \n",
       "3                 268  2024-06-26                 1.96    \n",
       "4                 120  2024-06-15                 0.91    \n",
       "\n",
       "   calculated_host_listings_count   availability_365   number_of_reviews_ltm   \\\n",
       "0                                1                223                       4   \n",
       "1                                1                173                      13   \n",
       "2                                6                115                      33   \n",
       "3                                2                129                      24   \n",
       "4                                1                 36                      26   \n",
       "\n",
       "                                             license  \n",
       "0                                         M130003350  \n",
       "1                                         M130000243  \n",
       "2  Hotels and Inns Business Act | 渋谷区保健所長 | 31渋健生...  \n",
       "3                                         M130001107  \n",
       "4                                         M130007760  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Preview Dataframes\n",
    "df_listings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0b29ccb6-ad32-4a25-af01-7d11e600bbb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 16518 entries, 0 to 16517\n",
      "Data columns (total 18 columns):\n",
      " #   Column                                                                                                          Non-Null Count  Dtype  \n",
      "---  ------                                                                                                          --------------  -----  \n",
      " 0   id                                                                                                              16518 non-null  int64  \n",
      " 1   name                                                                                                            16518 non-null  object \n",
      " 2   host_id                                                                                                         16518 non-null  int64  \n",
      " 3   host_name                                                                                                       16518 non-null  object \n",
      " 4   neighbourhood_group                                                                                             16518 non-null  object \n",
      " 5   neighbourhood                                                                                                   16518 non-null  object \n",
      " 6   latitude                                                                                                        16518 non-null  float64\n",
      " 7   longitude                                                                                                       16518 non-null  float64\n",
      " 8   room_type                                                                                                       16518 non-null  object \n",
      " 9   price                                                                                                           16518 non-null  object \n",
      " 10  minimum_nights                                                                                                  16518 non-null  int64  \n",
      " 11  number_of_reviews                                                                                               16518 non-null  int64  \n",
      " 12  last_review                                                                                                     16518 non-null  object \n",
      " 13  reviews_per_month                                                                                               16518 non-null  object \n",
      " 14  calculated_host_listings_count                                                                                  16518 non-null  int64  \n",
      " 15  availability_365                                                                                                16518 non-null  int64  \n",
      " 16  number_of_reviews_ltm                                                                                           16518 non-null  int64  \n",
      " 17  license                                                                                                         16508 non-null  object \n",
      "dtypes: float64(2), int64(7), object(9)\n",
      "memory usage: 2.3+ MB\n"
     ]
    }
   ],
   "source": [
    "# Overview of df_listings\n",
    "df_listings.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "092e7e4e-d023-4560-8339-d3c61a14834f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>host_id</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>minimum_nights</th>\n",
       "      <th>number_of_reviews</th>\n",
       "      <th>calculated_host_listings_count</th>\n",
       "      <th>availability_365</th>\n",
       "      <th>number_of_reviews_ltm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.651800e+04</td>\n",
       "      <td>1.651800e+04</td>\n",
       "      <td>16518.000000</td>\n",
       "      <td>16518.000000</td>\n",
       "      <td>16518.000000</td>\n",
       "      <td>16518.000000</td>\n",
       "      <td>16518.000000</td>\n",
       "      <td>16518.000000</td>\n",
       "      <td>16518.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>5.816491e+17</td>\n",
       "      <td>2.929799e+08</td>\n",
       "      <td>35.698748</td>\n",
       "      <td>139.736294</td>\n",
       "      <td>3.648868</td>\n",
       "      <td>37.783691</td>\n",
       "      <td>16.109335</td>\n",
       "      <td>154.140756</td>\n",
       "      <td>15.071013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4.970662e+17</td>\n",
       "      <td>1.846269e+08</td>\n",
       "      <td>0.041095</td>\n",
       "      <td>0.072995</td>\n",
       "      <td>8.920520</td>\n",
       "      <td>64.014216</td>\n",
       "      <td>19.674860</td>\n",
       "      <td>99.525334</td>\n",
       "      <td>21.505364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.976770e+05</td>\n",
       "      <td>3.222340e+05</td>\n",
       "      <td>35.520940</td>\n",
       "      <td>139.081322</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.144522e+07</td>\n",
       "      <td>1.290963e+08</td>\n",
       "      <td>35.688305</td>\n",
       "      <td>139.699183</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.301174e+17</td>\n",
       "      <td>2.721294e+08</td>\n",
       "      <td>35.703990</td>\n",
       "      <td>139.728136</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>144.000000</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.040939e+18</td>\n",
       "      <td>4.923137e+08</td>\n",
       "      <td>35.722587</td>\n",
       "      <td>139.790456</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>45.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>234.000000</td>\n",
       "      <td>22.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.189054e+18</td>\n",
       "      <td>5.859819e+08</td>\n",
       "      <td>35.840764</td>\n",
       "      <td>139.914020</td>\n",
       "      <td>365.000000</td>\n",
       "      <td>2660.000000</td>\n",
       "      <td>106.000000</td>\n",
       "      <td>365.000000</td>\n",
       "      <td>815.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id                      host_id     latitude             \\\n",
       "count          1.651800e+04  1.651800e+04         16518.000000   \n",
       "mean           5.816491e+17  2.929799e+08            35.698748   \n",
       "std            4.970662e+17  1.846269e+08             0.041095   \n",
       "min            1.976770e+05  3.222340e+05            35.520940   \n",
       "25%            4.144522e+07  1.290963e+08            35.688305   \n",
       "50%            8.301174e+17  2.721294e+08            35.703990   \n",
       "75%            1.040939e+18  4.923137e+08            35.722587   \n",
       "max            1.189054e+18  5.859819e+08            35.840764   \n",
       "\n",
       "       longitude            minimum_nights   number_of_reviews   \\\n",
       "count         16518.000000     16518.000000        16518.000000   \n",
       "mean            139.736294         3.648868           37.783691   \n",
       "std               0.072995         8.920520           64.014216   \n",
       "min             139.081322         1.000000            0.000000   \n",
       "25%             139.699183         1.000000            4.000000   \n",
       "50%             139.728136         2.000000           17.000000   \n",
       "75%             139.790456         2.000000           45.000000   \n",
       "max             139.914020       365.000000         2660.000000   \n",
       "\n",
       "       calculated_host_listings_count   availability_365   \\\n",
       "count                     16518.000000       16518.000000   \n",
       "mean                         16.109335         154.140756   \n",
       "std                          19.674860          99.525334   \n",
       "min                           1.000000           0.000000   \n",
       "25%                           3.000000          75.000000   \n",
       "50%                           9.000000         144.000000   \n",
       "75%                          19.000000         234.000000   \n",
       "max                         106.000000         365.000000   \n",
       "\n",
       "       number_of_reviews_ltm   \n",
       "count            16518.000000  \n",
       "mean                15.071013  \n",
       "std                 21.505364  \n",
       "min                  0.000000  \n",
       "25%                  2.000000  \n",
       "50%                 10.000000  \n",
       "75%                 22.000000  \n",
       "max                815.000000  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_listings.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "18105a08-12de-45ab-aff7-5432ceb42d3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id                  ',\n",
       "       'name                                                                                                          ',\n",
       "       'host_id   ', 'host_name                                             ',\n",
       "       'neighbourhood_group ', 'neighbourhood       ', 'latitude           ',\n",
       "       'longitude          ', 'room_type       ', 'price  ', 'minimum_nights ',\n",
       "       'number_of_reviews ', 'last_review ', 'reviews_per_month ',\n",
       "       'calculated_host_listings_count ', 'availability_365 ',\n",
       "       'number_of_reviews_ltm ', 'license'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_listings.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "abd9ac69-ee57-4087-a354-dcc8f156da3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cleaning df_listings\n",
    "listings_clean = (\n",
    "    df_listings.copy()\n",
    ")  # Creating copy of the df_listings before making changes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "734a1c49-b01b-4d2d-955e-40161c324a62",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cleaning column names since they contain white spaces\n",
    "listings_clean.columns = listings_clean.columns.str.strip().str.lower()\n",
    "listings_clean = (\n",
    "    listings_clean.drop(\n",
    "        columns=[\n",
    "            \"neighbourhood_group\",\n",
    "            \"minimum_nights\",\n",
    "            \"number_of_reviews\",\n",
    "            \"last_review\",\n",
    "            \"reviews_per_month\",\n",
    "            \"calculated_host_listings_count\",\n",
    "            \"availability_365\",\n",
    "            \"number_of_reviews_ltm\",\n",
    "            \"license\",\n",
    "        ]\n",
    "    ).drop_duplicates()  # Dropping duplicate data\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "04f5a110-315a-4024-9185-83fdd6ac5df7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>host_id</th>\n",
       "      <th>host_name</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>room_type</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>197677</td>\n",
       "      <td>Oshiage Holiday Apartment                     ...</td>\n",
       "      <td>964081</td>\n",
       "      <td>Yoshimi &amp; Marek                               ...</td>\n",
       "      <td>Sumida Ku</td>\n",
       "      <td>35.717070</td>\n",
       "      <td>139.826080</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>12000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>776070</td>\n",
       "      <td>Kero-kero house room 1                        ...</td>\n",
       "      <td>801494</td>\n",
       "      <td>Kei                                           ...</td>\n",
       "      <td>Kita Ku</td>\n",
       "      <td>35.738440</td>\n",
       "      <td>139.769170</td>\n",
       "      <td>Private room</td>\n",
       "      <td>9652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>905944</td>\n",
       "      <td>4F - Near Shinjuku &amp; Shibuya                  ...</td>\n",
       "      <td>4847803</td>\n",
       "      <td>Best Stay In Tokyo!                           ...</td>\n",
       "      <td>Shibuya Ku</td>\n",
       "      <td>35.678780</td>\n",
       "      <td>139.678470</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>25738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1016831</td>\n",
       "      <td>5 mins Shibuya Cat modern sunny  Shimokita    ...</td>\n",
       "      <td>5596383</td>\n",
       "      <td>Wakana                                        ...</td>\n",
       "      <td>Setagaya Ku</td>\n",
       "      <td>35.658000</td>\n",
       "      <td>139.671340</td>\n",
       "      <td>Private room</td>\n",
       "      <td>23286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1196177</td>\n",
       "      <td>Stay with host Cozy private room Senju area   ...</td>\n",
       "      <td>5686404</td>\n",
       "      <td>Yukiko                                        ...</td>\n",
       "      <td>Adachi Ku</td>\n",
       "      <td>35.744731</td>\n",
       "      <td>139.797384</td>\n",
       "      <td>Private room</td>\n",
       "      <td>7500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                               name  host_id  \\\n",
       "0   197677  Oshiage Holiday Apartment                     ...   964081   \n",
       "1   776070  Kero-kero house room 1                        ...   801494   \n",
       "2   905944  4F - Near Shinjuku & Shibuya                  ...  4847803   \n",
       "3  1016831  5 mins Shibuya Cat modern sunny  Shimokita    ...  5596383   \n",
       "4  1196177  Stay with host Cozy private room Senju area   ...  5686404   \n",
       "\n",
       "                                           host_name         neighbourhood  \\\n",
       "0  Yoshimi & Marek                               ...  Sumida Ku              \n",
       "1  Kei                                           ...  Kita Ku                \n",
       "2  Best Stay In Tokyo!                           ...  Shibuya Ku             \n",
       "3  Wakana                                        ...  Setagaya Ku            \n",
       "4  Yukiko                                        ...  Adachi Ku              \n",
       "\n",
       "    latitude   longitude         room_type    price  \n",
       "0  35.717070  139.826080  Entire home/apt    12000   \n",
       "1  35.738440  139.769170  Private room        9652   \n",
       "2  35.678780  139.678470  Entire home/apt    25738   \n",
       "3  35.658000  139.671340  Private room       23286   \n",
       "4  35.744731  139.797384  Private room        7500   "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Quick check to see if changes were made\n",
    "listings_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e4cf68af-afd1-4b80-a20f-6771e8b27d41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 16518 entries, 0 to 16517\n",
      "Data columns (total 9 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   id             16518 non-null  int64  \n",
      " 1   name           16518 non-null  object \n",
      " 2   host_id        16518 non-null  int64  \n",
      " 3   host_name      16518 non-null  object \n",
      " 4   neighbourhood  16518 non-null  object \n",
      " 5   latitude       16518 non-null  float64\n",
      " 6   longitude      16518 non-null  float64\n",
      " 7   room_type      16518 non-null  object \n",
      " 8   price          16518 non-null  object \n",
      "dtypes: float64(2), int64(2), object(5)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "listings_clean.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2534d656-60d7-4a37-b967-f77604e36ed8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replacing non-ASCII characters with blank spaces.\n",
    "listings_clean[\"name\"] = listings_clean[\"name\"].apply(\n",
    "    lambda x: \"\" if any(ord(char) > 127 for char in x) else x\n",
    ")\n",
    "listings_clean[\"host_name\"] = listings_clean[\"host_name\"].apply(\n",
    "    lambda x: \"\" if any(ord(char) > 127 for char in x) else x\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8e724449-b2fc-4f76-9fd6-e7102008c762",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace empty strings in the 'price' column with NaN.\n",
    "listings_clean[\"price\"] = pd.to_numeric(\n",
    "    listings_clean[\"price\"].replace(\"\", np.nan), errors=\"coerce\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6abafe18-fd58-4e18-a1b6-66a80ab0fb1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop rows with NaN values in 'price'\n",
    "listings_clean = listings_clean.dropna(subset=[\"price\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4ad717d6-e3a8-45b2-ae98-ed9de7ea7b93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to int64 (this removes decimal places)\n",
    "listings_clean[\"price\"] = listings_clean[\"price\"].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e83f2ab1-1e33-49a7-80a2-fa39b8982f72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>host_id</th>\n",
       "      <th>host_name</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>room_type</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>197677</td>\n",
       "      <td>Oshiage Holiday Apartment                     ...</td>\n",
       "      <td>964081</td>\n",
       "      <td>Yoshimi &amp; Marek                               ...</td>\n",
       "      <td>Sumida Ku</td>\n",
       "      <td>35.717070</td>\n",
       "      <td>139.826080</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>12000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>776070</td>\n",
       "      <td>Kero-kero house room 1                        ...</td>\n",
       "      <td>801494</td>\n",
       "      <td>Kei                                           ...</td>\n",
       "      <td>Kita Ku</td>\n",
       "      <td>35.738440</td>\n",
       "      <td>139.769170</td>\n",
       "      <td>Private room</td>\n",
       "      <td>9652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>905944</td>\n",
       "      <td>4F - Near Shinjuku &amp; Shibuya                  ...</td>\n",
       "      <td>4847803</td>\n",
       "      <td>Best Stay In Tokyo!                           ...</td>\n",
       "      <td>Shibuya Ku</td>\n",
       "      <td>35.678780</td>\n",
       "      <td>139.678470</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>25738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1016831</td>\n",
       "      <td>5 mins Shibuya Cat modern sunny  Shimokita    ...</td>\n",
       "      <td>5596383</td>\n",
       "      <td>Wakana                                        ...</td>\n",
       "      <td>Setagaya Ku</td>\n",
       "      <td>35.658000</td>\n",
       "      <td>139.671340</td>\n",
       "      <td>Private room</td>\n",
       "      <td>23286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1196177</td>\n",
       "      <td>Stay with host Cozy private room Senju area   ...</td>\n",
       "      <td>5686404</td>\n",
       "      <td>Yukiko                                        ...</td>\n",
       "      <td>Adachi Ku</td>\n",
       "      <td>35.744731</td>\n",
       "      <td>139.797384</td>\n",
       "      <td>Private room</td>\n",
       "      <td>7500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                               name  host_id  \\\n",
       "0   197677  Oshiage Holiday Apartment                     ...   964081   \n",
       "1   776070  Kero-kero house room 1                        ...   801494   \n",
       "2   905944  4F - Near Shinjuku & Shibuya                  ...  4847803   \n",
       "3  1016831  5 mins Shibuya Cat modern sunny  Shimokita    ...  5596383   \n",
       "4  1196177  Stay with host Cozy private room Senju area   ...  5686404   \n",
       "\n",
       "                                           host_name         neighbourhood  \\\n",
       "0  Yoshimi & Marek                               ...  Sumida Ku              \n",
       "1  Kei                                           ...  Kita Ku                \n",
       "2  Best Stay In Tokyo!                           ...  Shibuya Ku             \n",
       "3  Wakana                                        ...  Setagaya Ku            \n",
       "4  Yukiko                                        ...  Adachi Ku              \n",
       "\n",
       "    latitude   longitude         room_type  price  \n",
       "0  35.717070  139.826080  Entire home/apt   12000  \n",
       "1  35.738440  139.769170  Private room       9652  \n",
       "2  35.678780  139.678470  Entire home/apt   25738  \n",
       "3  35.658000  139.671340  Private room      23286  \n",
       "4  35.744731  139.797384  Private room       7500  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Quick check to see if changes were made\n",
    "listings_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "ffefd40c-8f12-4257-866c-5bc114729cf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 14805 entries, 0 to 16517\n",
      "Data columns (total 9 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   id             14805 non-null  int64  \n",
      " 1   name           14805 non-null  object \n",
      " 2   host_id        14805 non-null  int64  \n",
      " 3   host_name      14805 non-null  object \n",
      " 4   neighbourhood  14805 non-null  object \n",
      " 5   latitude       14805 non-null  float64\n",
      " 6   longitude      14805 non-null  float64\n",
      " 7   room_type      14805 non-null  object \n",
      " 8   price          14805 non-null  int64  \n",
      "dtypes: float64(2), int64(3), object(4)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "listings_clean.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e66bba6b-9dc8-49ba-91f5-349dd74b777a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a copy of the cleaned listings DataFrame\n",
    "df_listings_cleaned = listings_clean.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "27716edb-1da7-4582-b880-70337bf882c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>host_id</th>\n",
       "      <th>host_name</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>room_type</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>197677</td>\n",
       "      <td>Oshiage Holiday Apartment                     ...</td>\n",
       "      <td>964081</td>\n",
       "      <td>Yoshimi &amp; Marek                               ...</td>\n",
       "      <td>Sumida Ku</td>\n",
       "      <td>35.717070</td>\n",
       "      <td>139.826080</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>12000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>776070</td>\n",
       "      <td>Kero-kero house room 1                        ...</td>\n",
       "      <td>801494</td>\n",
       "      <td>Kei                                           ...</td>\n",
       "      <td>Kita Ku</td>\n",
       "      <td>35.738440</td>\n",
       "      <td>139.769170</td>\n",
       "      <td>Private room</td>\n",
       "      <td>9652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>905944</td>\n",
       "      <td>4F - Near Shinjuku &amp; Shibuya                  ...</td>\n",
       "      <td>4847803</td>\n",
       "      <td>Best Stay In Tokyo!                           ...</td>\n",
       "      <td>Shibuya Ku</td>\n",
       "      <td>35.678780</td>\n",
       "      <td>139.678470</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>25738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1016831</td>\n",
       "      <td>5 mins Shibuya Cat modern sunny  Shimokita    ...</td>\n",
       "      <td>5596383</td>\n",
       "      <td>Wakana                                        ...</td>\n",
       "      <td>Setagaya Ku</td>\n",
       "      <td>35.658000</td>\n",
       "      <td>139.671340</td>\n",
       "      <td>Private room</td>\n",
       "      <td>23286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1196177</td>\n",
       "      <td>Stay with host Cozy private room Senju area   ...</td>\n",
       "      <td>5686404</td>\n",
       "      <td>Yukiko                                        ...</td>\n",
       "      <td>Adachi Ku</td>\n",
       "      <td>35.744731</td>\n",
       "      <td>139.797384</td>\n",
       "      <td>Private room</td>\n",
       "      <td>7500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                               name  host_id  \\\n",
       "0   197677  Oshiage Holiday Apartment                     ...   964081   \n",
       "1   776070  Kero-kero house room 1                        ...   801494   \n",
       "2   905944  4F - Near Shinjuku & Shibuya                  ...  4847803   \n",
       "3  1016831  5 mins Shibuya Cat modern sunny  Shimokita    ...  5596383   \n",
       "4  1196177  Stay with host Cozy private room Senju area   ...  5686404   \n",
       "\n",
       "                                           host_name         neighbourhood  \\\n",
       "0  Yoshimi & Marek                               ...  Sumida Ku              \n",
       "1  Kei                                           ...  Kita Ku                \n",
       "2  Best Stay In Tokyo!                           ...  Shibuya Ku             \n",
       "3  Wakana                                        ...  Setagaya Ku            \n",
       "4  Yukiko                                        ...  Adachi Ku              \n",
       "\n",
       "    latitude   longitude         room_type  price  \n",
       "0  35.717070  139.826080  Entire home/apt   12000  \n",
       "1  35.738440  139.769170  Private room       9652  \n",
       "2  35.678780  139.678470  Entire home/apt   25738  \n",
       "3  35.658000  139.671340  Private room      23286  \n",
       "4  35.744731  139.797384  Private room       7500  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_listings_cleaned.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "122bfdfe-20e9-4cc9-98be-30a520896e8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating directory path for export of cleaned data.\n",
    "clean_data_dir = os.path.join(\"..\", \"..\", \"data\", \"clean\")\n",
    "cleaned_listings_export_path = os.path.abspath(\n",
    "    os.path.join(script_dir, clean_data_dir, \"cleaned_listings.csv\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "64fd287b-ce6a-4e52-aba4-4e1be698eabf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Cleaning Completed!\n"
     ]
    }
   ],
   "source": [
    "# Exporting cleaned data to directory.\n",
    "df_listings_cleaned.to_csv(cleaned_listings_export_path, index=False)\n",
    "print(\"Data Cleaning Completed!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".DATA-PROJ-VENV",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}