evaluation_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Booking.com Multi-Destinations Trips Dataset demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1166835, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>checkin</th>\n",
       "      <th>checkout</th>\n",
       "      <th>city_id</th>\n",
       "      <th>device_class</th>\n",
       "      <th>affiliate_id</th>\n",
       "      <th>booker_country</th>\n",
       "      <th>hotel_country</th>\n",
       "      <th>utrip_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000027</td>\n",
       "      <td>2016-08-13</td>\n",
       "      <td>2016-08-14</td>\n",
       "      <td>8183</td>\n",
       "      <td>desktop</td>\n",
       "      <td>7168</td>\n",
       "      <td>Elbonia</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>1000027_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000027</td>\n",
       "      <td>2016-08-14</td>\n",
       "      <td>2016-08-16</td>\n",
       "      <td>15626</td>\n",
       "      <td>desktop</td>\n",
       "      <td>7168</td>\n",
       "      <td>Elbonia</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>1000027_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000027</td>\n",
       "      <td>2016-08-16</td>\n",
       "      <td>2016-08-18</td>\n",
       "      <td>60902</td>\n",
       "      <td>desktop</td>\n",
       "      <td>7168</td>\n",
       "      <td>Elbonia</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>1000027_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000027</td>\n",
       "      <td>2016-08-18</td>\n",
       "      <td>2016-08-21</td>\n",
       "      <td>30628</td>\n",
       "      <td>desktop</td>\n",
       "      <td>253</td>\n",
       "      <td>Elbonia</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>1000027_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000033</td>\n",
       "      <td>2016-04-09</td>\n",
       "      <td>2016-04-11</td>\n",
       "      <td>38677</td>\n",
       "      <td>mobile</td>\n",
       "      <td>359</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>Cobra Island</td>\n",
       "      <td>1000033_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id     checkin    checkout  city_id device_class  affiliate_id  \\\n",
       "0  1000027  2016-08-13  2016-08-14     8183      desktop          7168   \n",
       "1  1000027  2016-08-14  2016-08-16    15626      desktop          7168   \n",
       "2  1000027  2016-08-16  2016-08-18    60902      desktop          7168   \n",
       "3  1000027  2016-08-18  2016-08-21    30628      desktop           253   \n",
       "4  1000033  2016-04-09  2016-04-11    38677       mobile           359   \n",
       "\n",
       "  booker_country hotel_country   utrip_id  \n",
       "0        Elbonia        Gondal  1000027_1  \n",
       "1        Elbonia        Gondal  1000027_1  \n",
       "2        Elbonia        Gondal  1000027_1  \n",
       "3        Elbonia        Gondal  1000027_1  \n",
       "4         Gondal  Cobra Island  1000033_1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set = pd.read_csv('train_set.csv').sort_values(by=['utrip_id','checkin'])\n",
    "\n",
    "print(train_set.shape)\n",
    "train_set.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(378667, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>checkin</th>\n",
       "      <th>checkout</th>\n",
       "      <th>device_class</th>\n",
       "      <th>affiliate_id</th>\n",
       "      <th>booker_country</th>\n",
       "      <th>utrip_id</th>\n",
       "      <th>city_id</th>\n",
       "      <th>hotel_country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000066</td>\n",
       "      <td>2016-07-21</td>\n",
       "      <td>2016-07-23</td>\n",
       "      <td>desktop</td>\n",
       "      <td>9924</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>1000066_2</td>\n",
       "      <td>56430</td>\n",
       "      <td>Urkesh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000066</td>\n",
       "      <td>2016-07-23</td>\n",
       "      <td>2016-07-25</td>\n",
       "      <td>desktop</td>\n",
       "      <td>9924</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>1000066_2</td>\n",
       "      <td>41971</td>\n",
       "      <td>Urkesh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000066</td>\n",
       "      <td>2016-07-25</td>\n",
       "      <td>2016-07-28</td>\n",
       "      <td>desktop</td>\n",
       "      <td>9924</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>1000066_2</td>\n",
       "      <td>5797</td>\n",
       "      <td>Urkesh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000066</td>\n",
       "      <td>2016-07-28</td>\n",
       "      <td>2016-07-31</td>\n",
       "      <td>mobile</td>\n",
       "      <td>2436</td>\n",
       "      <td>Gondal</td>\n",
       "      <td>1000066_2</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000270</td>\n",
       "      <td>2016-02-08</td>\n",
       "      <td>2016-02-09</td>\n",
       "      <td>mobile</td>\n",
       "      <td>9452</td>\n",
       "      <td>The Devilfire Empire</td>\n",
       "      <td>1000270_1</td>\n",
       "      <td>50075</td>\n",
       "      <td>The Devilfire Empire</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id     checkin    checkout device_class  affiliate_id  \\\n",
       "0  1000066  2016-07-21  2016-07-23      desktop          9924   \n",
       "1  1000066  2016-07-23  2016-07-25      desktop          9924   \n",
       "2  1000066  2016-07-25  2016-07-28      desktop          9924   \n",
       "3  1000066  2016-07-28  2016-07-31       mobile          2436   \n",
       "4  1000270  2016-02-08  2016-02-09       mobile          9452   \n",
       "\n",
       "         booker_country   utrip_id  city_id         hotel_country  \n",
       "0                Gondal  1000066_2    56430                Urkesh  \n",
       "1                Gondal  1000066_2    41971                Urkesh  \n",
       "2                Gondal  1000066_2     5797                Urkesh  \n",
       "3                Gondal  1000066_2        0                   NaN  \n",
       "4  The Devilfire Empire  1000270_1    50075  The Devilfire Empire  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set = pd.read_csv('test_set.csv').sort_values(by=['utrip_id','checkin'])\n",
    "print(test_set.shape)\n",
    "\n",
    "test_set.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Dummy Predictions - use top 4 cities in the trainset as benchmark recommendation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "topcities = train_set.city_id.value_counts().index[:4]\n",
    "\n",
    "test_trips = (test_set[['utrip_id']].drop_duplicates()).reset_index().drop('index', axis=1)\n",
    "cities_prediction = pd.DataFrame([topcities]*test_trips.shape[0]\n",
    "                                 , columns= ['city_id_1','city_id_2','city_id_3','city_id_4'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Submission file according to the format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70662, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>utrip_id</th>\n",
       "      <th>city_id_1</th>\n",
       "      <th>city_id_2</th>\n",
       "      <th>city_id_3</th>\n",
       "      <th>city_id_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000066_2</td>\n",
       "      <td>47499</td>\n",
       "      <td>23921</td>\n",
       "      <td>36063</td>\n",
       "      <td>17013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000270_1</td>\n",
       "      <td>47499</td>\n",
       "      <td>23921</td>\n",
       "      <td>36063</td>\n",
       "      <td>17013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000441_1</td>\n",
       "      <td>47499</td>\n",
       "      <td>23921</td>\n",
       "      <td>36063</td>\n",
       "      <td>17013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100048_1</td>\n",
       "      <td>47499</td>\n",
       "      <td>23921</td>\n",
       "      <td>36063</td>\n",
       "      <td>17013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000543_1</td>\n",
       "      <td>47499</td>\n",
       "      <td>23921</td>\n",
       "      <td>36063</td>\n",
       "      <td>17013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    utrip_id  city_id_1  city_id_2  city_id_3  city_id_4\n",
       "0  1000066_2      47499      23921      36063      17013\n",
       "1  1000270_1      47499      23921      36063      17013\n",
       "2  1000441_1      47499      23921      36063      17013\n",
       "3   100048_1      47499      23921      36063      17013\n",
       "4  1000543_1      47499      23921      36063      17013"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.concat([test_trips,cities_prediction], axis =1)\n",
    "print(submission.shape)\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('submission.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read submission file and ground truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth = pd.read_csv('ground_truth.csv',index_col=[0])\n",
    "submission = pd.read_csv('submission.csv',index_col=[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70662, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city_id</th>\n",
       "      <th>hotel_country</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>utrip_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1038944_1</th>\n",
       "      <td>54085</td>\n",
       "      <td>Sokovia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1068715_1</th>\n",
       "      <td>29319</td>\n",
       "      <td>Cobra Island</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1075528_1</th>\n",
       "      <td>55763</td>\n",
       "      <td>Bozatta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1110462_4</th>\n",
       "      <td>11930</td>\n",
       "      <td>Alvonia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1132565_1</th>\n",
       "      <td>58659</td>\n",
       "      <td>Axphain</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           city_id hotel_country\n",
       "utrip_id                        \n",
       "1038944_1    54085       Sokovia\n",
       "1068715_1    29319  Cobra Island\n",
       "1075528_1    55763       Bozatta\n",
       "1110462_4    11930       Alvonia\n",
       "1132565_1    58659       Axphain"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(ground_truth.shape)\n",
    "ground_truth.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate - use accuracy at 4 to evaluate the prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_accuracy_at_4(submission,ground_truth):\n",
    "    '''checks if the true city is within the four recommended cities'''\n",
    "    data = submission.join(ground_truth,on='utrip_id')\n",
    "\n",
    "    hits = ((data['city_id']==data['city_id_1'])|(data['city_id']==data['city_id_2'])|\n",
    "        (data['city_id']==data['city_id_3'])|(data['city_id']==data['city_id_4']))*1\n",
    "    return hits.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.05271574537941185"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_accuracy_at_4(submission,ground_truth)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}