data-prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Data Preparation"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Setup"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "%%capture\n",
    "%pip install kaggle"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "import pandas as pd"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "SAMPLE_NUM_CUSTOMERS = 1000 #set to 0 or less for no sampling\n",
    "FILTER_ARTICLES = False #whether to filter out certain intimate products for demo purposes (real data problems)\n",
    "RANDOM_SEED = 7474 #seed to use for replicating sampling"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "def camel_case(s):\n",
    "    ss = s.lower().split('_')\n",
    "    if len(ss) == 1:\n",
    "        return ss[0]\n",
    "    return ss[0] + ''.join(st.title() for st in ss[1:])\n",
    "\n",
    "def camel_case_dict(name_keys):\n",
    "    name_values = [camel_case(s) for s in name_keys]\n",
    "    return dict(zip(name_keys, name_values))\n",
    "\n",
    "def camel_case_rename_cols(df):\n",
    "    col_map = camel_case_dict(df.columns)\n",
    "    return df.rename(columns=col_map)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Get Source Data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "articles.csv.zip: Skipping, found more recently modified local copy (use --force to force download)\r\n",
      "customers.csv.zip: Skipping, found more recently modified local copy (use --force to force download)\r\n",
      "transactions_train.csv.zip: Skipping, found more recently modified local copy (use --force to force download)\r\n",
      "Archive:  customers.csv.zip\r\n",
      "\r\n",
      "Archive:  articles.csv.zip\r\n",
      "\r\n",
      "Archive:  transactions_train.csv.zip\r\n",
      "\r\n",
      "3 archives were successfully processed.\r\n"
     ]
    }
   ],
   "source": [
    "# configure authentication per instruction @ https://github.com/Kaggle/kaggle-api/blob/main/README.md\n",
    "!kaggle competitions download -c h-and-m-personalized-fashion-recommendations -f articles.csv -p data\n",
    "!kaggle competitions download -c h-and-m-personalized-fashion-recommendations -f customers.csv -p data\n",
    "!kaggle competitions download -c h-and-m-personalized-fashion-recommendations -f transactions_train.csv -p data\n",
    "!cd data && unzip -n '*.zip'"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Filter Articles"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "(105542, 25)"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_article_df = camel_case_rename_cols(pd.read_csv('data/articles.csv'))\n",
    "init_article_df.shape"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "# Filtering out some intimate products for demo purposes\n",
    "filtered_article_ids = init_article_df.articleId\n",
    "if FILTER_ARTICLES:\n",
    "    filtered_article_ids = init_article_df[init_article_df.garmentGroupName != 'Under-, Nightwear'].articleId"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "(31788324, 6)"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "init_transaction_df = camel_case_rename_cols(pd.read_csv('data/transactions_train.csv'))\n",
    "# generally a good idea to have an id for these (source data doesn't include ids)\n",
    "init_transaction_df['txId'] = range(init_transaction_df.shape[0])\n",
    "init_transaction_df.shape"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "0           000058a12d5b43e67d225668fa1f8d618c13dc232df0ca...\n2           00007d2de826758b65a93dd24ce629ed66842531df6699...\n7           00083cda041544b2fbb0e0d2905ad17da7cf1007526fb4...\n12          0008968c0d451dbc5a9968da03196fe20051965edde741...\n14          000aa7f0dc06cd7174389e76c9e132a67860c5f65f9706...\n                                  ...                        \n31788165    fe99a0069d6b3c64c2707d0ce53b9311540917471d82df...\n31788202    fecc5f77b5f7ee4570efde9ab05ec94d0de2bf80efb4f6...\n31788208    fece2f68864c311a0b5208e2eb735b3dcde7e41461d327...\n31788217    fee56cc5315dafb35a4490ccc6f711092cae913550c832...\n31788275    ff5b8a8b26bf93a66290e9bd1b73393ac6a58968a78519...\nName: customerId, Length: 1362281, dtype: object"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_customer_ids = init_transaction_df[init_transaction_df.articleId.isin(filtered_article_ids)].customerId.drop_duplicates()\n",
    "filtered_customer_ids"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Sample Customers"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "0      fdbe75e71e134938025dbbb9bc495bd302d578b449ac96...\n1      fb9310441b653525f1adad3fbe7ece522ba50e752cca62...\n2      f0a8599239eea199f1440af86ab9df78cb5d4e85f532fd...\n3      4dd8a1b3175c88f07b123b388a5c9b5dfe16b3ba6fdf62...\n4      696093ad8815f16ab92c07eb32d69c2d1e90daef479de7...\n                             ...                        \n995    8e43348085930e2063c673ce5f85579fcdb26293aa4efe...\n996    8b555d0d3c601c3e2c734f8ef99810248d7532cf0cbee0...\n997    83e43a73575426a3171610f218aeb638da207e6b73f869...\n998    181c9e80cdddcfa2eb0436524bb6de520ee44dee99b527...\n999    9f1dc11db29b3f458baeffb25e0644612b6aafa26c0728...\nName: customerId, Length: 1000, dtype: object"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customer_ids = filtered_customer_ids\n",
    "if SAMPLE_NUM_CUSTOMERS > 0:\n",
    "    customer_ids = filtered_customer_ids.sample(n=SAMPLE_NUM_CUSTOMERS, random_state=RANDOM_SEED).reset_index(drop=True)\n",
    "customer_ids"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "2445        653428002\n6182        636587001\n6183        640462002\n6188        645422002\n6190        645424001\n              ...    \n31779121    906639004\n31779122    684238003\n31779124    921266007\n31779125    812530004\n31779126    942187001\nName: articleId, Length: 13351, dtype: int64"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "article_ids = init_transaction_df[init_transaction_df.customerId.isin(customer_ids)].articleId.drop_duplicates()\n",
    "article_ids"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Sample Down Data\n",
    "Now that we have the list of customers and articles to include , `customer_ids`, and `article_ids` respectively, we can use them to filter the source data and stage for loading"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "                tDat                                         customerId  \\\n2445      2018-09-20  0ddcd6055c5830c1fda493843d051edb04ce1bf888aa4b...   \n6182      2018-09-20  210f113fe87db5d6391e986dc06b8e4369e46284e3b989...   \n6183      2018-09-20  210f113fe87db5d6391e986dc06b8e4369e46284e3b989...   \n6188      2018-09-20  211a2ef477fcfc8fc40a63ffa70bb41086dd06ca85d4af...   \n6189      2018-09-20  211a2ef477fcfc8fc40a63ffa70bb41086dd06ca85d4af...   \n...              ...                                                ...   \n31779124  2020-09-22  b6be55f233772b5fc4a1ebedf36542fb3e1b6c15c23c7e...   \n31779125  2020-09-22  b6be55f233772b5fc4a1ebedf36542fb3e1b6c15c23c7e...   \n31779126  2020-09-22  b6be55f233772b5fc4a1ebedf36542fb3e1b6c15c23c7e...   \n31779127  2020-09-22  b6be55f233772b5fc4a1ebedf36542fb3e1b6c15c23c7e...   \n31779858  2020-09-22  bc5e2e45ecc9b00eb0b01a3373edbb654aad248d75f12b...   \n\n          articleId     price  salesChannelId      txId  \n2445      653428002  0.135576               1      2445  \n6182      636587001  0.008458               1      6182  \n6183      640462002  0.032186               1      6183  \n6188      645422002  0.014390               2      6188  \n6189      645422002  0.014390               2      6189  \n...             ...       ...             ...       ...  \n31779124  921266007  0.016932               2  31779124  \n31779125  812530004  0.010153               2  31779125  \n31779126  942187001  0.016932               2  31779126  \n31779127  866731001  0.025407               2  31779127  \n31779858  678942001  0.016932               2  31779858  \n\n[23199 rows x 6 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>tDat</th>\n      <th>customerId</th>\n      <th>articleId</th>\n      <th>price</th>\n      <th>salesChannelId</th>\n      <th>txId</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2445</th>\n      <td>2018-09-20</td>\n      <td>0ddcd6055c5830c1fda493843d051edb04ce1bf888aa4b...</td>\n      <td>653428002</td>\n      <td>0.135576</td>\n      <td>1</td>\n      <td>2445</td>\n    </tr>\n    <tr>\n      <th>6182</th>\n      <td>2018-09-20</td>\n      <td>210f113fe87db5d6391e986dc06b8e4369e46284e3b989...</td>\n      <td>636587001</td>\n      <td>0.008458</td>\n      <td>1</td>\n      <td>6182</td>\n    </tr>\n    <tr>\n      <th>6183</th>\n      <td>2018-09-20</td>\n      <td>210f113fe87db5d6391e986dc06b8e4369e46284e3b989...</td>\n      <td>640462002</td>\n      <td>0.032186</td>\n      <td>1</td>\n      <td>6183</td>\n    </tr>\n    <tr>\n      <th>6188</th>\n      <td>2018-09-20</td>\n      <td>211a2ef477fcfc8fc40a63ffa70bb41086dd06ca85d4af...</td>\n      <td>645422002</td>\n      <td>0.014390</td>\n      <td>2</td>\n      <td>6188</td>\n    </tr>\n    <tr>\n      <th>6189</th>\n      <td>2018-09-20</td>\n      <td>211a2ef477fcfc8fc40a63ffa70bb41086dd06ca85d4af...</td>\n      <td>645422002</td>\n      <td>0.014390</td>\n      <td>2</td>\n      <td>6189</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>31779124</th>\n      <td>2020-09-22</td>\n      <td>b6be55f233772b5fc4a1ebedf36542fb3e1b6c15c23c7e...</td>\n      <td>921266007</td>\n      <td>0.016932</td>\n      <td>2</td>\n      <td>31779124</td>\n    </tr>\n    <tr>\n      <th>31779125</th>\n      <td>2020-09-22</td>\n      <td>b6be55f233772b5fc4a1ebedf36542fb3e1b6c15c23c7e...</td>\n      <td>812530004</td>\n      <td>0.010153</td>\n      <td>2</td>\n      <td>31779125</td>\n    </tr>\n    <tr>\n      <th>31779126</th>\n      <td>2020-09-22</td>\n      <td>b6be55f233772b5fc4a1ebedf36542fb3e1b6c15c23c7e...</td>\n      <td>942187001</td>\n      <td>0.016932</td>\n      <td>2</td>\n      <td>31779126</td>\n    </tr>\n    <tr>\n      <th>31779127</th>\n      <td>2020-09-22</td>\n      <td>b6be55f233772b5fc4a1ebedf36542fb3e1b6c15c23c7e...</td>\n      <td>866731001</td>\n      <td>0.025407</td>\n      <td>2</td>\n      <td>31779127</td>\n    </tr>\n    <tr>\n      <th>31779858</th>\n      <td>2020-09-22</td>\n      <td>bc5e2e45ecc9b00eb0b01a3373edbb654aad248d75f12b...</td>\n      <td>678942001</td>\n      <td>0.016932</td>\n      <td>2</td>\n      <td>31779858</td>\n    </tr>\n  </tbody>\n</table>\n<p>23199 rows × 6 columns</p>\n</div>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transaction_df = init_transaction_df[init_transaction_df.customerId.isin(customer_ids)]\n",
    "transaction_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "        articleId  productCode                   prodName  productTypeNo  \\\n0       108775015       108775                  Strap top            253   \n1       108775044       108775                  Strap top            253   \n3       110065001       110065          OP T-shirt (Idro)            306   \n6       111565001       111565        20 den 1p Stockings            304   \n8       111586001       111586  Shape Up 30 den 1p Tights            273   \n...           ...          ...                        ...            ...   \n105314  936862001       936862            EDC Marla dress            265   \n105319  936979001       936979     Class Filippa Necklace             77   \n105327  937138001       937138   Flirty Albin bracelet pk             68   \n105444  942187001       942187               ED Sasha tee            255   \n105536  952938001       952938                  Elton top            254   \n\n         productTypeName    productGroupName  graphicalAppearanceNo  \\\n0               Vest top  Garment Upper body                1010016   \n1               Vest top  Garment Upper body                1010016   \n3                    Bra           Underwear                1010016   \n6       Underwear Tights      Socks & Tights                1010016   \n8        Leggings/Tights  Garment Lower body                1010016   \n...                  ...                 ...                    ...   \n105314             Dress   Garment Full body                1010001   \n105319          Necklace         Accessories                1010016   \n105327          Bracelet         Accessories                1010016   \n105444           T-shirt  Garment Upper body                1010016   \n105536               Top  Garment Upper body                1010001   \n\n       graphicalAppearanceName  colourGroupCode colourGroupName  ...  \\\n0                        Solid                9           Black  ...   \n1                        Solid               10           White  ...   \n3                        Solid                9           Black  ...   \n6                        Solid                9           Black  ...   \n8                        Solid                9           Black  ...   \n...                        ...              ...             ...  ...   \n105314        All over pattern               52            Pink  ...   \n105319                   Solid                5            Gold  ...   \n105327                   Solid                5            Gold  ...   \n105444                   Solid                9           Black  ...   \n105536        All over pattern               13           Beige  ...   \n\n            departmentName indexCode           indexName indexGroupNo  \\\n0             Jersey Basic         A          Ladieswear            1   \n1             Jersey Basic         A          Ladieswear            1   \n3           Clean Lingerie         B    Lingeries/Tights            1   \n6             Tights basic         B    Lingeries/Tights            1   \n8             Tights basic         B    Lingeries/Tights            1   \n...                    ...       ...                 ...          ...   \n105314           Campaigns         A          Ladieswear            1   \n105319           Jewellery         C  Ladies Accessories            1   \n105327  Jewellery Extended         C  Ladies Accessories            1   \n105444              Jersey         A          Ladieswear            1   \n105536              Jersey         A          Ladieswear            1   \n\n        indexGroupName sectionNo                     sectionName  \\\n0           Ladieswear        16          Womens Everyday Basics   \n1           Ladieswear        16          Womens Everyday Basics   \n3           Ladieswear        61                 Womens Lingerie   \n6           Ladieswear        62  Womens Nightwear, Socks & Tigh   \n8           Ladieswear        62  Womens Nightwear, Socks & Tigh   \n...                ...       ...                             ...   \n105314      Ladieswear        15      Womens Everyday Collection   \n105319      Ladieswear        66        Womens Small accessories   \n105327      Ladieswear        66        Womens Small accessories   \n105444      Ladieswear         2                            H&M+   \n105536      Ladieswear        18                    Womens Trend   \n\n       garmentGroupNo   garmentGroupName  \\\n0                1002       Jersey Basic   \n1                1002       Jersey Basic   \n3                1017  Under-, Nightwear   \n6                1021   Socks and Tights   \n8                1021   Socks and Tights   \n...               ...                ...   \n105314           1023     Special Offers   \n105319           1019        Accessories   \n105327           1019        Accessories   \n105444           1005       Jersey Fancy   \n105536           1005       Jersey Fancy   \n\n                                               detailDesc  \n0                 Jersey top with narrow shoulder straps.  \n1                 Jersey top with narrow shoulder straps.  \n3       Microfibre T-shirt bra with underwired, moulde...  \n6       Semi shiny nylon stockings with a wide, reinfo...  \n8       Tights with built-in support to lift the botto...  \n...                                                   ...  \n105314  Calf-length dress in a patterned Tencel™ lyoce...  \n105319  Metal chain necklace with a pendant. Adjustabl...  \n105327  Metal chain bracelets. Two plain and two with ...  \n105444  Oversized, straight-cut T-shirt in a soft moda...  \n105536  Fitted top in jersey with a round neckline and...  \n\n[13351 rows x 25 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>articleId</th>\n      <th>productCode</th>\n      <th>prodName</th>\n      <th>productTypeNo</th>\n      <th>productTypeName</th>\n      <th>productGroupName</th>\n      <th>graphicalAppearanceNo</th>\n      <th>graphicalAppearanceName</th>\n      <th>colourGroupCode</th>\n      <th>colourGroupName</th>\n      <th>...</th>\n      <th>departmentName</th>\n      <th>indexCode</th>\n      <th>indexName</th>\n      <th>indexGroupNo</th>\n      <th>indexGroupName</th>\n      <th>sectionNo</th>\n      <th>sectionName</th>\n      <th>garmentGroupNo</th>\n      <th>garmentGroupName</th>\n      <th>detailDesc</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>108775015</td>\n      <td>108775</td>\n      <td>Strap top</td>\n      <td>253</td>\n      <td>Vest top</td>\n      <td>Garment Upper body</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n      <td>...</td>\n      <td>Jersey Basic</td>\n      <td>A</td>\n      <td>Ladieswear</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>16</td>\n      <td>Womens Everyday Basics</td>\n      <td>1002</td>\n      <td>Jersey Basic</td>\n      <td>Jersey top with narrow shoulder straps.</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>108775044</td>\n      <td>108775</td>\n      <td>Strap top</td>\n      <td>253</td>\n      <td>Vest top</td>\n      <td>Garment Upper body</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>10</td>\n      <td>White</td>\n      <td>...</td>\n      <td>Jersey Basic</td>\n      <td>A</td>\n      <td>Ladieswear</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>16</td>\n      <td>Womens Everyday Basics</td>\n      <td>1002</td>\n      <td>Jersey Basic</td>\n      <td>Jersey top with narrow shoulder straps.</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>110065001</td>\n      <td>110065</td>\n      <td>OP T-shirt (Idro)</td>\n      <td>306</td>\n      <td>Bra</td>\n      <td>Underwear</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n      <td>...</td>\n      <td>Clean Lingerie</td>\n      <td>B</td>\n      <td>Lingeries/Tights</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>61</td>\n      <td>Womens Lingerie</td>\n      <td>1017</td>\n      <td>Under-, Nightwear</td>\n      <td>Microfibre T-shirt bra with underwired, moulde...</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>111565001</td>\n      <td>111565</td>\n      <td>20 den 1p Stockings</td>\n      <td>304</td>\n      <td>Underwear Tights</td>\n      <td>Socks &amp; Tights</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n      <td>...</td>\n      <td>Tights basic</td>\n      <td>B</td>\n      <td>Lingeries/Tights</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>62</td>\n      <td>Womens Nightwear, Socks &amp; Tigh</td>\n      <td>1021</td>\n      <td>Socks and Tights</td>\n      <td>Semi shiny nylon stockings with a wide, reinfo...</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>111586001</td>\n      <td>111586</td>\n      <td>Shape Up 30 den 1p Tights</td>\n      <td>273</td>\n      <td>Leggings/Tights</td>\n      <td>Garment Lower body</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n      <td>...</td>\n      <td>Tights basic</td>\n      <td>B</td>\n      <td>Lingeries/Tights</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>62</td>\n      <td>Womens Nightwear, Socks &amp; Tigh</td>\n      <td>1021</td>\n      <td>Socks and Tights</td>\n      <td>Tights with built-in support to lift the botto...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>105314</th>\n      <td>936862001</td>\n      <td>936862</td>\n      <td>EDC Marla dress</td>\n      <td>265</td>\n      <td>Dress</td>\n      <td>Garment Full body</td>\n      <td>1010001</td>\n      <td>All over pattern</td>\n      <td>52</td>\n      <td>Pink</td>\n      <td>...</td>\n      <td>Campaigns</td>\n      <td>A</td>\n      <td>Ladieswear</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>15</td>\n      <td>Womens Everyday Collection</td>\n      <td>1023</td>\n      <td>Special Offers</td>\n      <td>Calf-length dress in a patterned Tencel™ lyoce...</td>\n    </tr>\n    <tr>\n      <th>105319</th>\n      <td>936979001</td>\n      <td>936979</td>\n      <td>Class Filippa Necklace</td>\n      <td>77</td>\n      <td>Necklace</td>\n      <td>Accessories</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>5</td>\n      <td>Gold</td>\n      <td>...</td>\n      <td>Jewellery</td>\n      <td>C</td>\n      <td>Ladies Accessories</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>66</td>\n      <td>Womens Small accessories</td>\n      <td>1019</td>\n      <td>Accessories</td>\n      <td>Metal chain necklace with a pendant. Adjustabl...</td>\n    </tr>\n    <tr>\n      <th>105327</th>\n      <td>937138001</td>\n      <td>937138</td>\n      <td>Flirty Albin bracelet pk</td>\n      <td>68</td>\n      <td>Bracelet</td>\n      <td>Accessories</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>5</td>\n      <td>Gold</td>\n      <td>...</td>\n      <td>Jewellery Extended</td>\n      <td>C</td>\n      <td>Ladies Accessories</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>66</td>\n      <td>Womens Small accessories</td>\n      <td>1019</td>\n      <td>Accessories</td>\n      <td>Metal chain bracelets. Two plain and two with ...</td>\n    </tr>\n    <tr>\n      <th>105444</th>\n      <td>942187001</td>\n      <td>942187</td>\n      <td>ED Sasha tee</td>\n      <td>255</td>\n      <td>T-shirt</td>\n      <td>Garment Upper body</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n      <td>...</td>\n      <td>Jersey</td>\n      <td>A</td>\n      <td>Ladieswear</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>2</td>\n      <td>H&amp;M+</td>\n      <td>1005</td>\n      <td>Jersey Fancy</td>\n      <td>Oversized, straight-cut T-shirt in a soft moda...</td>\n    </tr>\n    <tr>\n      <th>105536</th>\n      <td>952938001</td>\n      <td>952938</td>\n      <td>Elton top</td>\n      <td>254</td>\n      <td>Top</td>\n      <td>Garment Upper body</td>\n      <td>1010001</td>\n      <td>All over pattern</td>\n      <td>13</td>\n      <td>Beige</td>\n      <td>...</td>\n      <td>Jersey</td>\n      <td>A</td>\n      <td>Ladieswear</td>\n      <td>1</td>\n      <td>Ladieswear</td>\n      <td>18</td>\n      <td>Womens Trend</td>\n      <td>1005</td>\n      <td>Jersey Fancy</td>\n      <td>Fitted top in jersey with a round neckline and...</td>\n    </tr>\n  </tbody>\n</table>\n<p>13351 rows × 25 columns</p>\n</div>"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_article_df = init_article_df[init_article_df.articleId.isin(article_ids)]\n",
    "full_article_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Create Product, Department, and Article Dataframes"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "        productCode                   prodName  productTypeNo  \\\n0            108775                  Strap top            253   \n3            110065          OP T-shirt (Idro)            306   \n6            111565        20 den 1p Stockings            304   \n8            111586  Shape Up 30 den 1p Tights            273   \n9            111593   Support 40 den 1p Tights            304   \n...             ...                        ...            ...   \n105314       936862            EDC Marla dress            265   \n105319       936979     Class Filippa Necklace             77   \n105327       937138   Flirty Albin bracelet pk             68   \n105444       942187               ED Sasha tee            255   \n105536       952938                  Elton top            254   \n\n         productTypeName    productGroupName  garmentGroupNo  \\\n0               Vest top  Garment Upper body            1002   \n3                    Bra           Underwear            1017   \n6       Underwear Tights      Socks & Tights            1021   \n8        Leggings/Tights  Garment Lower body            1021   \n9       Underwear Tights      Socks & Tights            1021   \n...                  ...                 ...             ...   \n105314             Dress   Garment Full body            1023   \n105319          Necklace         Accessories            1019   \n105327          Bracelet         Accessories            1019   \n105444           T-shirt  Garment Upper body            1005   \n105536               Top  Garment Upper body            1005   \n\n         garmentGroupName                                         detailDesc  \n0            Jersey Basic            Jersey top with narrow shoulder straps.  \n3       Under-, Nightwear  Microfibre T-shirt bra with underwired, moulde...  \n6        Socks and Tights  Semi shiny nylon stockings with a wide, reinfo...  \n8        Socks and Tights  Tights with built-in support to lift the botto...  \n9        Socks and Tights  Semi shiny tights that shape the tummy, thighs...  \n...                   ...                                                ...  \n105314     Special Offers  Calf-length dress in a patterned Tencel™ lyoce...  \n105319        Accessories  Metal chain necklace with a pendant. Adjustabl...  \n105327        Accessories  Metal chain bracelets. Two plain and two with ...  \n105444       Jersey Fancy  Oversized, straight-cut T-shirt in a soft moda...  \n105536       Jersey Fancy  Fitted top in jersey with a round neckline and...  \n\n[8044 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>productCode</th>\n      <th>prodName</th>\n      <th>productTypeNo</th>\n      <th>productTypeName</th>\n      <th>productGroupName</th>\n      <th>garmentGroupNo</th>\n      <th>garmentGroupName</th>\n      <th>detailDesc</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>108775</td>\n      <td>Strap top</td>\n      <td>253</td>\n      <td>Vest top</td>\n      <td>Garment Upper body</td>\n      <td>1002</td>\n      <td>Jersey Basic</td>\n      <td>Jersey top with narrow shoulder straps.</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>110065</td>\n      <td>OP T-shirt (Idro)</td>\n      <td>306</td>\n      <td>Bra</td>\n      <td>Underwear</td>\n      <td>1017</td>\n      <td>Under-, Nightwear</td>\n      <td>Microfibre T-shirt bra with underwired, moulde...</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>111565</td>\n      <td>20 den 1p Stockings</td>\n      <td>304</td>\n      <td>Underwear Tights</td>\n      <td>Socks &amp; Tights</td>\n      <td>1021</td>\n      <td>Socks and Tights</td>\n      <td>Semi shiny nylon stockings with a wide, reinfo...</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>111586</td>\n      <td>Shape Up 30 den 1p Tights</td>\n      <td>273</td>\n      <td>Leggings/Tights</td>\n      <td>Garment Lower body</td>\n      <td>1021</td>\n      <td>Socks and Tights</td>\n      <td>Tights with built-in support to lift the botto...</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>111593</td>\n      <td>Support 40 den 1p Tights</td>\n      <td>304</td>\n      <td>Underwear Tights</td>\n      <td>Socks &amp; Tights</td>\n      <td>1021</td>\n      <td>Socks and Tights</td>\n      <td>Semi shiny tights that shape the tummy, thighs...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>105314</th>\n      <td>936862</td>\n      <td>EDC Marla dress</td>\n      <td>265</td>\n      <td>Dress</td>\n      <td>Garment Full body</td>\n      <td>1023</td>\n      <td>Special Offers</td>\n      <td>Calf-length dress in a patterned Tencel™ lyoce...</td>\n    </tr>\n    <tr>\n      <th>105319</th>\n      <td>936979</td>\n      <td>Class Filippa Necklace</td>\n      <td>77</td>\n      <td>Necklace</td>\n      <td>Accessories</td>\n      <td>1019</td>\n      <td>Accessories</td>\n      <td>Metal chain necklace with a pendant. Adjustabl...</td>\n    </tr>\n    <tr>\n      <th>105327</th>\n      <td>937138</td>\n      <td>Flirty Albin bracelet pk</td>\n      <td>68</td>\n      <td>Bracelet</td>\n      <td>Accessories</td>\n      <td>1019</td>\n      <td>Accessories</td>\n      <td>Metal chain bracelets. Two plain and two with ...</td>\n    </tr>\n    <tr>\n      <th>105444</th>\n      <td>942187</td>\n      <td>ED Sasha tee</td>\n      <td>255</td>\n      <td>T-shirt</td>\n      <td>Garment Upper body</td>\n      <td>1005</td>\n      <td>Jersey Fancy</td>\n      <td>Oversized, straight-cut T-shirt in a soft moda...</td>\n    </tr>\n    <tr>\n      <th>105536</th>\n      <td>952938</td>\n      <td>Elton top</td>\n      <td>254</td>\n      <td>Top</td>\n      <td>Garment Upper body</td>\n      <td>1005</td>\n      <td>Jersey Fancy</td>\n      <td>Fitted top in jersey with a round neckline and...</td>\n    </tr>\n  </tbody>\n</table>\n<p>8044 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "product_df = full_article_df[['productCode', 'prodName',\n",
    "                              'productTypeNo', 'productTypeName',\n",
    "                              'productGroupName', 'garmentGroupNo', 'garmentGroupName',\n",
    "                              'detailDesc']].drop_duplicates(subset='productCode')\n",
    "\n",
    "product_df.to_csv('product.csv', index=False)\n",
    "product_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "        departmentNo              departmentName  sectionNo  \\\n0               1676                Jersey Basic         16   \n3               1339              Clean Lingerie         61   \n6               3608                Tights basic         62   \n17              5883                Jersey Basic         26   \n23              2032                      Jersey          8   \n...              ...                         ...        ...   \n92964           7510                       Woven         28   \n97621           3420  Small Accessories Extended         66   \n98705           5231                      Jacket         31   \n101971          8090       Promotion/Other/Offer         29   \n103941          5888                      Blanks         26   \n\n                           sectionName  \n0               Womens Everyday Basics  \n3                      Womens Lingerie  \n6       Womens Nightwear, Socks & Tigh  \n17                       Men Underwear  \n23                                Mama  \n...                                ...  \n92964                      Men Edition  \n97621         Womens Small accessories  \n98705                   Mens Outerwear  \n101971                       Men Other  \n103941                   Men Underwear  \n\n[266 rows x 4 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>departmentNo</th>\n      <th>departmentName</th>\n      <th>sectionNo</th>\n      <th>sectionName</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1676</td>\n      <td>Jersey Basic</td>\n      <td>16</td>\n      <td>Womens Everyday Basics</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1339</td>\n      <td>Clean Lingerie</td>\n      <td>61</td>\n      <td>Womens Lingerie</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>3608</td>\n      <td>Tights basic</td>\n      <td>62</td>\n      <td>Womens Nightwear, Socks &amp; Tigh</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>5883</td>\n      <td>Jersey Basic</td>\n      <td>26</td>\n      <td>Men Underwear</td>\n    </tr>\n    <tr>\n      <th>23</th>\n      <td>2032</td>\n      <td>Jersey</td>\n      <td>8</td>\n      <td>Mama</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>92964</th>\n      <td>7510</td>\n      <td>Woven</td>\n      <td>28</td>\n      <td>Men Edition</td>\n    </tr>\n    <tr>\n      <th>97621</th>\n      <td>3420</td>\n      <td>Small Accessories Extended</td>\n      <td>66</td>\n      <td>Womens Small accessories</td>\n    </tr>\n    <tr>\n      <th>98705</th>\n      <td>5231</td>\n      <td>Jacket</td>\n      <td>31</td>\n      <td>Mens Outerwear</td>\n    </tr>\n    <tr>\n      <th>101971</th>\n      <td>8090</td>\n      <td>Promotion/Other/Offer</td>\n      <td>29</td>\n      <td>Men Other</td>\n    </tr>\n    <tr>\n      <th>103941</th>\n      <td>5888</td>\n      <td>Blanks</td>\n      <td>26</td>\n      <td>Men Underwear</td>\n    </tr>\n  </tbody>\n</table>\n<p>266 rows × 4 columns</p>\n</div>"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "department_df = full_article_df[['departmentNo', 'departmentName', 'sectionNo', 'sectionName']]\\\n",
    "    .drop_duplicates(subset='departmentNo')\n",
    "\n",
    "department_df.to_csv('department.csv', index=False)\n",
    "department_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "        articleId  productCode  departmentNo                   prodName  \\\n0       108775015       108775          1676                  Strap top   \n1       108775044       108775          1676                  Strap top   \n3       110065001       110065          1339          OP T-shirt (Idro)   \n6       111565001       111565          3608        20 den 1p Stockings   \n8       111586001       111586          3608  Shape Up 30 den 1p Tights   \n...           ...          ...           ...                        ...   \n105314  936862001       936862          3090            EDC Marla dress   \n105319  936979001       936979          4344     Class Filippa Necklace   \n105327  937138001       937138          4345   Flirty Albin bracelet pk   \n105444  942187001       942187          1919               ED Sasha tee   \n105536  952938001       952938          1641                  Elton top   \n\n         productTypeName  graphicalAppearanceNo graphicalAppearanceName  \\\n0               Vest top                1010016                   Solid   \n1               Vest top                1010016                   Solid   \n3                    Bra                1010016                   Solid   \n6       Underwear Tights                1010016                   Solid   \n8        Leggings/Tights                1010016                   Solid   \n...                  ...                    ...                     ...   \n105314             Dress                1010001        All over pattern   \n105319          Necklace                1010016                   Solid   \n105327          Bracelet                1010016                   Solid   \n105444           T-shirt                1010016                   Solid   \n105536               Top                1010001        All over pattern   \n\n        colourGroupCode colourGroupName  \n0                     9           Black  \n1                    10           White  \n3                     9           Black  \n6                     9           Black  \n8                     9           Black  \n...                 ...             ...  \n105314               52            Pink  \n105319                5            Gold  \n105327                5            Gold  \n105444                9           Black  \n105536               13           Beige  \n\n[13351 rows x 9 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>articleId</th>\n      <th>productCode</th>\n      <th>departmentNo</th>\n      <th>prodName</th>\n      <th>productTypeName</th>\n      <th>graphicalAppearanceNo</th>\n      <th>graphicalAppearanceName</th>\n      <th>colourGroupCode</th>\n      <th>colourGroupName</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>108775015</td>\n      <td>108775</td>\n      <td>1676</td>\n      <td>Strap top</td>\n      <td>Vest top</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>108775044</td>\n      <td>108775</td>\n      <td>1676</td>\n      <td>Strap top</td>\n      <td>Vest top</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>10</td>\n      <td>White</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>110065001</td>\n      <td>110065</td>\n      <td>1339</td>\n      <td>OP T-shirt (Idro)</td>\n      <td>Bra</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>111565001</td>\n      <td>111565</td>\n      <td>3608</td>\n      <td>20 den 1p Stockings</td>\n      <td>Underwear Tights</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>111586001</td>\n      <td>111586</td>\n      <td>3608</td>\n      <td>Shape Up 30 den 1p Tights</td>\n      <td>Leggings/Tights</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>105314</th>\n      <td>936862001</td>\n      <td>936862</td>\n      <td>3090</td>\n      <td>EDC Marla dress</td>\n      <td>Dress</td>\n      <td>1010001</td>\n      <td>All over pattern</td>\n      <td>52</td>\n      <td>Pink</td>\n    </tr>\n    <tr>\n      <th>105319</th>\n      <td>936979001</td>\n      <td>936979</td>\n      <td>4344</td>\n      <td>Class Filippa Necklace</td>\n      <td>Necklace</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>5</td>\n      <td>Gold</td>\n    </tr>\n    <tr>\n      <th>105327</th>\n      <td>937138001</td>\n      <td>937138</td>\n      <td>4345</td>\n      <td>Flirty Albin bracelet pk</td>\n      <td>Bracelet</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>5</td>\n      <td>Gold</td>\n    </tr>\n    <tr>\n      <th>105444</th>\n      <td>942187001</td>\n      <td>942187</td>\n      <td>1919</td>\n      <td>ED Sasha tee</td>\n      <td>T-shirt</td>\n      <td>1010016</td>\n      <td>Solid</td>\n      <td>9</td>\n      <td>Black</td>\n    </tr>\n    <tr>\n      <th>105536</th>\n      <td>952938001</td>\n      <td>952938</td>\n      <td>1641</td>\n      <td>Elton top</td>\n      <td>Top</td>\n      <td>1010001</td>\n      <td>All over pattern</td>\n      <td>13</td>\n      <td>Beige</td>\n    </tr>\n  </tbody>\n</table>\n<p>13351 rows × 9 columns</p>\n</div>"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "article_df = full_article_df[['articleId', 'productCode', 'departmentNo', 'prodName', 'productTypeName',\n",
    "                              'graphicalAppearanceNo', 'graphicalAppearanceName', 'colourGroupCode', 'colourGroupName']]\n",
    "article_df.to_csv('article.csv', index=False)\n",
    "article_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "                                                customerId   fn  active  \\\n810      00264b7d4cd6498292e8a355b699c2d07725d123f04867...  1.0     1.0   \n1947     005c6d3bb66c86aab606814cd9995a12f99b3a44b58c72...  NaN     NaN   \n3548     00abec3de294e03d192db15b91e154853ee1c89415e7cd...  NaN     NaN   \n5027     00f311a42124fc44d117135f34e1fca29fcac271e6fbd0...  1.0     1.0   \n6432     0132cd2eb3c6b1f66784f65f94ddd8352add2653e0caf5...  NaN     NaN   \n...                                                    ...  ...     ...   \n1361059  fdf1294f414faac2b00a725f5d80c34f98a744d9b8b3ce...  NaN     NaN   \n1363646  fe6faeed37fe86e885928d3ab30d8d9b072d6643c8aa15...  1.0     1.0   \n1366543  fef793ec3a7d62d782824517355d74ded50964dce33009...  NaN     NaN   \n1370498  ffb925b11e1bb2e375d22a02d67907994eb8cb92ec2e7d...  NaN     NaN   \n1371509  ffe8f77be0ce1a5bde91c97cb76aa887fb200bba79fe5a...  1.0     1.0   \n\n        clubMemberStatus fashionNewsFrequency   age  \\\n810               ACTIVE            Regularly  53.0   \n1947          PRE-CREATE                 NONE   NaN   \n3548              ACTIVE                 NONE  49.0   \n5027              ACTIVE            Regularly  55.0   \n6432              ACTIVE                 NONE  49.0   \n...                  ...                  ...   ...   \n1361059           ACTIVE                 NONE  32.0   \n1363646           ACTIVE            Regularly  46.0   \n1366543           ACTIVE                 NONE  46.0   \n1370498           ACTIVE                 NONE  34.0   \n1371509           ACTIVE            Regularly  26.0   \n\n                                                postalCode  \n810      2c29ae653a9282cce4151bd87643c907644e09541abc28...  \n1947     177b4a2258a85a2247daaa7cdffba96a74c741ea8a6605...  \n3548     86557a458110ac98f4ca80e5a815ba2e8ea086dd8039b0...  \n5027     1a80c5651ae36327a86e71d5b967cf62c31126d1b57ae0...  \n6432     49f7ec29bcacbbf2120af5162f9f99c212e9dd26b48d79...  \n...                                                    ...  \n1361059  0cd87888c3a13ebbb1e90cac6b9fbf34c51afa40865f55...  \n1363646  fe234b03107b233aec5695dc4c3fbe8e638338643f4e14...  \n1366543  5799a39cffe701ebdb12181348bf10f9e23abcc3868c43...  \n1370498  ebdd8c5c893683c3cf52c011d4e35024e46d183c95f0fa...  \n1371509  bf90cabe02380b161574072e5ca4f77f62b313421e3f62...  \n\n[1000 rows x 7 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>customerId</th>\n      <th>fn</th>\n      <th>active</th>\n      <th>clubMemberStatus</th>\n      <th>fashionNewsFrequency</th>\n      <th>age</th>\n      <th>postalCode</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>810</th>\n      <td>00264b7d4cd6498292e8a355b699c2d07725d123f04867...</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>ACTIVE</td>\n      <td>Regularly</td>\n      <td>53.0</td>\n      <td>2c29ae653a9282cce4151bd87643c907644e09541abc28...</td>\n    </tr>\n    <tr>\n      <th>1947</th>\n      <td>005c6d3bb66c86aab606814cd9995a12f99b3a44b58c72...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>PRE-CREATE</td>\n      <td>NONE</td>\n      <td>NaN</td>\n      <td>177b4a2258a85a2247daaa7cdffba96a74c741ea8a6605...</td>\n    </tr>\n    <tr>\n      <th>3548</th>\n      <td>00abec3de294e03d192db15b91e154853ee1c89415e7cd...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ACTIVE</td>\n      <td>NONE</td>\n      <td>49.0</td>\n      <td>86557a458110ac98f4ca80e5a815ba2e8ea086dd8039b0...</td>\n    </tr>\n    <tr>\n      <th>5027</th>\n      <td>00f311a42124fc44d117135f34e1fca29fcac271e6fbd0...</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>ACTIVE</td>\n      <td>Regularly</td>\n      <td>55.0</td>\n      <td>1a80c5651ae36327a86e71d5b967cf62c31126d1b57ae0...</td>\n    </tr>\n    <tr>\n      <th>6432</th>\n      <td>0132cd2eb3c6b1f66784f65f94ddd8352add2653e0caf5...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ACTIVE</td>\n      <td>NONE</td>\n      <td>49.0</td>\n      <td>49f7ec29bcacbbf2120af5162f9f99c212e9dd26b48d79...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1361059</th>\n      <td>fdf1294f414faac2b00a725f5d80c34f98a744d9b8b3ce...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ACTIVE</td>\n      <td>NONE</td>\n      <td>32.0</td>\n      <td>0cd87888c3a13ebbb1e90cac6b9fbf34c51afa40865f55...</td>\n    </tr>\n    <tr>\n      <th>1363646</th>\n      <td>fe6faeed37fe86e885928d3ab30d8d9b072d6643c8aa15...</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>ACTIVE</td>\n      <td>Regularly</td>\n      <td>46.0</td>\n      <td>fe234b03107b233aec5695dc4c3fbe8e638338643f4e14...</td>\n    </tr>\n    <tr>\n      <th>1366543</th>\n      <td>fef793ec3a7d62d782824517355d74ded50964dce33009...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ACTIVE</td>\n      <td>NONE</td>\n      <td>46.0</td>\n      <td>5799a39cffe701ebdb12181348bf10f9e23abcc3868c43...</td>\n    </tr>\n    <tr>\n      <th>1370498</th>\n      <td>ffb925b11e1bb2e375d22a02d67907994eb8cb92ec2e7d...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ACTIVE</td>\n      <td>NONE</td>\n      <td>34.0</td>\n      <td>ebdd8c5c893683c3cf52c011d4e35024e46d183c95f0fa...</td>\n    </tr>\n    <tr>\n      <th>1371509</th>\n      <td>ffe8f77be0ce1a5bde91c97cb76aa887fb200bba79fe5a...</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>ACTIVE</td>\n      <td>Regularly</td>\n      <td>26.0</td>\n      <td>bf90cabe02380b161574072e5ca4f77f62b313421e3f62...</td>\n    </tr>\n  </tbody>\n</table>\n<p>1000 rows × 7 columns</p>\n</div>"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customer_df = camel_case_rename_cols(pd.read_csv('data/customers.csv'))\n",
    "customer_df = customer_df[customer_df.customerId.isin(customer_ids)]\n",
    "customer_df.to_csv('customer.csv', index=False)\n",
    "customer_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Write Transactions Dataframe to csv"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "transaction_df.to_csv('transaction.csv', index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Upload to Google Cloud"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://product.csv [Content-Type=text/csv]...\r\n",
      "/ [1 files][  1.6 MiB/  1.6 MiB]                                                \r\n",
      "Operation completed over 1 objects/1.6 MiB.                                      \r\n"
     ]
    }
   ],
   "source": [
    "!gsutil cp product.csv gs://neo4j-workshop-data/genai-hm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://department.csv [Content-Type=text/csv]...\r\n",
      "/ [1 files][ 10.4 KiB/ 10.4 KiB]                                                \r\n",
      "Operation completed over 1 objects/10.4 KiB.                                     \r\n"
     ]
    }
   ],
   "source": [
    "!gsutil cp department.csv gs://neo4j-workshop-data/genai-hm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://article.csv [Content-Type=text/csv]...\r\n",
      "- [1 files][959.7 KiB/959.7 KiB]                                                \r\n",
      "Operation completed over 1 objects/959.7 KiB.                                    \r\n"
     ]
    }
   ],
   "source": [
    "!gsutil cp article.csv gs://neo4j-workshop-data/genai-hm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://customer.csv [Content-Type=text/csv]...\r\n",
      "/ [1 files][149.5 KiB/149.5 KiB]                                                \r\n",
      "Operation completed over 1 objects/149.5 KiB.                                    \r\n"
     ]
    }
   ],
   "source": [
    "!gsutil cp customer.csv gs://neo4j-workshop-data/genai-hm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying file://transaction.csv [Content-Type=text/csv]...\r\n",
      "- [1 files][  2.6 MiB/  2.6 MiB]                                                \r\n",
      "Operation completed over 1 objects/2.6 MiB.                                      \r\n"
     ]
    }
   ],
   "source": [
    "!gsutil cp transaction.csv gs://neo4j-workshop-data/genai-hm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}