exploration/1-data-cleanup.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dealing with unclean data\n",
    "\n",
    "We're going to look at data that may require some cleansing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the admissions data that is not so clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_location = \"../data/college-admissions/admission-data-dirty.csv\"\n",
    "data_location  = 'https://elephantscale-public.s3.amazonaws.com/data/college-admissions/admission-data-dirty.csv'\n",
    "\n",
    "admissions = pd.read_csv(data_location)\n",
    "print(\"admissions shape : \", admissions.shape)\n",
    "admissions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Summary\n",
    "See what we get.  It will skip null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## TODO : use 'describe' functions \n",
    "admissions.???()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## TODO : make describe include all columns\n",
    "admissions.describe(include = 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## TODO : Describe more than one column : gre and gpa\n",
    "## Hint : add 'gpa' column\n",
    "admissions[['gre', '???']].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Drop all null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"raw data shape : \", admissions.shape)\n",
    "\n",
    "## TODO : use 'dropna' function\n",
    "dropped_na = admissions.???()\n",
    "print()\n",
    "print(\"after drop shape : \", dropped_na.shape)\n",
    "dropped_na\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# only drop nulls from admit & gre column\n",
    "print(\"raw data shape : \", admissions.shape)\n",
    "\n",
    "print()\n",
    "\n",
    "dropped2 = admissions.dropna(subset=['admit', 'gre'])\n",
    "print(\"after drop shape : \", dropped2.shape)\n",
    "dropped2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fill in the values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## TODO :  fill every thing with zero\n",
    "## Hint : use 'fillna'\n",
    "zero_fill = admissions.???(0)\n",
    "zero_fill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# or we can specify per column default value\n",
    "## TODO : specify different default values per column\n",
    "##        default value for gre = -100\n",
    "fill2 = admissions.fillna({'admit': -1, 'gre':??? , 'gpa':-1, 'rank':10})\n",
    "fill2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replace values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print (admissions)\n",
    "\n",
    "admissions2 = admissions.copy(deep=True)\n",
    "\n",
    "## TODO : use replace to change 800 to 1000\n",
    "## Hint : replace (800, 1000)\n",
    "admissions2['gre'].replace(???, ???, inplace=True)\n",
    "\n",
    "print()\n",
    "print (admissions2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean out RANK column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## TODO : filter out any thing other than 1,2,3,4  in rank column\n",
    "a = admissions[admissions['rank'].isin(['1','2','3','4'])]\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}