PySpark-ML-Use-Case-Logistic-Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# PySpark Project-Get a handle on using Python with Spark through this hands-on data processing spark python tutorial."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This series of PySpark project will look at installing Apache Spark on the cluster and explore various data analysis tasks using PySpark for various big data and data science applications."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "spark_path = 'E:\\spark-2.0.1-bin-hadoop2.7\\spark-2.0.1-bin-hadoop2.7'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "os.environ['SPARK_HOME'] = spark_path\n",
    "os.environ['HADOOP_HOME'] = spark_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sys.path.append(spark_path + \"/bin\")\n",
    "sys.path.append(spark_path + \"/python\")\n",
    "sys.path.append(spark_path + \"/python/pypark/\")\n",
    "sys.path.append(spark_path + \"/python/lib\")\n",
    "sys.path.append(spark_path + \"/python/lib/pyspark.zip\")\n",
    "sys.path.append(spark_path + \"/python/lib/py4j-0.10.3-src\")\n",
    "               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkConf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = SparkContext(\"local\",'test123')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.context.SparkContext at 0x9adf978>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.stop()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What id RDD (Resilient Distributed Data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = SparkContext(master='local[2]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.context.SparkContext at 0x9b2d940>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = [12,32,45,65,67,89]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[12, 32, 45, 65, 67, 89]\n"
     ]
    }
   ],
   "source": [
    "print data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#parallelization\n",
    "rdd1 = sc.parallelize(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ParallelCollectionRDD[0] at parallelize at PythonRDD.scala:475\n"
     ]
    }
   ],
   "source": [
    "print rdd1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[12, 32, 45, 65, 67, 89]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'2.0.1'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.7'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.pythonVer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'local[2]'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.getNumPartitions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.parallelize([]).isEmpty()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc.parallelize([data]).isEmpty()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[12, 32, 45, 65, 67, 89]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# basic statistics\n",
    "rdd1.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "89"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25.203615260954571"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.stdev()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51.666666666666664"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "635.2222222222222"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.variance()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(count: 6, mean: 51.6666666667, stdev: 25.203615261, max: 89.0, min: 12.0)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pyspark.mllib.stat import Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mat = sc.parallelize(\n",
    "    [np.array([10.1,12.4,14.5,16.8,21]),np.array([21.3,24.2,35.4,36.4,31.7]),np.array([21.1,23.,54.,65.,71.])]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ParallelCollectionRDD[6] at parallelize at PythonRDD.scala:475\n"
     ]
    }
   ],
   "source": [
    "print mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary=Statistics.colStats(mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 17.5       ,  19.86666667,  34.63333333,  39.4       ,  41.23333333])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  41.08      ,   42.17333333,  390.50333333,  587.56      ,\n",
       "        693.16333333])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary.variance()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.,  3.,  3.,  3.,  3.])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary.numNonzeros()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# calculate correlation\n",
    "X = sc.parallelize([10.1,12.4,14.5,16.8,21])\n",
    "Y = sc.parallelize([21.3,24.2,35.4,36.4,31.7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corr = Statistics.corr(X,Y,method='pearson')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6779641435411099"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.linalg import Matrices, Vectors\n",
    "from pyspark.mllib.regression import LabeledPoint\n",
    "from pyspark.mllib.stat import Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vec = Vectors.dense(10.1,12.4,14.5,16.8,21,21.3,24.2,35.4,36.4,31.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseVector([10.1, 12.4, 14.5, 16.8, 21.0, 21.3, 24.2, 35.4, 36.4, 31.7])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "goodnestest = Statistics.chiSqTest(vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chi squared test summary:\n",
      "method: pearson\n",
      "degrees of freedom = 9 \n",
      "statistic = 35.878284182305634 \n",
      "pValue = 4.166733496191455E-5 \n",
      "Very strong presumption against null hypothesis: observed follows the same distribution as expected..\n"
     ]
    }
   ],
   "source": [
    "print (goodnestest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# take the wine quality dataset\n",
    "data = pd.read_csv(\"https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data\",header=None, sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>14.23</td>\n",
       "      <td>1.71</td>\n",
       "      <td>2.43</td>\n",
       "      <td>15.6</td>\n",
       "      <td>127</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.06</td>\n",
       "      <td>0.28</td>\n",
       "      <td>2.29</td>\n",
       "      <td>5.64</td>\n",
       "      <td>1.04</td>\n",
       "      <td>3.92</td>\n",
       "      <td>1065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>13.20</td>\n",
       "      <td>1.78</td>\n",
       "      <td>2.14</td>\n",
       "      <td>11.2</td>\n",
       "      <td>100</td>\n",
       "      <td>2.65</td>\n",
       "      <td>2.76</td>\n",
       "      <td>0.26</td>\n",
       "      <td>1.28</td>\n",
       "      <td>4.38</td>\n",
       "      <td>1.05</td>\n",
       "      <td>3.40</td>\n",
       "      <td>1050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>13.16</td>\n",
       "      <td>2.36</td>\n",
       "      <td>2.67</td>\n",
       "      <td>18.6</td>\n",
       "      <td>101</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.24</td>\n",
       "      <td>0.30</td>\n",
       "      <td>2.81</td>\n",
       "      <td>5.68</td>\n",
       "      <td>1.03</td>\n",
       "      <td>3.17</td>\n",
       "      <td>1185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>14.37</td>\n",
       "      <td>1.95</td>\n",
       "      <td>2.50</td>\n",
       "      <td>16.8</td>\n",
       "      <td>113</td>\n",
       "      <td>3.85</td>\n",
       "      <td>3.49</td>\n",
       "      <td>0.24</td>\n",
       "      <td>2.18</td>\n",
       "      <td>7.80</td>\n",
       "      <td>0.86</td>\n",
       "      <td>3.45</td>\n",
       "      <td>1480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>13.24</td>\n",
       "      <td>2.59</td>\n",
       "      <td>2.87</td>\n",
       "      <td>21.0</td>\n",
       "      <td>118</td>\n",
       "      <td>2.80</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0.39</td>\n",
       "      <td>1.82</td>\n",
       "      <td>4.32</td>\n",
       "      <td>1.04</td>\n",
       "      <td>2.93</td>\n",
       "      <td>735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0      1     2     3     4    5     6     7     8     9     10    11    12  \\\n",
       "0   1  14.23  1.71  2.43  15.6  127  2.80  3.06  0.28  2.29  5.64  1.04  3.92   \n",
       "1   1  13.20  1.78  2.14  11.2  100  2.65  2.76  0.26  1.28  4.38  1.05  3.40   \n",
       "2   1  13.16  2.36  2.67  18.6  101  2.80  3.24  0.30  2.81  5.68  1.03  3.17   \n",
       "3   1  14.37  1.95  2.50  16.8  113  3.85  3.49  0.24  2.18  7.80  0.86  3.45   \n",
       "4   1  13.24  2.59  2.87  21.0  118  2.80  2.69  0.39  1.82  4.32  1.04  2.93   \n",
       "\n",
       "     13  \n",
       "0  1065  \n",
       "1  1050  \n",
       "2  1185  \n",
       "3  1480  \n",
       "4   735  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.regression import LabeledPoint, LinearRegressionModel, LinearRegressionWithSGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sc.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sc = SparkContext(appName='MLAlgo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = sc.textFile(\"C:\\Users\\Dell\\Documents\\winequality.csv\") \\\n",
    "       .map(lambda line: line.split(\",\")) \\\n",
    "    .filter(lambda line: len(line)>1)\\\n",
    "    .map(lambda line: (line[0],line[3],line[2]))\\\n",
    "    .collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'1', u'2.43', u'1.71'), (u'1', u'2.14', u'1.78'), (u'1', u'2.67', u'2.36'), (u'1', u'2.5', u'1.95'), (u'1', u'2.87', u'2.59'), (u'1', u'2.45', u'1.76'), (u'1', u'2.45', u'1.87'), (u'1', u'2.61', u'2.15'), (u'1', u'2.17', u'1.64'), (u'1', u'2.27', u'1.35'), (u'1', u'2.3', u'2.16'), (u'1', u'2.32', u'1.48'), (u'1', u'2.41', u'1.73'), (u'1', u'2.39', u'1.73'), (u'1', u'2.38', u'1.87'), (u'1', u'2.7', u'1.81'), (u'1', u'2.72', u'1.92'), (u'1', u'2.62', u'1.57'), (u'1', u'2.48', u'1.59'), (u'1', u'2.56', u'3.1'), (u'1', u'2.28', u'1.63'), (u'1', u'2.65', u'3.8'), (u'1', u'2.36', u'1.86'), (u'1', u'2.52', u'1.6'), (u'1', u'2.61', u'1.81'), (u'1', u'3.22', u'2.05'), (u'1', u'2.62', u'1.77'), (u'1', u'2.14', u'1.72'), (u'1', u'2.8', u'1.9'), (u'1', u'2.21', u'1.68'), (u'1', u'2.7', u'1.5'), (u'1', u'2.36', u'1.66'), (u'1', u'2.36', u'1.83'), (u'1', u'2.7', u'1.53'), (u'1', u'2.65', u'1.8'), (u'1', u'2.41', u'1.81'), (u'1', u'2.84', u'1.64'), (u'1', u'2.55', u'1.65'), (u'1', u'2.1', u'1.5'), (u'1', u'2.51', u'3.99'), (u'1', u'2.31', u'1.71'), (u'1', u'2.12', u'3.84'), (u'1', u'2.59', u'1.89'), (u'1', u'2.29', u'3.98'), (u'1', u'2.1', u'1.77'), (u'1', u'2.44', u'4.04'), (u'1', u'2.28', u'3.59'), (u'1', u'2.12', u'1.68'), (u'1', u'2.4', u'2.02'), (u'1', u'2.27', u'1.73'), (u'1', u'2.04', u'1.73'), (u'1', u'2.6', u'1.65'), (u'1', u'2.42', u'1.75'), (u'1', u'2.68', u'1.9'), (u'1', u'2.25', u'1.67'), (u'1', u'2.46', u'1.73'), (u'1', u'2.3', u'1.7'), (u'1', u'2.68', u'1.97'), (u'1', u'2.5', u'1.43'), (u'2', u'1.36', u'0.94'), (u'2', u'2.28', u'1.1'), (u'2', u'2.02', u'1.36'), (u'2', u'1.92', u'1.25'), (u'2', u'2.16', u'1.13'), (u'2', u'2.53', u'1.45'), (u'2', u'2.56', u'1.21'), (u'2', u'1.7', u'1.01'), (u'2', u'1.92', u'1.17'), (u'2', u'2.36', u'0.94'), (u'2', u'1.75', u'1.19'), (u'2', u'2.21', u'1.61'), (u'2', u'2.67', u'1.51'), (u'2', u'2.24', u'1.66'), (u'2', u'2.6', u'1.67'), (u'2', u'2.3', u'1.09'), (u'2', u'1.92', u'1.88'), (u'2', u'1.71', u'0.9'), (u'2', u'2.23', u'2.89'), (u'2', u'1.95', u'0.99'), (u'2', u'2.4', u'3.87'), (u'2', u'2', u'0.92'), (u'2', u'2.2', u'1.81'), (u'2', u'2.51', u'1.13'), (u'2', u'2.32', u'3.86'), (u'2', u'2.58', u'0.89'), (u'2', u'2.24', u'0.98'), (u'2', u'2.31', u'1.61'), (u'2', u'2.62', u'1.67'), (u'2', u'2.46', u'2.06'), (u'2', u'2.3', u'1.33'), (u'2', u'2.32', u'1.83'), (u'2', u'2.42', u'1.51'), (u'2', u'2.26', u'1.53'), (u'2', u'2.22', u'2.83'), (u'2', u'2.28', u'1.99'), (u'2', u'2.2', u'1.52'), (u'2', u'2.74', u'2.12'), (u'2', u'1.98', u'1.41'), (u'2', u'2.1', u'1.07'), (u'2', u'2.21', u'3.17'), (u'2', u'1.7', u'2.08'), (u'2', u'1.9', u'1.34'), (u'2', u'2.46', u'2.45'), (u'2', u'1.88', u'1.72'), (u'2', u'1.98', u'1.73'), (u'2', u'2.27', u'2.55'), (u'2', u'2.12', u'1.73'), (u'2', u'2.28', u'1.75'), (u'2', u'1.94', u'1.29'), (u'2', u'2.7', u'1.35'), (u'2', u'1.82', u'3.74'), (u'2', u'2.17', u'2.43'), (u'2', u'2.92', u'2.68'), (u'2', u'2.5', u'0.74'), (u'2', u'2.5', u'1.39'), (u'2', u'2.2', u'1.51'), (u'2', u'1.99', u'1.47'), (u'2', u'2.19', u'1.61'), (u'2', u'1.98', u'3.43'), (u'2', u'2', u'3.43'), (u'2', u'2.42', u'2.4'), (u'2', u'3.23', u'2.05'), (u'2', u'2.73', u'4.43'), (u'2', u'2.13', u'5.8'), (u'2', u'2.39', u'4.31'), (u'2', u'2.17', u'2.16'), (u'2', u'2.29', u'1.53'), (u'2', u'2.78', u'2.13'), (u'2', u'2.3', u'1.63'), (u'2', u'2.38', u'4.3'), (u'3', u'2.32', u'1.35'), (u'3', u'2.4', u'2.99'), (u'3', u'2.4', u'2.31'), (u'3', u'2.36', u'3.55'), (u'3', u'2.25', u'1.24'), (u'3', u'2.2', u'2.46'), (u'3', u'2.54', u'4.72'), (u'3', u'2.64', u'5.51'), (u'3', u'2.19', u'3.59'), (u'3', u'2.61', u'2.96'), (u'3', u'2.7', u'2.81'), (u'3', u'2.35', u'2.56'), (u'3', u'2.72', u'3.17'), (u'3', u'2.35', u'4.95'), (u'3', u'2.2', u'3.88'), (u'3', u'2.15', u'3.57'), (u'3', u'2.23', u'5.04'), (u'3', u'2.48', u'4.61'), (u'3', u'2.38', u'3.24'), (u'3', u'2.36', u'3.9'), (u'3', u'2.62', u'3.12'), (u'3', u'2.48', u'2.67'), (u'3', u'2.75', u'1.9'), (u'3', u'2.28', u'3.3'), (u'3', u'2.1', u'1.29'), (u'3', u'2.32', u'5.19'), (u'3', u'2.38', u'4.12'), (u'3', u'2.64', u'3.03'), (u'3', u'2.7', u'1.68'), (u'3', u'2.64', u'1.67'), (u'3', u'2.38', u'3.83'), (u'3', u'2.54', u'3.26'), (u'3', u'2.58', u'3.27'), (u'3', u'2.35', u'3.45'), (u'3', u'2.3', u'2.76'), (u'3', u'2.26', u'4.36'), (u'3', u'2.6', u'3.7'), (u'3', u'2.3', u'3.37'), (u'3', u'2.69', u'2.58'), (u'3', u'2.86', u'4.6'), (u'3', u'2.32', u'3.03'), (u'3', u'2.28', u'2.39'), (u'3', u'2.48', u'2.51'), (u'3', u'2.45', u'5.65'), (u'3', u'2.48', u'3.91'), (u'3', u'2.26', u'4.28'), (u'3', u'2.37', u'2.59'), (u'3', u'2.74', u'4.1')]\n"
     ]
    }
   ],
   "source": [
    "print (data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# apply logistic regression model using MLLIB in pyspark\n",
    "parsed_data = [LabeledPoint(0.0,[14.23,1.71,2.43,15.6]),\n",
    "              LabeledPoint(0.0,[13.2,1.78,2.14,11.2]),\n",
    "              LabeledPoint(1.0,[21.3,32.4,3.5,21.4]),\n",
    "              LabeledPoint(1.0,[12.4,21.4,21.7,32.8]),\n",
    "              LabeledPoint(2.0,[21,65,45,21]),\n",
    "              LabeledPoint(2.0,[21.5,76.8,54.6,54.9])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(0.0, [14.23,1.71,2.43,15.6]),\n",
       " LabeledPoint(0.0, [13.2,1.78,2.14,11.2]),\n",
       " LabeledPoint(1.0, [21.3,32.4,3.5,21.4]),\n",
       " LabeledPoint(1.0, [12.4,21.4,21.7,32.8]),\n",
       " LabeledPoint(2.0, [21.0,65.0,45.0,21.0]),\n",
       " LabeledPoint(2.0, [21.5,76.8,54.6,54.9])]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parsed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.classification import LogisticRegressionModel, LogisticRegressionWithLBFGS, LogisticRegressionWithSGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegressionWithLBFGS.train(sc.parallelize(parsed_data),numClasses=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.intercept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseVector([-3.0592, 1.5696, -0.5426, 1.5941, -6.3434, 1.8031, 2.3828, 0.0215])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:Anaconda3]",
   "language": "python",
   "name": "conda-env-Anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}