Twitter_Sentiment_ETL-Custom.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building training data...\n",
      "Building classifier...\n",
      "Importing Twitter Data...\n",
      "Preparing ./Twitter_Full_08172018.csv...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 2854: expected 7 fields, saw 8\\nSkipping line 3079: expected 7 fields, saw 8\\nSkipping line 3457: expected 7 fields, saw 8\\nSkipping line 3955: expected 7 fields, saw 8\\n'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No duplicate detected. Continuing...\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# coding: utf-8\n",
    "\n",
    "# # USDA Twitter Sentiment Analysis\n",
    "\n",
    "# Performs sentiment analysis on Twitter data mentions of various USDA accounts\n",
    "# \n",
    "# Author: Erik Bethke\n",
    "# Last modified: 7.30.18\n",
    "\n",
    "# ## Installation of nltk and data\n",
    "\n",
    "# pip3 install nltk==3.2.4\n",
    "\n",
    "# sudo python3 -m nltk.downloader all\n",
    "\n",
    "# Importing modules and basic functions\n",
    "\n",
    "# Import modules for analysis\n",
    "import nltk\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import sys\n",
    "import string\n",
    "import datetime\n",
    "import shutil\n",
    "from pathlib import Path\n",
    "from nltk.classify import NaiveBayesClassifier\n",
    "from nltk.classify.util import accuracy\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "\n",
    "now = datetime.datetime.now()\n",
    "dmy =  str(now.month) + \"_\" + str(now.day) + \"_\" + str(now.year)\n",
    "\n",
    "def format_sentence(sent):\n",
    "    return({word: True for word in nltk.word_tokenize(sent)})\n",
    "\n",
    "\n",
    "# ## Setting up training model\n",
    "\n",
    "# ### Import positive tweet training data\n",
    "\n",
    "pos = []\n",
    "with open(\"./pos_tweets.txt\") as f:\n",
    "    for i in f:\n",
    "        pos.append([format_sentence(i), 'pos'])\n",
    "\n",
    "\n",
    "# ### Import negative tweet training data\n",
    "\n",
    "#Establish negative tweet trainig dat\n",
    "neg = []\n",
    "with open(\"./neg_tweets.txt\", encoding='utf-8') as f:\n",
    "    for i in f:\n",
    "        neg.append([format_sentence(i), 'neg'])\n",
    "\n",
    "all_training = pos + neg\n",
    "print(\"Building training data...\")\n",
    "\n",
    "\n",
    "# ### Build classifier\n",
    "\n",
    "classifier = NaiveBayesClassifier.train(all_training)\n",
    "print(\"Building classifier...\")\n",
    "\n",
    "# ## USDA Twitter Data\n",
    "\n",
    "# ### Import USDA Twitter Data\n",
    "\n",
    "# Opening in pandas dataframe\n",
    "\n",
    "print(\"Importing Twitter Data...\")\n",
    "csv = glob.glob('*Twitter_Full*')\n",
    "csv = './' + csv[0]\n",
    "print(\"Preparing \" + csv + \"...\")\n",
    "df = pd.read_csv(csv, sep='|', error_bad_lines=False, warn_bad_lines=True, encoding='latin-1')\n",
    "\n",
    "#df = pd.read_csv('./Twitter_Full_18062018_parsed.csv')\n",
    "#df = pd.read_excel('./Twitter_Final_18062018.xlsx', 'Sheet1')\n",
    "\n",
    "\n",
    "# Creating a data frame to hold the file name and date, to check against existing data in master list (and later for export)\n",
    "\n",
    "df_fileList = pd.DataFrame(columns = ['FileName', 'Date'])\n",
    "df_fileList.loc[(len(df_fileList))] = [csv, dmy]\n",
    "\n",
    "\n",
    "# Checking for Twitter Master, then checking if analysis has already been completed\n",
    "\n",
    "master = Path('./Twitter_Master.xlsx')\n",
    "if master.is_file():\n",
    "    print('Comparing file to list of completed sentiment analyses for duplicates...')\n",
    "    # Read master file list, append df_fileList to master list\n",
    "    df_master_fileList = pd.read_excel(master, 'Sheet2', index=False)\n",
    "    if (df_master_fileList['FileName'].str.contains(csv).sum() > 0):\n",
    "        print('Sentiment analysis already completed on this file. Exiting...')\n",
    "        print('Please remove duplicate file from root directory.')\n",
    "        sys.exit()\n",
    "    else:\n",
    "        print('No duplicate detected. Continuing...')\n",
    "else:\n",
    "    print('No duplicate detected. Continuing...')\n",
    "\n",
    "\n",
    "# Run classifier on tweet texts, check for errors\n",
    "\n",
    "tweetSentiment = []\n",
    "for tweet in df['tweetText']:\n",
    "    try:\n",
    "        tweetSentiment.append(classifier.classify(format_sentence(tweet)))\n",
    "    except:\n",
    "        tweetSentiment.append('error')       \n",
    "\n",
    "\n",
    "# Push tweetSentiment to dataframe\n",
    "\n",
    "df['tweetSentiment'] = tweetSentiment\n",
    "df = df.fillna('')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweetId</th>\n",
       "      <th>tweetDate</th>\n",
       "      <th>tweetText</th>\n",
       "      <th>hashtagList</th>\n",
       "      <th>location</th>\n",
       "      <th>numberRetweets</th>\n",
       "      <th>userMentions_userName</th>\n",
       "      <th>tweetSentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1023719028395847680</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @amjoyshow: SHIRLEY SHERROD former @USDA of...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1023718430329847808</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @SecretarySonny: The aid for farmers announ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1416</td>\n",
       "      <td>@SecretarySonny</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1023717896978030593</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1023717437592633344</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@FoxBusiness @SecretarySonny @POTUS China does...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1023717233506181120</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>Retweeted FOX Business (@FoxBusiness):</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1023717218167619584</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1023717188685975552</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1023716928928538626</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1023716905797005312</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1023716878991196162</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>?Obviously this is not going to make farmers w...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>@SecretarySonny,@hughreuters</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1023716510177681408</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @amjoyshow: SHIRLEY SHERROD former @USDA of...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1023716464941887488</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1023716125853532160</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1023715891320692736</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @SecretarySonny: Celebrating the reintroduc...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>60</td>\n",
       "      <td>@SecretarySonny,@USAmbassadorARG</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1023715628094504960</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@FoxBusiness @SecretarySonny @POTUS yeah borro...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1023715561145028610</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1023715525287927809</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @SecretarySonny: The aid for farmers announ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1416</td>\n",
       "      <td>@SecretarySonny</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1023715029453926400</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1023714998651183104</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1023714800981954561</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1023714800612909057</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1023714800503803904</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FoxBusiness: .@SecretarySonny on @POTUS' $...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@FoxBusiness,@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1023714789900484608</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>.@SecretarySonny on @POTUS' $12B farm aid plan...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>27</td>\n",
       "      <td>@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1023714419354693633</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@SecretarySonny @USDAForeignAg  https://t.co/N...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@SecretarySonny,@USDAForeignAg</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1023713836212387842</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @amjoyshow: SHIRLEY SHERROD former @USDA of...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1023713362998423552</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @amjoyshow: SHIRLEY SHERROD former @USDA of...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1023713275144536064</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @amjoyshow: SHIRLEY SHERROD former @USDA of...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>1023712892439457792</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @amjoyshow: SHIRLEY SHERROD former @USDA of...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>1023712209719971841</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @mtnsoccerfan: @kylegriffin1 Farmers are ve...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>@mtnsoccerfan,@kylegriffin1</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1023711934024220672</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@TheMizzouTiger @FarmPolicy @SecretarySonny Bi...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@TheMizzouTiger,@FarmPolicy,@SecretarySonny</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>991</th>\n",
       "      <td>1023519975410032640</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @SecretarySonny: .@POTUS has made clear tha...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>74</td>\n",
       "      <td>@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>1023519884142170112</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @SecretarySonny: The aid for farmers announ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1416</td>\n",
       "      <td>@SecretarySonny</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>993</th>\n",
       "      <td>1023519139154022400</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @USDA: Raise a glass...or a jug, it's #Nati...</td>\n",
       "      <td>#NationalMilkChocolateDay</td>\n",
       "      <td></td>\n",
       "      <td>44</td>\n",
       "      <td>@USDA</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>1023509343063425024</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @RepJenniffer: Los agricultores recibieron ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>@RepJenniffer,@USDA,@usdaRD,@usdafsa_es,@SBA_P...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>1023496076404215809</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @SecretarySonny: .@POTUS has made clear tha...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>74</td>\n",
       "      <td>@SecretarySonny,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>1023487647916777473</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @Archimedes2020: 45* accidentally appointed...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>@Archimedes2020,@USDA</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>1023482047770320896</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @ENERGY: WATCH: These cool-looking research...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>35</td>\n",
       "      <td>@ENERGY</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>1023476191439802368</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @nafoalliance: 14,000,000 acres would be a ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>1023476000573804544</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @RepJenniffer: Los agricultores recibieron ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>@RepJenniffer,@USDA,@usdaRD,@usdafsa_es,@SBA_P...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1001</th>\n",
       "      <td>1023467454083866624</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @ENERGY: WATCH: These cool-looking research...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>35</td>\n",
       "      <td>@ENERGY</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>1023467435981197312</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@USDA @foxnews</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1005</th>\n",
       "      <td>1023463776488579073</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @SecretarySonny: The aid for farmers announ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1416</td>\n",
       "      <td>@SecretarySonny</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1006</th>\n",
       "      <td>1023461994890899457</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @USDA: Whether it?s a family BBQ, camping, ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>35</td>\n",
       "      <td>@USDA</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1007</th>\n",
       "      <td>1023459748056690688</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @Cabinet: .@SecretarySonny writes that Pres...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>379</td>\n",
       "      <td>@Cabinet,@SecretarySonny</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1008</th>\n",
       "      <td>1023450321979203585</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@SecretarySonny @PressSec @POTUS No. It's a cl...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@SecretarySonny,@PressSec,@POTUS</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1009</th>\n",
       "      <td>1023446639791497216</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @Dairy_is_scary: @omarrestrepo1 @Monica26Re...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>@Dairy_is_scary,@omarrestrepo1,@Monica26Renee,...</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1010</th>\n",
       "      <td>1023446625321148416</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @FarmersGov: At just 15, McKenzie came to t...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>@FarmersGov,@usdafsa</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1011</th>\n",
       "      <td>1023446539270844416</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@SecretarySonny @PressSec @POTUS Really?</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1016</th>\n",
       "      <td>1023442850623700992</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @SecretarySonny: The aid for farmers announ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>1416</td>\n",
       "      <td>@SecretarySonny</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1017</th>\n",
       "      <td>1023438118765187072</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>45* accidentally appointed someone competent @...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>@USDA</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1018</th>\n",
       "      <td>1023435540845092864</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @BadAssKittyG: .@akcdoglovers @TheKennelClu...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1020</th>\n",
       "      <td>1023434359347597314</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @foe_us: We applaud EU's court for ruling t...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>1023434035849306112</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>#NoBrainer #Cancer @USDA https://t.co/LRybhu5S2b</td>\n",
       "      <td>#NoBrainer,#Cancer</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@USDA</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1023</th>\n",
       "      <td>1023431265243090944</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@agrichad @MisscoTom @joelbrown22 @Micajahmc @...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@agrichad,@MisscoTom,@joelbrown22,@Micajahmc,@...</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1024</th>\n",
       "      <td>1023431201430749184</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @foe_us: We applaud EU's court for ruling t...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1026</th>\n",
       "      <td>1023428452026855424</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @GOP: ?President Trump has pledged to stand...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>296</td>\n",
       "      <td>@GOP</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1027</th>\n",
       "      <td>1023428309663662082</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @James_Stacey_: @SecretarySonny admits US f...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>@James_Stacey_,@SecretarySonny</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1028</th>\n",
       "      <td>1023423644687425536</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>?@POTUS? CASH Cow 4Misogynist Male Voters. Usi...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@POTUS,@GOP</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029</th>\n",
       "      <td>1023420825158922241</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>RT @James_Stacey_: @SecretarySonny admits US f...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>@James_Stacey_,@SecretarySonny</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1030</th>\n",
       "      <td>1023418720675418112</td>\n",
       "      <td>07-29-2018</td>\n",
       "      <td>@Monica26Renee @TheHumaneLeague @omarrestrepo1...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>@Monica26Renee,@TheHumaneLeague,@omarrestrepo1...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>673 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  tweetId   tweetDate  \\\n",
       "0     1023719028395847680  07-29-2018   \n",
       "2     1023718430329847808  07-29-2018   \n",
       "3     1023717896978030593  07-29-2018   \n",
       "4     1023717437592633344  07-29-2018   \n",
       "5     1023717233506181120  07-29-2018   \n",
       "7     1023717218167619584  07-29-2018   \n",
       "8     1023717188685975552  07-29-2018   \n",
       "9     1023716928928538626  07-29-2018   \n",
       "10    1023716905797005312  07-29-2018   \n",
       "11    1023716878991196162  07-29-2018   \n",
       "12    1023716510177681408  07-29-2018   \n",
       "14    1023716464941887488  07-29-2018   \n",
       "15    1023716125853532160  07-29-2018   \n",
       "16    1023715891320692736  07-29-2018   \n",
       "17    1023715628094504960  07-29-2018   \n",
       "18    1023715561145028610  07-29-2018   \n",
       "19    1023715525287927809  07-29-2018   \n",
       "20    1023715029453926400  07-29-2018   \n",
       "21    1023714998651183104  07-29-2018   \n",
       "22    1023714800981954561  07-29-2018   \n",
       "23    1023714800612909057  07-29-2018   \n",
       "24    1023714800503803904  07-29-2018   \n",
       "25    1023714789900484608  07-29-2018   \n",
       "26    1023714419354693633  07-29-2018   \n",
       "27    1023713836212387842  07-29-2018   \n",
       "29    1023713362998423552  07-29-2018   \n",
       "31    1023713275144536064  07-29-2018   \n",
       "33    1023712892439457792  07-29-2018   \n",
       "35    1023712209719971841  07-29-2018   \n",
       "36    1023711934024220672  07-29-2018   \n",
       "...                   ...         ...   \n",
       "991   1023519975410032640  07-29-2018   \n",
       "992   1023519884142170112  07-29-2018   \n",
       "993   1023519139154022400  07-29-2018   \n",
       "994   1023509343063425024  07-29-2018   \n",
       "995   1023496076404215809  07-29-2018   \n",
       "996   1023487647916777473  07-29-2018   \n",
       "997   1023482047770320896  07-29-2018   \n",
       "998   1023476191439802368  07-29-2018   \n",
       "1000  1023476000573804544  07-29-2018   \n",
       "1001  1023467454083866624  07-29-2018   \n",
       "1002  1023467435981197312  07-29-2018   \n",
       "1005  1023463776488579073  07-29-2018   \n",
       "1006  1023461994890899457  07-29-2018   \n",
       "1007  1023459748056690688  07-29-2018   \n",
       "1008  1023450321979203585  07-29-2018   \n",
       "1009  1023446639791497216  07-29-2018   \n",
       "1010  1023446625321148416  07-29-2018   \n",
       "1011  1023446539270844416  07-29-2018   \n",
       "1016  1023442850623700992  07-29-2018   \n",
       "1017  1023438118765187072  07-29-2018   \n",
       "1018  1023435540845092864  07-29-2018   \n",
       "1020  1023434359347597314  07-29-2018   \n",
       "1022  1023434035849306112  07-29-2018   \n",
       "1023  1023431265243090944  07-29-2018   \n",
       "1024  1023431201430749184  07-29-2018   \n",
       "1026  1023428452026855424  07-29-2018   \n",
       "1027  1023428309663662082  07-29-2018   \n",
       "1028  1023423644687425536  07-29-2018   \n",
       "1029  1023420825158922241  07-29-2018   \n",
       "1030  1023418720675418112  07-29-2018   \n",
       "\n",
       "                                              tweetText  \\\n",
       "0     RT @amjoyshow: SHIRLEY SHERROD former @USDA of...   \n",
       "2     RT @SecretarySonny: The aid for farmers announ...   \n",
       "3     RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "4     @FoxBusiness @SecretarySonny @POTUS China does...   \n",
       "5                Retweeted FOX Business (@FoxBusiness):   \n",
       "7     RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "8     RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "9     RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "10    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "11    ?Obviously this is not going to make farmers w...   \n",
       "12    RT @amjoyshow: SHIRLEY SHERROD former @USDA of...   \n",
       "14    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "15    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "16    RT @SecretarySonny: Celebrating the reintroduc...   \n",
       "17    @FoxBusiness @SecretarySonny @POTUS yeah borro...   \n",
       "18    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "19    RT @SecretarySonny: The aid for farmers announ...   \n",
       "20    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "21    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "22    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "23    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "24    RT @FoxBusiness: .@SecretarySonny on @POTUS' $...   \n",
       "25    .@SecretarySonny on @POTUS' $12B farm aid plan...   \n",
       "26    @SecretarySonny @USDAForeignAg  https://t.co/N...   \n",
       "27    RT @amjoyshow: SHIRLEY SHERROD former @USDA of...   \n",
       "29    RT @amjoyshow: SHIRLEY SHERROD former @USDA of...   \n",
       "31    RT @amjoyshow: SHIRLEY SHERROD former @USDA of...   \n",
       "33    RT @amjoyshow: SHIRLEY SHERROD former @USDA of...   \n",
       "35    RT @mtnsoccerfan: @kylegriffin1 Farmers are ve...   \n",
       "36    @TheMizzouTiger @FarmPolicy @SecretarySonny Bi...   \n",
       "...                                                 ...   \n",
       "991   RT @SecretarySonny: .@POTUS has made clear tha...   \n",
       "992   RT @SecretarySonny: The aid for farmers announ...   \n",
       "993   RT @USDA: Raise a glass...or a jug, it's #Nati...   \n",
       "994   RT @RepJenniffer: Los agricultores recibieron ...   \n",
       "995   RT @SecretarySonny: .@POTUS has made clear tha...   \n",
       "996   RT @Archimedes2020: 45* accidentally appointed...   \n",
       "997   RT @ENERGY: WATCH: These cool-looking research...   \n",
       "998   RT @nafoalliance: 14,000,000 acres would be a ...   \n",
       "1000  RT @RepJenniffer: Los agricultores recibieron ...   \n",
       "1001  RT @ENERGY: WATCH: These cool-looking research...   \n",
       "1002                                    @USDA @foxnews    \n",
       "1005  RT @SecretarySonny: The aid for farmers announ...   \n",
       "1006  RT @USDA: Whether it?s a family BBQ, camping, ...   \n",
       "1007  RT @Cabinet: .@SecretarySonny writes that Pres...   \n",
       "1008  @SecretarySonny @PressSec @POTUS No. It's a cl...   \n",
       "1009  RT @Dairy_is_scary: @omarrestrepo1 @Monica26Re...   \n",
       "1010  RT @FarmersGov: At just 15, McKenzie came to t...   \n",
       "1011           @SecretarySonny @PressSec @POTUS Really?   \n",
       "1016  RT @SecretarySonny: The aid for farmers announ...   \n",
       "1017  45* accidentally appointed someone competent @...   \n",
       "1018  RT @BadAssKittyG: .@akcdoglovers @TheKennelClu...   \n",
       "1020  RT @foe_us: We applaud EU's court for ruling t...   \n",
       "1022   #NoBrainer #Cancer @USDA https://t.co/LRybhu5S2b   \n",
       "1023  @agrichad @MisscoTom @joelbrown22 @Micajahmc @...   \n",
       "1024  RT @foe_us: We applaud EU's court for ruling t...   \n",
       "1026  RT @GOP: ?President Trump has pledged to stand...   \n",
       "1027  RT @James_Stacey_: @SecretarySonny admits US f...   \n",
       "1028  ?@POTUS? CASH Cow 4Misogynist Male Voters. Usi...   \n",
       "1029  RT @James_Stacey_: @SecretarySonny admits US f...   \n",
       "1030  @Monica26Renee @TheHumaneLeague @omarrestrepo1...   \n",
       "\n",
       "                    hashtagList location numberRetweets  \\\n",
       "0                                                         \n",
       "2                                                  1416   \n",
       "3                                                    27   \n",
       "4                                                     0   \n",
       "5                                                         \n",
       "7                                                    27   \n",
       "8                                                    27   \n",
       "9                                                    27   \n",
       "10                                                   27   \n",
       "11                                                    2   \n",
       "12                                                        \n",
       "14                                                   27   \n",
       "15                                                   27   \n",
       "16                                                   60   \n",
       "17                                                    0   \n",
       "18                                                   27   \n",
       "19                                                 1416   \n",
       "20                                                   27   \n",
       "21                                                   27   \n",
       "22                                                   27   \n",
       "23                                                   27   \n",
       "24                                                   27   \n",
       "25                                                   27   \n",
       "26                                                    0   \n",
       "27                                                        \n",
       "29                                                        \n",
       "31                                                        \n",
       "33                                                        \n",
       "35                                                    2   \n",
       "36                                                    0   \n",
       "...                         ...      ...            ...   \n",
       "991                                                  74   \n",
       "992                                                1416   \n",
       "993   #NationalMilkChocolateDay                      44   \n",
       "994                                                  16   \n",
       "995                                                  74   \n",
       "996                                                   3   \n",
       "997                                                  35   \n",
       "998                                                       \n",
       "1000                                                 16   \n",
       "1001                                                 35   \n",
       "1002                                                      \n",
       "1005                                               1416   \n",
       "1006                                                 35   \n",
       "1007                                                379   \n",
       "1008                                                  0   \n",
       "1009                                                  5   \n",
       "1010                                                 15   \n",
       "1011                                                      \n",
       "1016                                               1416   \n",
       "1017                                                  3   \n",
       "1018                                                      \n",
       "1020                                                      \n",
       "1022         #NoBrainer,#Cancer                       0   \n",
       "1023                                                  0   \n",
       "1024                                                      \n",
       "1026                                                296   \n",
       "1027                                                  3   \n",
       "1028                                                  0   \n",
       "1029                                                  3   \n",
       "1030                                                  0   \n",
       "\n",
       "                                  userMentions_userName tweetSentiment  \n",
       "0                                                                  neg  \n",
       "2                                       @SecretarySonny            neg  \n",
       "3                   @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "4                   @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "5                                                                  pos  \n",
       "7                   @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "8                   @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "9                   @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "10                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "11                         @SecretarySonny,@hughreuters            neg  \n",
       "12                                                                 neg  \n",
       "14                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "15                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "16                     @SecretarySonny,@USAmbassadorARG            pos  \n",
       "17                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "18                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "19                                      @SecretarySonny            neg  \n",
       "20                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "21                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "22                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "23                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "24                  @FoxBusiness,@SecretarySonny,@POTUS            neg  \n",
       "25                               @SecretarySonny,@POTUS            neg  \n",
       "26                       @SecretarySonny,@USDAForeignAg            neg  \n",
       "27                                                                 neg  \n",
       "29                                                                 neg  \n",
       "31                                                                 neg  \n",
       "33                                                                 neg  \n",
       "35                          @mtnsoccerfan,@kylegriffin1            neg  \n",
       "36          @TheMizzouTiger,@FarmPolicy,@SecretarySonny            neg  \n",
       "...                                                 ...            ...  \n",
       "991                              @SecretarySonny,@POTUS            neg  \n",
       "992                                     @SecretarySonny            neg  \n",
       "993                                               @USDA            neg  \n",
       "994   @RepJenniffer,@USDA,@usdaRD,@usdafsa_es,@SBA_P...            pos  \n",
       "995                              @SecretarySonny,@POTUS            neg  \n",
       "996                               @Archimedes2020,@USDA            neg  \n",
       "997                                             @ENERGY            neg  \n",
       "998                                                                pos  \n",
       "1000  @RepJenniffer,@USDA,@usdaRD,@usdafsa_es,@SBA_P...            pos  \n",
       "1001                                            @ENERGY            neg  \n",
       "1002                                                               neg  \n",
       "1005                                    @SecretarySonny            neg  \n",
       "1006                                              @USDA            pos  \n",
       "1007                           @Cabinet,@SecretarySonny            pos  \n",
       "1008                   @SecretarySonny,@PressSec,@POTUS            neg  \n",
       "1009  @Dairy_is_scary,@omarrestrepo1,@Monica26Renee,...            neg  \n",
       "1010                               @FarmersGov,@usdafsa            pos  \n",
       "1011                                                               neg  \n",
       "1016                                    @SecretarySonny            neg  \n",
       "1017                                              @USDA            neg  \n",
       "1018                                                               neg  \n",
       "1020                                                               pos  \n",
       "1022                                              @USDA            neg  \n",
       "1023  @agrichad,@MisscoTom,@joelbrown22,@Micajahmc,@...            neg  \n",
       "1024                                                               pos  \n",
       "1026                                               @GOP            pos  \n",
       "1027                     @James_Stacey_,@SecretarySonny            neg  \n",
       "1028                                        @POTUS,@GOP            pos  \n",
       "1029                     @James_Stacey_,@SecretarySonny            neg  \n",
       "1030  @Monica26Renee,@TheHumaneLeague,@omarrestrepo1...            pos  \n",
       "\n",
       "[673 rows x 8 columns]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter for only yesterday's data to avoid duplicate date data\n",
    "#df = df[df['tweetDate'] == (datetime.date.today() - datetime.timedelta(1)).strftime('%m-%d-%Y')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing stopwords...\n",
      "Generating parsed word file...\n",
      "Word file parsed...\n",
      "Writing output files...\n",
      "Job completed\n"
     ]
    }
   ],
   "source": [
    "# ### Import stopwords data and push to data frame\n",
    "\n",
    "print('Importing stopwords...')\n",
    "df_stop = pd.read_excel('./stopwords.xlsx', header=None, names=['stop'])\n",
    "\n",
    "\n",
    "# Push to list\n",
    "\n",
    "stopList = df_stop['stop'].values.tolist()\n",
    "\n",
    "\n",
    "# ### Split tweet text, create new dataframe for each word in each tweet with associated sentiment\n",
    "\n",
    "# Prepare new dataframe df_word for words\n",
    "\n",
    "# New dataframe for individual words\n",
    "df_word_cols = list(df.columns)\n",
    "df_word_cols.append('tweetWord')\n",
    "\n",
    "df_word = pd.DataFrame(columns = df_word_cols)\n",
    "\n",
    "# Split words into list format, set up dataframe locators\n",
    "wordList = df['tweetText'].str.split(' ')\n",
    "\n",
    "\n",
    "# Populate dataframe df_word\n",
    "\n",
    "df_word_vals = []\n",
    "df_loc = 0\n",
    "word_loc = 0\n",
    "lowerExcept = ['US', 'U.S.']\n",
    "\n",
    "# Build translator using string module to remove punctuation from words (excluding @ or #)\n",
    "translatorWords = string.punctuation\n",
    "translatorBuild = str.maketrans('', '', '@#')\n",
    "translatorWords = translatorWords.translate(translatorBuild)\n",
    "translator = str.maketrans('', '', translatorWords)\n",
    "\n",
    "print(\"Generating parsed word file...\")\n",
    "\n",
    "# Parse through all word lists, then parse through all words to populate df_word\n",
    "for words in wordList:\n",
    "    for word in words:\n",
    "        # Drop punctuation from word, ignoring # or @\n",
    "        # Then convert to lower case if not in list of exceptions\n",
    "        if word in lowerExcept:\n",
    "            wordDrop = word.translate(translator)\n",
    "        else:\n",
    "            wordDrop = word.translate(translator).lower()\n",
    "        \n",
    "        # Check word against stop list\n",
    "        if wordDrop in stopList:\n",
    "            pass\n",
    "        #something broken here\n",
    "        elif wordDrop not in stopList:\n",
    "            # something going wrong in this section\n",
    "            df_append = df.iloc[df_loc]\n",
    "            df_word = df_word.append(df_append, ignore_index=True)\n",
    "            try:\n",
    "                df_word.at[word_loc, 'tweetWord'] = wordDrop\n",
    "            except:\n",
    "                df_word = df_word.replace(np.nan, '', regex=True)\n",
    "                df_word.at[word_loc, 'tweetWord'] = wordDrop\n",
    "            word_loc += 1\n",
    "    df_loc += 1\n",
    "    \n",
    "print(\"Word file parsed...\")\n",
    "\n",
    "\n",
    "# Combine all columns into 1 for improved filesize\n",
    "\n",
    "df_word['UID'] = (df_word.index + 1).astype(str) + '-' +  df_word['tweetDate']\n",
    "\n",
    "df_word_out = df_word[['UID', 'tweetId', 'tweetDate', 'tweetWord', 'tweetSentiment']].copy()\n",
    "\n",
    "\n",
    "# Outputting to CSV, archives Full_Data, Sentence Sentiment, and Word Sentiment appropriately\n",
    "\n",
    "print(\"Writing output files...\")\n",
    "#df.to_csv('./archive/Sentence_Sentiment/Twitter_PythonSentiment_' + dmy + '.csv', encoding='utf-8', index=False)\n",
    "#df_word_out.to_csv('./archive/Word_Sentiment/Twitter_PythonSentiment_Word_' + dmy + '.csv', encoding='utf-8', index=False)\n",
    "#shutil.move(csv, './archive/Full_Data/')\n",
    "\n",
    "\n",
    "# ### Exporting to Master Data\n",
    "\n",
    "# Check for Twitter_Master Excel, then write to appropriate locations\n",
    "\n",
    "master = Path('./Twitter_Master.xlsx')\n",
    "if master.is_file():\n",
    "    # Read master data, append df_word_out to master list\n",
    "    df_master = pd.read_excel(master, 'Sheet1', index=False)\n",
    "    df_master = df_master.append(df_word_out, ignore_index=True)\n",
    "    \n",
    "    # Read master file list, append df_fileList to master list\n",
    "    df_master_fileList = pd.read_excel(master, 'Sheet2', index=False)\n",
    "    df_master_fileList = df_master_fileList.append(df_fileList)\n",
    "    \n",
    "    writer = pd.ExcelWriter('./Twitter_Master.xlsx')\n",
    "    df_master.to_excel(writer,'Sheet1', index=False)\n",
    "    df_master_fileList.to_excel(writer,'Sheet2', index=False)\n",
    "    writer.save()\n",
    "else:\n",
    "    writer = pd.ExcelWriter('./Twitter_Master.xlsx')\n",
    "    df_word_out.to_excel(writer,'Sheet1', index=False)\n",
    "    df_fileList.to_excel(writer,'Sheet2', index=False)\n",
    "    writer.save()\n",
    "    \n",
    "print('Job completed')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}