web_crawler4_submit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Crawl single site"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_title(url): \n",
    "    res = requests.get(url) #get the website, return request.Response object\n",
    "    #print(res.status_code) #statu_code: return 200(found web), 404(not found)\n",
    "    soup = BeautifulSoup(res.text, 'html.parser')\n",
    "    us_news_div = soup.find_all('div', re.compile('article-content'))\n",
    "    \n",
    "    title_list = []\n",
    "    for i in range(len(us_news_div)):\n",
    "        us_news_h3 = us_news_div[i].find_all('h3', recursive=False) #header\n",
    "        us_news_a = us_news_h3[0].find_all('a', recursive=False) #anchor tag\n",
    "        for index, item in enumerate(us_news_a[:]):\n",
    "            title = item.text.strip()\n",
    "            title_list.append(title)\n",
    "    return title_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_url(title_list):\n",
    "    title_name = ''\n",
    "    url_list = []\n",
    "    pattern = re.compile('([^\\s\\w]|_)+')\n",
    "\n",
    "    for i in range(len(title_list)):\n",
    "        title = re.sub(\"[‘’]\", '', title_list[i])\n",
    "        #print(title)\n",
    "        strippedList = pattern.sub(' ', title)\n",
    "        a = strippedList.split(\" \")\n",
    "        empty_string = ''\n",
    "        if empty_string in a:\n",
    "            a = [x for x in a if x != '']\n",
    "            title_name = '-'.join(a)\n",
    "        else:\n",
    "            title_name = '-'.join(a)\n",
    "        url = 'https://www.infowars.com/'+title_name\n",
    "        url_list.append(url)        \n",
    "    return url_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_content(url):\n",
    "    res_content = requests.get(url) #get the website, return request.Response object\n",
    "    if(res_content.status_code == 200):\n",
    "        soup2 = BeautifulSoup(res_content.text, 'html.parser')\n",
    "        #article = soup2.find_all('article')\n",
    "        #content = article[0].find_all('p')\n",
    "        \n",
    "        full_content = \"\"\n",
    "        \n",
    "        for p in soup2.select('article > p'):\n",
    "            if p.div or p.a or p.script:  # if div is present - skip\n",
    "                continue\n",
    "            full_content = full_content+p.text.strip()\n",
    "        return full_content\n",
    "    else:\n",
    "        return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "url1 = 'https://www.infowars.com/news/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "content_list = []\n",
    "title_list = get_title(url1)\n",
    "url_list = get_url(title_list)\n",
    "for i in range(len(title_list)):\n",
    "    content_list.append(get_content(url_list[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>URL</th>\n",
       "      <th>Content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MS-13 Member Gets 30 Years for Hacking a Man w...</td>\n",
       "      <td>https://www.infowars.com/MS-13-Member-Gets-30-...</td>\n",
       "      <td>A Virginia judge sentenced an MS-13 member to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Get On Alternative Social Media</td>\n",
       "      <td>https://www.infowars.com/Get-On-Alternative-So...</td>\n",
       "      <td>Big Tech is committed to the leftist agenda an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Truth about CIA’s Illegal MKUltra Mind-control...</td>\n",
       "      <td>https://www.infowars.com/Truth-about-CIAs-Ille...</td>\n",
       "      <td>DISTURBING details of secret mind-control expe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CNN Blames Trump For Bomb Threat: POTUS ‘Inspi...</td>\n",
       "      <td>https://www.infowars.com/CNN-Blames-Trump-For-...</td>\n",
       "      <td>“We were told to evacuate the building and to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Former Dem Comms Director Arrested For Child Porn</td>\n",
       "      <td>https://www.infowars.com/Former-Dem-Comms-Dire...</td>\n",
       "      <td>The former communications director for the Ark...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title  \\\n",
       "0  MS-13 Member Gets 30 Years for Hacking a Man w...   \n",
       "1                    Get On Alternative Social Media   \n",
       "2  Truth about CIA’s Illegal MKUltra Mind-control...   \n",
       "3  CNN Blames Trump For Bomb Threat: POTUS ‘Inspi...   \n",
       "4  Former Dem Comms Director Arrested For Child Porn   \n",
       "\n",
       "                                                 URL  \\\n",
       "0  https://www.infowars.com/MS-13-Member-Gets-30-...   \n",
       "1  https://www.infowars.com/Get-On-Alternative-So...   \n",
       "2  https://www.infowars.com/Truth-about-CIAs-Ille...   \n",
       "3  https://www.infowars.com/CNN-Blames-Trump-For-...   \n",
       "4  https://www.infowars.com/Former-Dem-Comms-Dire...   \n",
       "\n",
       "                                             Content  \n",
       "0  A Virginia judge sentenced an MS-13 member to ...  \n",
       "1  Big Tech is committed to the leftist agenda an...  \n",
       "2  DISTURBING details of secret mind-control expe...  \n",
       "3  “We were told to evacuate the building and to ...  \n",
       "4  The former communications director for the Ark...  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Dataframe: News | URL | Content\n",
    "data = {'Title':title_list,'URL':url_list,'Content':content_list}\n",
    "df = pd.DataFrame(data)\n",
    "df = df[['Title', 'URL', 'Content']]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('result.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Crawl Multiple Web Pages\n",
    "## Dataframe: News | URL | Content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.infowars.com'\n",
    "res = requests.get(url) #get the website, return request.Response object\n",
    "print(res.status_code) #statu_code: return 200(found web), 404(not found)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(res.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tag(res, tag):   \n",
    "    soup = BeautifulSoup(res.text, 'html.parser')\n",
    "    tag_li = soup.find_all('li', re.compile(tag))\n",
    "    pattern = re.compile('([^\\s\\w]|_)+')\n",
    "    tag_list = []\n",
    "    \n",
    "    for i in range(len(tag_li)):\n",
    "        tag_a = tag_li[i].find_all('a', recursive=False)\n",
    "        for index, item in enumerate(tag_a[:]):\n",
    "            tag = item.text.strip()\n",
    "            tag = pattern.sub(' ', re.sub(\"[‘’.]\", '', tag))\n",
    "            tag_list.append(tag)\n",
    "\n",
    "    return list(set(tag_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_category_url(title_list):\n",
    "    title_name = ''\n",
    "    url_list = []\n",
    "    pattern = re.compile('([^\\s\\w]|_)+')\n",
    "\n",
    "    for i in range(len(title_list)):\n",
    "        title = re.sub(\"[‘’]\", '', title_list[i])\n",
    "        #print(title)\n",
    "        strippedList = pattern.sub(' ', title)\n",
    "        a = strippedList.split(\" \")\n",
    "        empty_string = ''\n",
    "        if empty_string in a:\n",
    "            a = [x for x in a if x != '']\n",
    "            title_name = '-'.join(a)\n",
    "        else:\n",
    "            title_name = '-'.join(a)\n",
    "        url = 'https://www.infowars.com/category/'+title_name\n",
    "        url_list.append(url)        \n",
    "    return url_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.infowars.com/category/Government',\n",
       " 'https://www.infowars.com/category/US-News',\n",
       " 'https://www.infowars.com/category/World-News',\n",
       " 'https://www.infowars.com/category/Science-Technology',\n",
       " 'https://www.infowars.com/category/Hot-News',\n",
       " 'https://www.infowars.com/category/Globalism',\n",
       " 'https://www.infowars.com/category/Health',\n",
       " 'https://www.infowars.com/category/World-at-War',\n",
       " 'https://www.infowars.com/category/Economy',\n",
       " 'https://www.infowars.com/category/Special-Reports']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news_tag = 'menu-item menu-item-type-taxonomy menu-item-object-category'\n",
    "category_url_list = get_category_url(get_tag(res, news_tag))\n",
    "category_url_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n"
     ]
    }
   ],
   "source": [
    "#make sure the news webpages exist\n",
    "for i in range(len(category_url_list)):\n",
    "    res = requests.get(category_url_list[i]) #get the website, return request.Response object\n",
    "    print(res.status_code) #statu_code: return 200(found web), 404(not found)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.infowars.com/category/Government',\n",
       " 'https://www.infowars.com/category/US-News',\n",
       " 'https://www.infowars.com/category/World-News',\n",
       " 'https://www.infowars.com/category/Science-Technology',\n",
       " 'https://www.infowars.com/category/Hot-News',\n",
       " 'https://www.infowars.com/category/Globalism',\n",
       " 'https://www.infowars.com/category/Health',\n",
       " 'https://www.infowars.com/category/World-at-War',\n",
       " 'https://www.infowars.com/category/Economy',\n",
       " 'https://www.infowars.com/category/Special-Reports']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "category_url_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Trump Picks William Barr as Next Attorney General',\n",
       " 'Sen. Ted Cruz, Others Announce Bill to Fund $25 Billion Border Wall',\n",
       " 'What the Fake History of Guns Can Teach Us',\n",
       " 'Clinton Foundation Whistleblowers Produce Devastating Evidence',\n",
       " 'Dem Shopping List for Gun Control',\n",
       " 'House Passes Bill to Delay Gov Shutdown, Border Wall Funding',\n",
       " 'Mexico’s New Leader Eager to Talk to Trump About Immigration',\n",
       " 'Kamala Harris Aide Resigns After Harassment, Retaliation Settlement Surfaces',\n",
       " 'House Democrats Plan Push to Criminalize Private Gun Sales',\n",
       " 'Trump Task Force Lets Postal Service Hike Prices For Amazon',\n",
       " 'Sen. Grassley Wants Answers About FBI Raid on Clinton Foundation Whistleblower',\n",
       " 'Roger Stone: Trump Doesn’t Know He Signed Free Speech Killing Agreement',\n",
       " 'Christmas-Themed Satanic Sculpture Installed At Illinois Statehouse',\n",
       " 'Roger Stone Calls Out Adam Schiff; If You Have Evidence, Present It Now',\n",
       " 'Watch: D’Souza DESTROYS “Proud Democrat” in Heated Q&A',\n",
       " 'Calif. Dems Plan to Extend Medicaid to Illegal Immigrants',\n",
       " 'Eric Trump Blasts Kellyanne Conway’s Husband For Disrespecting Own Wife',\n",
       " 'Why Did a Congressman Threaten to Nuke Those Who Don’t Submit to Gun Control?',\n",
       " 'The Many Ways Governments Create Monopolies',\n",
       " 'San Francisco’s Wealthy Leftists Are Making Homelessness Worse',\n",
       " 'Learn Why Mueller Is Raiding More Trump Lawyers',\n",
       " 'More Women Traveling Out of State For Abortions During Pro-Life Policy Surge',\n",
       " 'House Dems Won’t Compromise on $15 Minimum Wage',\n",
       " 'Slower Police Response Times Highlight Need For Second Amendment',\n",
       " 'Senators Seeking To Shoot Down Yemen Bill Were Paid By Saudi Lobbyists – Report',\n",
       " 'Dems Jump On Cohen Plea, Say More Trump Allies Lied to Congress',\n",
       " 'They Spy on Your Sh*t, But Presidential Poop is Classified — LITERALLY',\n",
       " 'Pence Breaks Senate Tie to Advance Trump’s Judicial Nominee',\n",
       " 'State Funds Nonprofit That Provided ‘Material Support’ to Iran, Hamas',\n",
       " 'House Committee Investigating Twitter CEO For Lying Under Oath',\n",
       " 'President Trump Rallies in Mississippi To Stop Democrat Steal Of Senate Seat',\n",
       " 'Dershowitz: Mueller ‘Not Going To Produce Balanced, Fair Report’']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_title(category_url_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_title_list = []\n",
    "for i in range(len(category_url_list)):\n",
    "    title_list = get_title(category_url_list[i])\n",
    "    news_title_list = news_title_list + title_list\n",
    "    news_title_list = list(set(news_title_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "319"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(news_title_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_content_list = []\n",
    "multi_page_url_list = get_url(news_title_list)\n",
    "for i in range(len(news_title_list)):\n",
    "    new_content_list.append(get_content(multi_page_url_list[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dataframe: News | URL | Content\n",
    "data = {'Title':news_title_list,'URL':multi_page_url_list,'Content':new_content_list}\n",
    "df = pd.DataFrame(data)\n",
    "df = df[['Title', 'URL', 'Content']]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('result.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Crawl Breaking News "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_breaking = \"https://www.infowars.com/breaking-news/\"\n",
    "#https://www.infowars.com/breaking-news/page/2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_article_title(url, num_of_pages): \n",
    "    page = 1\n",
    "    full_title_list = []\n",
    "    while page <= num_of_pages:\n",
    "        url_2 = url + \"page/%s/\" % page #get multiple page with \"load page button\"\n",
    "        res = requests.get(url_2)\n",
    "        soup = BeautifulSoup(res.text, 'html.parser')\n",
    "        us_news_div = soup.find_all('article', re.compile('post-'))\n",
    "        \n",
    "        title_list = []\n",
    "        for i in range(len(us_news_div)):\n",
    "            us_news_a = us_news_div[i].find_all('a')\n",
    "            us_news_h3 = us_news_a[0].find_all('h3')\n",
    "\n",
    "            for index, item in enumerate(us_news_h3[:]):\n",
    "                title = item.text.strip()\n",
    "                #print(title)\n",
    "                title_list.append(title)\n",
    "        \n",
    "        full_title_list.extend(title_list)\n",
    "        page += 1\n",
    "\n",
    "    return full_title_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "content_list = []\n",
    "title_list = get_article_title(url_breaking, 500)\n",
    "url_list = get_url(title_list)\n",
    "for i in range(len(title_list)):\n",
    "    content_list.append(get_content(url_list[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {'Title':title_list,'URL':url_list,'Content':content_list}\n",
    "df = pd.DataFrame(data)\n",
    "df = df[['Title', 'URL', 'Content']]\n",
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('infowars5000_new.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}