n11_web_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Requirment Libraries"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "4d00a91d4d11a28e"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "import requests\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.common.exceptions import NoSuchWindowException\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.common.exceptions import StaleElementReferenceException\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-15T09:57:47.331755Z",
     "start_time": "2024-03-15T09:57:47.318582Z"
    }
   },
   "id": "1ed82030eecce787",
   "execution_count": 8
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Selenium Web Driver Setup"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "64531aa1eb09a113"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('--start-maximized')  # starts the browser in maximized window\n",
    "\n",
    "options.add_argument('--disable-extensions')  # disables browser extensions\n",
    "options.add_argument('--disable-gpu')  # disables hardware acceleration\n",
    "driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()),options=options)\n"
   ],
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-03-15T08:38:12.787507Z",
     "start_time": "2024-03-15T08:38:07.602482Z"
    }
   },
   "id": "initial_id",
   "execution_count": 2
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Definition Of Element Lists"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "1be7568667d0a16c"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "\n",
    "product_links = []\n",
    "product_names = []\n",
    "reviews = []\n",
    "ids=[]\n",
    "product_ratings=[]\n",
    "product_categories=[]"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c2de883479046a51",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Reforming a Dataframe"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "695f83670d681ecd"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df = pd.DataFrame(columns=(\"product-id\",\"product-name\",\"product-links\",\"product-category\",\"product-reviews\",\"product-rating\"))\n"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "6fe08b83a63ebbb5",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Collecting Product ID numbers\n",
    "\n",
    "Define the categories that you need to scrap reviews. You can add links sorted by reviews, so this model can get more reviews per second.\n",
    "You can change number of pages\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "61f67ec47e9dea69"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "websites=(\"https://www.n11.com/bebek-bezi-ve-islak-mendil?srt=REVIEWS&pg=\",\"https://www.n11.com/bilgisayar/bilgisayar-bilesenleri/islemci?pg=\")\n",
    "for a in websites:\n",
    "    for i in range(1, 5):\n",
    "        try:\n",
    "            website= a\n",
    "            website += str(i)\n",
    "            driver.get(website)\n",
    "            website = a\n",
    "            names = driver.find_elements(By.CLASS_NAME, 'productName')\n",
    "            time.sleep(1)\n",
    "            for name in names:\n",
    "                product_names.append(name.text)\n",
    "            # Wait for the columns to load\n",
    "                wait = WebDriverWait(driver, 10)\n",
    "                columns = wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'column')))\n",
    "\n",
    "            for column in columns:\n",
    "                \n",
    "                column_contents = column.find_elements(By.CLASS_NAME, 'columnContent')\n",
    "                \n",
    "                for content in column_contents:\n",
    "                    \n",
    "                    id_element = content.find_element(By.TAG_NAME, \"a\")\n",
    "                    id = id_element.get_attribute(\"data-id\")\n",
    "                    product_link = id_element.get_attribute(\"href\")\n",
    "                    try:\n",
    "                        category = driver.find_element(By.XPATH, '//*[@id=\"breadCrumb\"]/ul/li[3]/a')\n",
    "                    except NoSuchElementException:\n",
    "                        category =driver.find_element(By.XPATH, '//*[@id=\"breadCrumb\"]/ul/li[2]/a')\n",
    "\n",
    "                    if id is not None:\n",
    "\n",
    "                        ids.append(id)\n",
    "                        product_links.append(product_link)\n",
    "                        product_categories.append(category.text)\n",
    "                    else:\n",
    "                        continue\n",
    "\n",
    "\n",
    "        except TimeoutException:\n",
    "            print(\"TimeoutException error ignored.\")\n",
    "            continue\n",
    "    \n",
    "print(len(ids))"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "199e74cebf06c53e",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "8fc78d4df1474259",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Collecting Reviews"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "8b375290c7cf723b"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "for a in ids:\n",
    "    i=1\n",
    "    website = \"https://www.n11.com/component/render/groupProductReviews?page=\"+str(i)+\"&productId=\"+str(a)\n",
    "    driver.get(website)\n",
    "    time.sleep(0.1)\n",
    "    try:\n",
    "        try:\n",
    "            page= int(driver.find_element(By.XPATH, '//*[@id=\"reviewContent\"]/div/div/span').text)\n",
    "        except NoSuchElementException:\n",
    "            page=1\n",
    "        for i in range (1,page+1):\n",
    "            website = \"https://www.n11.com/component/render/groupProductReviews?page=\"+str(i)+\"&productId=\"+str(a)\n",
    "            driver.get(website)\n",
    "            step1 = driver.find_elements(By.CLASS_NAME,'comment')\n",
    "            for step in step1:\n",
    "                try:\n",
    "                    comment = step.find_element(By.TAG_NAME,'p').text\n",
    "                    reviews.append(comment)\n",
    "                    rating = step.find_element(By.CLASS_NAME,'commentTop').find_element(By.CLASS_NAME,'ratingCont').find_element(By.TAG_NAME,'span').get_attribute(\"class\")\n",
    "                    rating = int(rating.split(\"rating r\")[1])/20\n",
    "                    product_ratings.append(rating)\n",
    "                    df.loc[len(df)]=[a,product_names[ids.index(a)],product_links[ids.index(a)],product_categories[ids.index(a)],reviews[-1],product_ratings[-1]]\n",
    "                except IndexError:\n",
    "                    continue\n",
    "    except TimeoutException:\n",
    "        continue\n",
    "        "
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "9193e983b6dec03c",
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Check The Data"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "429f516040530145"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "len(reviews),len(product_ratings)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "a82c8214523df9ce",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df.head()"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "9eae282f3f791306",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df.info()"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "f7e6d2b51af9c9a5",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df[\"product-reviews\"].duplicated().value_counts()"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "751a9be44b3992",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df[\"product-rating\"].value_counts()"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "4f2ff32e28f8bc3b",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df[\"product-reviews\"]"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "baba6f29d28778eb",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "data = df.copy()\n",
    "data=data.drop_duplicates(\"product-reviews\")"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c9bbd622b20fa9ee",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "data.to_csv('data-dropped-duplicated-reviews4.csv')"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "6f271e2b7cdfbf15",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "data.info()"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "66d415082a977053",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "data[\"product-category\"].value_counts()"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "190d64452d00aeec",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "cfbd0560f83755e1",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "9b84a8b200566396",
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}