Notebook_2_School_links.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Notebook_2_School_links.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "yJYIpT2-4x_s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#import all the necessary libraries\n",
        "from requests import get\n",
        "from bs4 import BeautifulSoup\n",
        "from IPython.core.display import clear_output\n",
        "import pandas as pd\n",
        "import sys"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3VUASh0oC3K-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cluster_links = pd.read_csv('/content/drive/My Drive/data_s/data_c.csv') #read all the cluster links stored in csv file"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YqP_wTRBDW3e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "links = cluster_links['links']  #assign cluster links to links"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o8pmVvV7q79j",
        "colab_type": "text"
      },
      "source": [
        "###Schools Links"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "04hBv1H5AvDr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#define function fetch url from previous notebook to fetch all <a> tags\n",
        "\n",
        "def fetch_url(url):\n",
        "  \n",
        "  response = get(url)\n",
        "  html = response.text\n",
        "  soup = BeautifulSoup(html,'html.parser')\n",
        "  div_class = soup.find_all('div', 'my-3 p-3 bg-white rounded shadow-sm')\n",
        "  lst = div_class[1]\n",
        "  link_tag = lst.table.find_all('a') \n",
        "  return link_tag"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vXVd9YIStKAE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "base_url = 'https://schools.org.in/'  #base url\n",
        "\n",
        "from time import time  #import time\n",
        "from time import sleep #import randint\n",
        "from random import randint #import random\n",
        "from warnings import warn #import warn \n",
        "warn(\"Warning Simulation\") #warn simulation warnings\n",
        "\n",
        "start_time = time() #start time\n",
        "requests = 0  #initialize requests to 0\n",
        "\n",
        "school_links = []  #create a empty school links\n",
        "\n",
        "for i in links: #iterate over all the cluster links\n",
        "  uttar_p_skl = fetch_url(i)  #fecth all the <a> tags\n",
        "  \n",
        "  #monitor the request\n",
        "  requests +=1 #update request by 1 unit after each iteration\n",
        "  elapsed_time = time() - start_time  #count the elapsed time\n",
        "  print('Request:{}; Frequency: {} requests/s'.format(requests, requests/elapsed_time))  #count the frequency of requests\n",
        "  clear_output(wait = True) #clear output after each requests\n",
        "  \n",
        "  for tag in uttar_p_skl: #iterate over all the <a> tags\n",
        "    school_links.append(base_url + tag.get('href')) #append the school_links with the school links\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "haZoYKmJkedv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(school_links)  #check the length of school links"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OQEhl9KN5ZWV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#create a DataFrame for school links\n",
        "data = pd.DataFrame({'links' : school_links})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iivw57Gv5Znn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#convert the dataframe to csv and save it in drive\n",
        "data.to_csv('school_links_8__.csv')\n",
        "!cp school_links_8__.csv '/content/drive/My Drive/data_s'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VPCtN8hD5Z1m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}