Notebook_4_Merged_School_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Notebook_4_Merged_School_data.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "XXK_1Imc4Px5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "from glob import glob  #glob for merging data with similar names"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3cBtGuPa-bFj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filename = glob('/content/drive/My Drive/data_s/data_s_*.csv') #all files with pattern data_s_* "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tAnY5mLi_wgr",
        "colab_type": "code",
        "outputId": "d292c09c-cc8d-4c88-f78b-df621e59224d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "dfs = [] #create empty list\n",
        "for f in filename: #iterate over filename\n",
        "    dfs.append(pd.read_csv(f))  #read and append all the files\n",
        "len(dfs)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q7grNOwa_V7s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "up_school_data = pd.concat(dfs, ignore_index=True)  #concat all files"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ys9rCrIAGNg4",
        "colab_type": "code",
        "outputId": "155661eb-7445-4654-e292-29ae3a9f2b8c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 496
        }
      },
      "source": [
        "up_school_data.head()  #check the head"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Instruction_Medium</th>\n",
              "      <th>Male_Teachers</th>\n",
              "      <th>Pre_Primary_Sectin_Avilable</th>\n",
              "      <th>Board_for_Class_10th</th>\n",
              "      <th>School_Type</th>\n",
              "      <th>Classes</th>\n",
              "      <th>Female_Teacher</th>\n",
              "      <th>Pre_Primary_Teachers</th>\n",
              "      <th>Board_for_Class_10+2</th>\n",
              "      <th>Meal_Not_Applicable</th>\n",
              "      <th>Establishment</th>\n",
              "      <th>School_Area</th>\n",
              "      <th>School_Shifted_to_New_Place</th>\n",
              "      <th>Head_Teachers</th>\n",
              "      <th>Head_Teacher</th>\n",
              "      <th>Is_School_Residential</th>\n",
              "      <th>Residential_Type</th>\n",
              "      <th>Total_Teachers</th>\n",
              "      <th>Contract_Teachers</th>\n",
              "      <th>Management</th>\n",
              "      <th>Village_/_Town</th>\n",
              "      <th>Cluster</th>\n",
              "      <th>Block</th>\n",
              "      <th>District</th>\n",
              "      <th>State</th>\n",
              "      <th>UDISE_Code</th>\n",
              "      <th>Building</th>\n",
              "      <th>Class_Rooms</th>\n",
              "      <th>Boys_Toilet</th>\n",
              "      <th>Girls_Toilet</th>\n",
              "      <th>Computer_Aided_Learning</th>\n",
              "      <th>Electricity</th>\n",
              "      <th>Wall</th>\n",
              "      <th>Library</th>\n",
              "      <th>Playground</th>\n",
              "      <th>Books_in_Library</th>\n",
              "      <th>Drinking_Water</th>\n",
              "      <th>Ramps_for_Disable</th>\n",
              "      <th>Computers</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Hindi</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Co-educational</td>\n",
              "      <td>From Class 6 to Class 10</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Meal</td>\n",
              "      <td>2013</td>\n",
              "      <td>Rural</td>\n",
              "      <td>No</td>\n",
              "      <td>0</td>\n",
              "      <td></td>\n",
              "      <td>Yes</td>\n",
              "      <td>Private</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>Pvt. Unaided</td>\n",
              "      <td>Bhilawati</td>\n",
              "      <td>Town Area</td>\n",
              "      <td>Acchnera</td>\n",
              "      <td>Agra</td>\n",
              "      <td>Uttar Pradesh</td>\n",
              "      <td>9150205603</td>\n",
              "      <td>Private</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Pucca</td>\n",
              "      <td>Yes</td>\n",
              "      <td>No</td>\n",
              "      <td>1</td>\n",
              "      <td>Hand Pumps</td>\n",
              "      <td>Yes</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Hindi</td>\n",
              "      <td>2</td>\n",
              "      <td>No</td>\n",
              "      <td>State Board</td>\n",
              "      <td>Co-educational</td>\n",
              "      <td>From Class 6 to Class 10</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Meal</td>\n",
              "      <td>2013</td>\n",
              "      <td>Rural</td>\n",
              "      <td>No</td>\n",
              "      <td>0</td>\n",
              "      <td></td>\n",
              "      <td>Yes</td>\n",
              "      <td>Private</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>Pvt. Unaided</td>\n",
              "      <td>Gopau</td>\n",
              "      <td>Town Area</td>\n",
              "      <td>Acchnera</td>\n",
              "      <td>Agra</td>\n",
              "      <td>Uttar Pradesh</td>\n",
              "      <td>9150202203</td>\n",
              "      <td>Private</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Pucca</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>1</td>\n",
              "      <td>Hand Pumps</td>\n",
              "      <td>Yes</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>Hindi</td>\n",
              "      <td>5</td>\n",
              "      <td>No</td>\n",
              "      <td>Others</td>\n",
              "      <td>Co-educational</td>\n",
              "      <td>From Class 6 to Class 8</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>Others</td>\n",
              "      <td>Meal</td>\n",
              "      <td>2006</td>\n",
              "      <td>Rural</td>\n",
              "      <td>No</td>\n",
              "      <td>0</td>\n",
              "      <td></td>\n",
              "      <td>No</td>\n",
              "      <td>NaN</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>Department of Education</td>\n",
              "      <td>Chhai Pokhar</td>\n",
              "      <td>Arrua Khash</td>\n",
              "      <td>Acchnera</td>\n",
              "      <td>Agra</td>\n",
              "      <td>Uttar Pradesh</td>\n",
              "      <td>9150201502</td>\n",
              "      <td>Government</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>Pucca</td>\n",
              "      <td>No</td>\n",
              "      <td>Yes</td>\n",
              "      <td>0</td>\n",
              "      <td>Hand Pumps</td>\n",
              "      <td>Yes</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Hindi</td>\n",
              "      <td>1</td>\n",
              "      <td>No</td>\n",
              "      <td>Others</td>\n",
              "      <td>Co-educational</td>\n",
              "      <td>From Class 1 to Class 5</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>Others</td>\n",
              "      <td>Meal</td>\n",
              "      <td>1949</td>\n",
              "      <td>Rural</td>\n",
              "      <td>No</td>\n",
              "      <td>1</td>\n",
              "      <td>rajni jayant</td>\n",
              "      <td>No</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>Department of Education</td>\n",
              "      <td>Ardaya</td>\n",
              "      <td>Arrua Khash</td>\n",
              "      <td>Acchnera</td>\n",
              "      <td>Agra</td>\n",
              "      <td>Uttar Pradesh</td>\n",
              "      <td>9150201401</td>\n",
              "      <td>Government</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>No</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Pucca</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>300</td>\n",
              "      <td>Hand Pumps</td>\n",
              "      <td>Yes</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>Hindi</td>\n",
              "      <td>0</td>\n",
              "      <td>No</td>\n",
              "      <td>Others</td>\n",
              "      <td>Co-educational</td>\n",
              "      <td>From Class 1 to Class 5</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>Others</td>\n",
              "      <td>Meal</td>\n",
              "      <td>1906</td>\n",
              "      <td>Urban</td>\n",
              "      <td>No</td>\n",
              "      <td>0</td>\n",
              "      <td></td>\n",
              "      <td>No</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>Department of Education</td>\n",
              "      <td>Achnera</td>\n",
              "      <td>Arrua Khash</td>\n",
              "      <td>Acchnera</td>\n",
              "      <td>Agra</td>\n",
              "      <td>Uttar Pradesh</td>\n",
              "      <td>9150200102</td>\n",
              "      <td>Government</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>No</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Pucca</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>45</td>\n",
              "      <td>Hand Pumps</td>\n",
              "      <td>Yes</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0 Instruction_Medium  ...  Ramps_for_Disable Computers\n",
              "0           0              Hindi  ...                Yes         1\n",
              "1           1              Hindi  ...                Yes         1\n",
              "2           2              Hindi  ...                Yes         1\n",
              "3           3              Hindi  ...                Yes         0\n",
              "4           4              Hindi  ...                Yes         0\n",
              "\n",
              "[5 rows x 40 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ey-8rdf4VNM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "up_school_data.drop('Unnamed: 0',axis=1,inplace=True) #drop unnamed: 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRF_VkKOAWi3",
        "colab_type": "code",
        "outputId": "25a0748f-8690-4ff6-d68e-c0472e5e501c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "up_school_data.shape #check the shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(234276, 39)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GewdWRDJAit9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#save the data\n",
        "up_school_data.to_csv('up_school_data')\n",
        "!cp up_school_data '/content/drive/My Drive/data_s'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LizaEmHyt4Hs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}