split_big_json_files.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### split big json files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import json\n",
    "import os\n",
    "import pprint\n",
    "import math\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/jpy/Dropbox/_FLASK/__JUPYTER__/OCF\n"
     ]
    }
   ],
   "source": [
    "cwd = os.getcwd()\n",
    "print cwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "pp = pprint.PrettyPrinter(indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "accounts_folder_path = 'datasets/account_results'\n",
    "accounts_file_prefix = 'accounts'\n",
    "files_names_accounts = [\n",
    "    { 'filename' : 'accounts_04-pageStart_0_to_1000'   , 'startP' : 0,    'endP' : 1000 }, \n",
    "    { 'filename' : 'accounts_04-pageStart_1000_to_2000', 'startP' : 1000, 'endP' : 2000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_2000_to_3000', 'startP' : 2000, 'endP' : 3000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_3000_to_4000', 'startP' : 3000, 'endP' : 4000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_4000_to_5000', 'startP' : 4000, 'endP' : 5000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_5000_to_6000', 'startP' : 5000, 'endP' : 6000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_6000_to_7000', 'startP' : 6000, 'endP' : 7000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_7000_to_8000', 'startP' : 7000, 'endP' : 8000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_8000_to_9000', 'startP' : 8000, 'endP' : 9000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_9000_to_10000',  'startP' : 9000,  'endP' : 10000 },\n",
    "    { 'filename' : 'accounts_04-pageStart_10000_to_11000', 'startP' : 10000, 'endP' : 11000 },\n",
    "]\n",
    "files_extension_accounts = '.json'\n",
    "\n",
    "output_filename_accounts = accounts_folder_path + '/' + accounts_file_prefix\n",
    "output_filename_batch_number = '_08'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number_of_pages_accounts : 10539\n"
     ]
    }
   ],
   "source": [
    "number_of_results_account = 316182\n",
    "results_per_page = 30\n",
    "number_of_pages_accounts = int( math.ceil( number_of_results_account / results_per_page ) )\n",
    "print 'number_of_pages_accounts : {}'.format(number_of_pages_accounts)\n",
    "pages_to_scrap_accounts = range( number_of_pages_accounts + 1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpos_folder_path = 'datasets/corporations'\n",
    "corpos_file_prefix = 'corpos'\n",
    "files_names_corpos = [\n",
    "    { 'filename' : 'corpos_07-pageStart_0_to_2499'   , 'startP' : 0,    'endP' : 2500 }, \n",
    "    { 'filename' : 'corpos_07-pageStart_2500_to_4999', 'startP' : 2500, 'endP' : 5000 },\n",
    "    { 'filename' : 'corpos_07-pageStart_5000_to_7499', 'startP' : 5000, 'endP' : 7500 },\n",
    "    { 'filename' : 'corpos_07-pageStart_7500_to_9999', 'startP' : 7500, 'endP' : 10000 },\n",
    "    { 'filename' : 'corpos_07-pageStart_10000_to_12499', 'startP' : 10000, 'endP' : 12500 },\n",
    "    { 'filename' : 'corpos_07-pageStart_12500_to_14999', 'startP' : 12500, 'endP' : 15000 },\n",
    "    { 'filename' : 'corpos_07-pageStart_15000_to_17499', 'startP' : 15000, 'endP' : 17500 },\n",
    "    { 'filename' : 'corpos_07-pageStart_17500_to_19243', 'startP' : 17500, 'endP' : 19244 },\n",
    "\n",
    "]\n",
    "files_extension_corpos = '.json'\n",
    "\n",
    "output_filename_corpos = corpos_folder_path + '/' + corpos_file_prefix\n",
    "output_filename_batch_number_corpos = '_08'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number_of_pages_corpos : 19243\n"
     ]
    }
   ],
   "source": [
    "number_of_results_corpos = 577297\n",
    "results_per_page_corpos = 30\n",
    "number_of_pages_corpos = int( math.ceil( number_of_results_corpos / results_per_page_corpos ) )\n",
    "print 'number_of_pages_corpos : {}'.format(number_of_pages_corpos)\n",
    "pages_to_scrap_corpos = range( number_of_pages_corpos + 1 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_filename( output_filename, start_page, end_page, only_filename=False, extension='.json' ) :\n",
    "    \n",
    "    if only_filename : \n",
    "        prefix = ''\n",
    "    else : \n",
    "        prefix = cwd + '/'\n",
    "        \n",
    "    filename = prefix + output_filename + '-pageStart_' + str(start_page) + '_to_' + str(end_page) + extension\n",
    "    \n",
    "    return filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_list(a_list, dont_split=False ):\n",
    "    \n",
    "    half = len(a_list)//2\n",
    "    \n",
    "    if dont_split == False :\n",
    "        return a_list[:half], a_list[half:]\n",
    "    \n",
    "    else : \n",
    "        return a_list, []\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### read files and reexport"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def splitFiles (\n",
    "        files_names_list, \n",
    "        folder_path, \n",
    "        output_filename_batch_number, \n",
    "        output_filename_path, \n",
    "        extension='.json'\n",
    "    ) :\n",
    "\n",
    "    for file_ in files_names_list : \n",
    "        \n",
    "        print 'file_[\"filename\"] : {}'.format(file_['filename'])\n",
    "        filepath = cwd + '/' + folder_path + '/' + file_['filename'] + extension\n",
    "        \n",
    "        print filepath\n",
    "        print\n",
    "        \n",
    "        startP = file_['startP']\n",
    "        endP   = file_['endP']\n",
    "        halfP  = (file_['startP'] + file_['endP']) / 2\n",
    "\n",
    "        with open(filepath) as json_file:\n",
    "            data = json.load(json_file)\n",
    "            # print 'data[:1] : ', data[:1]\n",
    "\n",
    "            data_A, data_B = split_list(data)\n",
    "            \n",
    "            data_splitted = [\n",
    "                { 'position' : 'dataA', 'data' : data_A, 'startP' : startP, 'endP' : halfP-1 },\n",
    "                { 'position' : 'dataB', 'data' : data_B, 'startP' : halfP,  'endP' : endP-1 }\n",
    "            ]\n",
    "            \n",
    "            for data_split in data_splitted : \n",
    "                \n",
    "                print \"position : {} / startP : {}\".format(data_split['position'], data_split['startP'])\n",
    "                \n",
    "                output_filename = output_filename_path + output_filename_batch_number\n",
    "                file_data_json = generate_filename( output_filename, data_split['startP'], data_split['endP'])\n",
    "                print file_data_json\n",
    "                \n",
    "                with open(file_data_json, 'w') as json_file :\n",
    "                    json_file.write(json.dumps( data_split['data'] ).encode())\n",
    "                    print '...saved'\n",
    "                    \n",
    "                print\n",
    "                \n",
    "                \n",
    "                \n",
    "            clear_output(wait=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file_[\"filename\"] : accounts_04-pageStart_10000_to_11000\n",
      "\n",
      "position : dataA / startP : 10000\n",
      "/Users/jpy/Dropbox/_FLASK/__JUPYTER__/OCF/datasets/account_results/accounts_07-pageStart_10000_to_10499.json\n",
      "...saved\n",
      "\n",
      "position : dataB / startP : 10500\n",
      "/Users/jpy/Dropbox/_FLASK/__JUPYTER__/OCF/datasets/account_results/accounts_07-pageStart_10500_to_10999.json\n",
      "...saved\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results_accounts = splitFiles(\n",
    "    files_names_accounts, \n",
    "    accounts_folder_path, \n",
    "    output_filename_batch_number_accounts,\n",
    "    output_filename_accounts\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file_[\"filename\"] : corpos_07-pageStart_17500_to_19243\n",
      "/Users/jpy/Dropbox/_FLASK/__JUPYTER__/OCF/datasets/corporations/corpos_07-pageStart_17500_to_19243.json\n",
      "\n",
      "position : dataA / startP : 17500\n",
      "/Users/jpy/Dropbox/_FLASK/__JUPYTER__/OCF/datasets/corporations/corpos_08-pageStart_17500_to_18371.json\n",
      "...saved\n",
      "\n",
      "position : dataB / startP : 18372\n",
      "/Users/jpy/Dropbox/_FLASK/__JUPYTER__/OCF/datasets/corporations/corpos_08-pageStart_18372_to_19243.json\n",
      "...saved\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results_accounts = splitFiles(\n",
    "    files_names_corpos, \n",
    "    corpos_folder_path, \n",
    "    output_filename_batch_number_corpos,\n",
    "    output_filename_corpos\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}