load OCF data to file-TEST 2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import requests\n",
    "import math\n",
    "import pprint\n",
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "from pandas.io.json import json_normalize\n",
    "from datetime import datetime, date, time\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = os.getcwd()\n",
    "print cwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pp = pprint.PrettyPrinter(indent=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### common functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findFromPathString(path_string, JSON, separator=\"/\", debug=False):     \n",
    "    paths_ = path_string.split(separator)\n",
    "    data = JSON\n",
    "    if debug :\n",
    "        print \"\\n data : \"\n",
    "        print pp.pprint(data)\n",
    "    try : \n",
    "        for i in range(0,len(paths_)):\n",
    "            data = data[paths_[i]]\n",
    "        return data\n",
    "    except :\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten_json(y):\n",
    "    out = {}\n",
    "\n",
    "    def flatten(x, name=''):\n",
    "        if type(x) is dict:\n",
    "            for a in x:\n",
    "                flatten(x[a], name + a + '_')\n",
    "        elif type(x) is list:\n",
    "            i = 0\n",
    "            for a in x:\n",
    "                flatten(a, name + str(i) + '_')\n",
    "                i += 1\n",
    "        else:\n",
    "            out[name[:-1]] = x\n",
    "\n",
    "    flatten(y)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "root_url_corpos = \"http://opencorporatefacts.fr/api/corporates\"\n",
    "# params_corpos = { 'page' : 1 , 'id' : None , 'CompanyNumber' : None }\n",
    "results_path_corpos = \"hydra:member\"\n",
    "output_filename_corpos = \"datasets/corporations/corpos\"\n",
    "headers_corpos = {'accept': 'application/ld+json'}\n",
    "\n",
    "root_url_accounts = \"http://opencorporatefacts.fr/api/compte_de_resultats\"\n",
    "# params_accounts = { 'page' : 1, 'Corporate' : None }\n",
    "results_path_accounts = \"hydra:member\"\n",
    "output_filename_accounts = \"datasets/account_results/accounts\"\n",
    "headers_accounts = {'accept': 'application/ld+json'}\n",
    "\n",
    "root_url_companies = \"http://api.cquest.org/company/\" ### pass CompanyNumber after root_url\n",
    "output_filename_companies = \"datasets/companies_data/companies\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print pp.pprint(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### main requests functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_filename( output_filename, page_treshold, split_files_treshold_pages, max_page, only_filename=False ) :\n",
    "    \n",
    "    start_page = page_treshold\n",
    "    end_page = page_treshold + split_files_treshold_pages - 1\n",
    "    if end_page > max_page : \n",
    "        end_page = max_page\n",
    "    \n",
    "    if only_filename : \n",
    "        prefix = ''\n",
    "    else : \n",
    "        prefix = cwd + '/'\n",
    "        \n",
    "    filename = prefix + output_filename + '-pageStart_' + str(start_page) + '_to_' + str(end_page)\n",
    "    \n",
    "    return filename\n",
    "    \n",
    "def get_req_results ( \n",
    "    url, \n",
    "    pages_range, \n",
    "    number_of_pages, \n",
    "    results_path, \n",
    "    output_filename, \n",
    "    start_page=1, \n",
    "    split_files_treshold_pages=1000,\n",
    "    debug=False, \n",
    "    out_formats=[\"csv\"] \n",
    "    ) : \n",
    "    ''' \n",
    "    retrieve results from requests \n",
    "    and write csv \n",
    "    works with API platform only for now\n",
    "    '''\n",
    "    \n",
    "    print \"url : {}\\n\".format(url)\n",
    "    \n",
    "    start_time = datetime.now()\n",
    "    \n",
    "    ### count pages\n",
    "    count = 0\n",
    "    page_treshold = 0\n",
    "    \n",
    "    ### requests and writing file\n",
    "    \n",
    "    for page in pages_range : \n",
    "    \n",
    "        if page > 0 and page >= start_page  : \n",
    "            \n",
    "            ### treshold and split into several files\n",
    "            if page % split_files_treshold_pages == 0 :\n",
    "                page_treshold = page\n",
    "                count = 0\n",
    "                \n",
    "            ### output files\n",
    "            if \"csv\" in out_formats :\n",
    "                file_data_csv_filename  = generate_filename( output_filename, page_treshold, split_files_treshold_pages, number_of_pages )\n",
    "                file_data_csv  = file_data_csv_filename + '.csv'\n",
    "\n",
    "            if \"json\" in out_formats : \n",
    "                file_data_json_filename = generate_filename( output_filename, page_treshold, split_files_treshold_pages, number_of_pages )\n",
    "                file_data_json = file_data_json_filename + '.json'\n",
    "            \n",
    "            if debug == False : \n",
    "                clear_output( wait=True ) \n",
    "            \n",
    "            resp = requests.get( url, params = { 'page' : page } ).json()\n",
    "            results = findFromPathString( results_path, resp, debug=debug )\n",
    "            \n",
    "            page_time = datetime.now()\n",
    "            delta_time = page_time - start_time\n",
    "            \n",
    "            filename = generate_filename( output_filename, page_treshold, split_files_treshold_pages, number_of_pages, only_filename=True )\n",
    "            \n",
    "            print \"\\nPage number : {} / {} \\\n",
    "                    \\ndelta_time (after request) : {}\\\n",
    "                    \\npage_treshold : {}\\\n",
    "                    \\nfilename : {}\\\n",
    "                    \\n\".format(page, number_of_pages, delta_time, page_treshold, filename)\n",
    "            \n",
    "            if results : \n",
    "                # loop results\n",
    "                for data in results : \n",
    "\n",
    "                    if debug : \n",
    "                        # print pp.pprint(data) \n",
    "                        print data\n",
    "                        print\n",
    "                    data_flat = flatten_json(data)\n",
    "                    # print data_flat\n",
    "\n",
    "                    ### CSV\n",
    "                    if \"csv\" in out_formats : \n",
    "\n",
    "                        ### creates headers\n",
    "                        if count == 0:\n",
    "                            with open(file_data_csv, 'w') as csv_file :\n",
    "                                csv_writer = csv.writer(csv_file, delimiter=';')\n",
    "                                header = data.keys()\n",
    "                                csv_writer.writerow(header)\n",
    "                                count += 1\n",
    "                        else : \n",
    "                            with open(file_data_csv, 'a') as csv_file :\n",
    "                                csv_writer = csv.writer(csv_file, delimiter=';')\n",
    "                                csv_writer.writerow( data.values() )\n",
    "                                count += 1\n",
    "\n",
    "\n",
    "                    ### JSON\n",
    "                    if \"json\" in out_formats : \n",
    "                        if count == 0 :\n",
    "                            with open(file_data_json, 'w') as json_file :\n",
    "                                json_file.write(json.dumps( [data] ).encode())         #If empty, write an array\n",
    "                                count += 1\n",
    "                        else : \n",
    "                            with open(file_data_json, 'ab+') as json_file :\n",
    "                                json_file.seek(-1,2)           \n",
    "                                json_file.truncate()                           #Remove the last character, open the array\n",
    "                                json_file.write(' , '.encode())                #Write the separator\n",
    "                                json_file.write(json.dumps(data).encode())     #Dump the dictionary\n",
    "                                json_file.write(']'.encode())                  #Close the array\n",
    "                                count += 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### getting data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "### TEST / get accounts \n",
    "number_of_pages_accounts_test = 30\n",
    "pages_range_test = range(number_of_pages_accounts_test)\n",
    "get_req_results( \n",
    "    root_url_accounts, \n",
    "    pages_range_test, \n",
    "    number_of_pages_accounts_test,\n",
    "    results_path_accounts, \n",
    "    output_filename_accounts+'_test', \n",
    "    start_page=1, \n",
    "    split_files_treshold_pages=10,\n",
    "    debug=True, \n",
    "    out_formats=[\"json\"] \n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "### get corpos data\n",
    "number_of_results_corpos = 577297\n",
    "results_per_page = 30\n",
    "number_of_pages_corpos = int( math.ceil( number_of_results_corpos / results_per_page ) )\n",
    "print number_of_pages_corpos\n",
    "pages_to_scrap_corpos = range( number_of_pages_corpos + 1 )\n",
    "print pages_to_scrap_corpos[:3]\n",
    "\n",
    "start_page_corpos = 1\n",
    "file_suffix_corpos = '_06'\n",
    "file_treshold_corpos = 2500\n",
    "\n",
    "get_req_results( \n",
    "    root_url_corpos, \n",
    "    pages_to_scrap_corpos, \n",
    "    number_of_pages_corpos,\n",
    "    results_path_corpos, \n",
    "    output_filename_corpos+file_suffix_corpos, \n",
    "    start_page=start_page_corpos, \n",
    "    split_files_treshold_pages=file_treshold_corpos,\n",
    "    debug=False, \n",
    "    out_formats=[\"json\"] \n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### get accounts data\n",
    "number_of_results_account = 316182\n",
    "results_per_page = 30\n",
    "number_of_pages_accounts = int( math.ceil( number_of_results_account / results_per_page ) )\n",
    "print number_of_pages_accounts\n",
    "pages_to_scrap_accounts = range( number_of_pages_accounts + 1 )\n",
    "print pages_to_scrap_accounts[:3]\n",
    "\n",
    "start_page_accounts = 1\n",
    "file_suffix_accounts = '_06'\n",
    "file_treshold_accounts = 500\n",
    "\n",
    "get_req_results( \n",
    "    root_url_accounts, \n",
    "    pages_to_scrap_accounts, \n",
    "    number_of_pages_accounts,\n",
    "    results_path_accounts, \n",
    "    output_filename_accounts+file_suffix_accounts, \n",
    "    start_page=start_page_accounts, \n",
    "    split_files_treshold_pages=file_treshold_accounts,\n",
    "    debug=False, \n",
    "    out_formats=[\"json\"] \n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}