thredds_crawler_dea_s2beta.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from thredds_crawler.crawl import Crawl\n",
    "from pathlib import Path, PureWindowsPath, PurePath\n",
    "import geopandas as gpd\n",
    "import kml2geojson\n",
    "import geopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileserver = 'http://dapds00.nci.org.au/thredds/fileServer/'\n",
    "catalog = 'http://dapds00.nci.org.au/thredds/catalog/if87/catalog.xml'\n",
    "# Below for testing - comment out for expected execution\n",
    "catalog = 'http://dapds00.nci.org.au/thredds/catalog/if87/2015-07-12/catalog.xml'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "149.1012676 -35.2975906\n"
     ]
    }
   ],
   "source": [
    "# Use geopy to get latitude and longitude for intersecting with the MGRS tile polygons\n",
    "from geopy.geocoders import Nominatim\n",
    "geolocator = Nominatim(user_agent=\"useragent\")\n",
    "location = geolocator.geocode(\"Canberra\")\n",
    "print(location.longitude, location.latitude)\n",
    "westlon = location.longitude - 0.01\n",
    "southlat = location.latitude - 0.01\n",
    "eastlon = location.longitude + 0.01\n",
    "northlat = location.latitude + 0.01\n",
    "bbox = (westlon, southlat, eastlon, northlat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Or - define bounding box\n",
    "westlon = 147\n",
    "southlat = -38\n",
    "eastlon = 154\n",
    "northlat = -27\n",
    "bbox = (westlon, southlat, eastlon, northlat)\n",
    "# TODO - implement time selection\n",
    "# start_date = '2019-12-15'\n",
    "# end_date = '2020-01-15'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the MGRS reference tile polygons from ESA\n",
    "wget.download('https://sentinel.esa.int/documents/247904/1955685/S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Geopandas doesn't support direct read of KML so convert to GeoJSON\n",
    "kml2geojson.convert('S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml', '.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load MGRS to geopandas using the bbox\n",
    "mgrs_gdf = gpd.read_file('S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.geojson', bbox=bbox)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mgrs_gdf only contains the intersecting geometry so dump a list of mgrs tile references\n",
    "mgrs_tile_list = list(mgrs_gdf.name)\n",
    "# for testing - comment out below for actual run\n",
    "mgrs_tile_list = ['52JCM']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use THREDDS crawler to get a file listing based on mgrs_tile_list\n",
    "filelist = []\n",
    "for i in mgrs_tile_list:\n",
    "    c = Crawl(catalog, select=[\".*\"+i+\"*/*\"])\n",
    "    for dataset in c.datasets:\n",
    "        filelist.append([fileserver+dataset.id, './'+dataset.id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start downloading\n",
    "filelist\n",
    "for remotefile in filelist:\n",
    "    print(remotefile[0])\n",
    "    Path(remotefile[1]).parent.mkdir(parents=True, exist_ok=True)\n",
    "    localfile = str(PureWindowsPath(Path(remotefile[1]).absolute()))\n",
    "    wget.download(remotefile[0], out=localfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Path('./if87/2015-07-12/S2A_OPER_MSI_ARD_TL_EPA__20160810T192637_A000271_T52JCM_N02.04/NBAR/S2A_OPER_MSI_ARD_TL_EPA__20160810T192637_A000271_T52JCM_NBAR_B01.TIF').parent.mkdir(parents=True, exist_ok=True)\n",
    "#wget.download('http://dapds00.nci.org.au/thredds/fileServer/if87/2015-07-12/S2A_OPER_MSI_ARD_TL_EPA__20160810T192637_A000271_T52JCM_N02.04/NBAR/S2A_OPER_MSI_ARD_TL_EPA__20160810T192637_A000271_T52JCM_NBAR_B01.TIF', out=str(PureWindowsPath(Path('./if87/2015-07-12/S2A_OPER_MSI_ARD_TL_EPA__20160810T192637_A000271_T52JCM_N02.04/NBAR/S2A_OPER_MSI_ARD_TL_EPA__20160810T192637_A000271_T52JCM_NBAR_B01.TIF').parent)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}