From a560e19deb65a2ffd85e15306e5a8e99d7042831 Mon Sep 17 00:00:00 2001 From: spestana Date: Thu, 26 Sep 2024 17:10:39 +0000 Subject: [PATCH] Update download functions and add example notebook --- docs/examples/download.ipynb | 621 ++++++++++++++++++++++++++++++----- docs/index.rst | 1 + pyproject.toml | 1 + src/goes_ortho/get_data.py | 206 +++++++++--- 4 files changed, 705 insertions(+), 124 deletions(-) diff --git a/docs/examples/download.ipynb b/docs/examples/download.ipynb index 3e4fddf..ddc9aac 100644 --- a/docs/examples/download.ipynb +++ b/docs/examples/download.ipynb @@ -12,10 +12,21 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 1, "id": "5c0d77eb-457b-4b4d-a669-73778a54637e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/spestana/.conda/envs/goes-test-env/lib/python3.12/site-packages/goes2go/data.py:665: FutureWarning: 'H' is deprecated and will be removed in a future version. Please use 'h' instead of 'H'.\n", + " within=pd.to_timedelta(config[\"nearesttime\"].get(\"within\", \"1h\")),\n", + "/home/spestana/.conda/envs/goes-test-env/lib/python3.12/site-packages/goes2go/NEW.py:188: FutureWarning: 'H' is deprecated and will be removed in a future version. Please use 'h' instead of 'H'.\n", + " within=pd.to_timedelta(config[\"nearesttime\"].get(\"within\", \"1h\")),\n" + ] + } + ], "source": [ "import goes_ortho as go\n", "import xarray as xr" @@ -35,65 +46,35 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "6fc6c19c-f21f-4319-a45e-47434b933600", - "metadata": {}, - "outputs": [], - "source": [ - "lon = -119.31212\n", - "lat = 37.88175\n", - "z = 2811 # elevation in meters" - ] - }, - { - "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "id": "28208c5f-e93c-4ce9-b4f7-c4c7ea14e12e", "metadata": {}, "outputs": [], "source": [ - "startDatetime = \"2024-04-19T00:00:00Z\"\n", - "endDatetime = \"2024-04-19T00:59:00Z\"\n", - "[min_lon, min_lat, max_lon, max_lat] = [lon-1, lat-1, lon+1, lat+1] # set bounds to just a little area around the location we want\n", - "satellite = \"goes18\"\n", - "product = \"ABI-L1b-RadC\"\n", - "band = 2\n", - "variable = \"Rad\"\n", - "OPENTOPO_API_KEY = \"585b1d1639bc5ef8a4a5bdea7e45a8d1\"" + "# Make request file from user input\n", + "request_filepath = go.get_data.make_request_json(workflowName = \"example\",\n", + " startDatetime = \"2020-04-19T00:00:00Z\",\n", + " endDatetime = \"2020-04-21T00:59:00Z\",\n", + " bounds = go.get_data.bounds_from_geojson(\"grand_mesa.geojson\"),\n", + " satellite = \"goes16\",\n", + " product = \"ABI-L2-LSTC\",\n", + " band = 2,\n", + " variable = \"LST\",\n", + " apiKey = None, #585b1d1639bc5ef8a4a5bdea7e45a8d1\n", + " )" ] }, { "cell_type": "markdown", - "id": "8a4c97cb-b630-4771-8e80-4c7f70c0f1bf", + "id": "8eebc03e-2e78-44b8-beeb-e6914e454b2e", "metadata": {}, "source": [ - "Using the data above, we make the required json file." + "The functions below demonstrate downloading GOES imagery using two different downloader packages: [goes2go](https://goes2go.readthedocs.io/en/latest/) and [goespy](https://github.com/spestana/goes-py) (the goespy functions are now integrated directly within the goes-ortho package). I have found goes2go is typically faster." ] }, { "cell_type": "code", - "execution_count": 9, - "id": "cac7ae71-01db-4fa3-9e9f-856b82ff738b", - "metadata": {}, - "outputs": [], - "source": [ - "# Make request file from user input\n", - "go.get_data.make_request_json(\n", - " f\"{satellite}-example-ACMC\",\n", - " startDatetime,\n", - " endDatetime,\n", - " [min_lon, min_lat, max_lon, max_lat],\n", - " satellite,\n", - " product,\n", - " band,\n", - " variable,\n", - " OPENTOPO_API_KEY,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, + "execution_count": 3, "id": "9fbfaf9c-938a-475a-a1f7-f2a36b062505", "metadata": {}, "outputs": [ @@ -101,18 +82,277 @@ "name": "stdout", "output_type": "stream", "text": [ - "Estimated 1 batches to download\n", + "Estimated 17 batches to download\n", "Batch number 1\n", - "Download batch of imagery from 2024-04-19 00:00:00+00:00 to 2024-04-19 03:00:00+00:00\n", - "📦 Finished downloading [576] files to [/home/spestana/data/noaa-goes18/ABI-L1b-RadC].\n", - "Cropping image batch to [-120.31212, 36.88175, -118.31212, 38.88175]\n" + "Download batch of imagery from 2020-04-19 00:00:00+00:00 to 2020-04-19 03:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 576/576 [01:44<00:00, 5.52it/s]" + "100%|██████████| 3/3 [00:01<00:00, 2.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 2\n", + "Download batch of imagery from 2020-04-19 03:00:00+00:00 to 2020-04-19 06:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 10.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 3\n", + "Download batch of imagery from 2020-04-19 06:00:00+00:00 to 2020-04-19 09:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 10.80it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 4\n", + "Download batch of imagery from 2020-04-19 09:00:00+00:00 to 2020-04-19 12:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 10.74it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 5\n", + "Download batch of imagery from 2020-04-19 12:00:00+00:00 to 2020-04-19 15:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 10.90it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 6\n", + "Download batch of imagery from 2020-04-19 15:00:00+00:00 to 2020-04-19 18:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.84it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 7\n", + "Download batch of imagery from 2020-04-19 18:00:00+00:00 to 2020-04-19 21:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.98it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 8\n", + "Download batch of imagery from 2020-04-19 21:00:00+00:00 to 2020-04-20 00:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.92it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 9\n", + "Download batch of imagery from 2020-04-20 00:00:00+00:00 to 2020-04-20 03:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.67it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 10\n", + "Download batch of imagery from 2020-04-20 03:00:00+00:00 to 2020-04-20 06:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.58it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 11\n", + "Download batch of imagery from 2020-04-20 06:00:00+00:00 to 2020-04-20 09:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.43it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 12\n", + "Download batch of imagery from 2020-04-20 09:00:00+00:00 to 2020-04-20 12:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.46it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 13\n", + "Download batch of imagery from 2020-04-20 12:00:00+00:00 to 2020-04-20 15:00:00+00:00\n", + "FileNotFoundError encountered. The requested image may not exist. Because this searched a time window of 3 hours, there may be some valid imagery within the time window. Try a smaller time window to search for valid imagery.\n", + "noaa-goes16/ABI-L2-LSTC/2020/111/13\n", + "Batch number 14\n", + "Download batch of imagery from 2020-04-20 15:00:00+00:00 to 2020-04-20 18:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 15\n", + "Download batch of imagery from 2020-04-20 18:00:00+00:00 to 2020-04-20 21:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.70it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 16\n", + "Download batch of imagery from 2020-04-20 21:00:00+00:00 to 2020-04-21 00:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.65it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch number 17\n", + "Download batch of imagery from 2020-04-21 00:00:00+00:00 to 2020-04-21 03:00:00+00:00\n", + "📦 Finished downloading [3] files to [/home/spestana/data/noaa-goes16/ABI-L2-LSTC].\n", + "Cropping image batch to [-108.368202, 38.80429, -107.627676, 39.211234]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 3/3 [00:00<00:00, 11.43it/s]" ] }, { @@ -120,8 +360,8 @@ "output_type": "stream", "text": [ "Done\n", - "CPU times: user 1min 58s, sys: 22 s, total: 2min 20s\n", - "Wall time: 2min 50s\n" + "CPU times: user 5.94 s, sys: 863 ms, total: 6.8 s\n", + "Wall time: 22.3 s\n" ] }, { @@ -134,12 +374,12 @@ ], "source": [ "%%time\n", - "filepaths = go.get_data.download_abi_goes2go(f\"{satellite}-example-ACMC.json\")" + "filepaths = go.get_data.download_abi_goes2go(request_filepath)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 4, "id": "738b548c-8807-4fb8-82c2-eb380a5284c8", "metadata": {}, "outputs": [ @@ -149,45 +389,270 @@ "text": [ "\n", "Files will be downloaded and then cropped to these bounds:\n", - "\t(-120.31212,38.88175).\t.(-118.31212,38.88175)\n", + "\t(-108.368202,39.211234).\t.(-107.627676,39.211234)\n", "\n", "\n", "\n", - "\t(-120.31212,36.88175).\t.(-118.31212,36.88175)\n", + "\t(-108.368202,38.80429).\t.(-107.627676,38.80429)\n", "\n", "For each S3 bucket, download the corresponding observations\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02\n", - "Downloading... the product ABI-L1b-RadC \n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 00 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100001131_e20201100003504_c20201100004343.nc: [##########] 100% 1.701659 MB/1.701659 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/00\n", + "goes16/2020/4/19/ABI-L2-LSTC/00/OR_ABI-L2-LSTC-M6_G16_s20201100001131_e20201100003504_c20201100004343.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 01 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100101131_e20201100103504_c20201100104361.nc: [##########] 100% 1.762344 MB/1.762344 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/01\n", + "goes16/2020/4/19/ABI-L2-LSTC/01/OR_ABI-L2-LSTC-M6_G16_s20201100101131_e20201100103504_c20201100104361.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 02 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100201132_e20201100203504_c20201100204346.nc: [##########] 100% 1.827398 MB/1.827398 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/02\n", + "goes16/2020/4/19/ABI-L2-LSTC/02/OR_ABI-L2-LSTC-M6_G16_s20201100201132_e20201100203504_c20201100204346.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 03 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100301132_e20201100303504_c20201100304348.nc: [##########] 100% 1.893841 MB/1.893841 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/03\n", + "goes16/2020/4/19/ABI-L2-LSTC/03/OR_ABI-L2-LSTC-M6_G16_s20201100301132_e20201100303504_c20201100304348.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 04 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100401132_e20201100403504_c20201100404347.nc: [##########] 100% 1.912388 MB/1.912388 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/04\n", + "goes16/2020/4/19/ABI-L2-LSTC/04/OR_ABI-L2-LSTC-M6_G16_s20201100401132_e20201100403504_c20201100404347.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 05 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100501132_e20201100503504_c20201100504340.nc: [##########] 100% 1.948882 MB/1.948882 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/05\n", + "goes16/2020/4/19/ABI-L2-LSTC/05/OR_ABI-L2-LSTC-M6_G16_s20201100501132_e20201100503504_c20201100504340.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 06 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100601132_e20201100603505_c20201100604316.nc: [##########] 100% 1.973455 MB/1.973455 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/06\n", + "goes16/2020/4/19/ABI-L2-LSTC/06/OR_ABI-L2-LSTC-M6_G16_s20201100601132_e20201100603505_c20201100604316.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 07 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100701132_e20201100703505_c20201100704354.nc: [##########] 100% 1.979452 MB/1.979452 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/07\n", + "goes16/2020/4/19/ABI-L2-LSTC/07/OR_ABI-L2-LSTC-M6_G16_s20201100701132_e20201100703505_c20201100704354.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 08 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100801132_e20201100803505_c20201100804369.nc: [##########] 100% 2.035866 MB/2.035866 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/08\n", + "goes16/2020/4/19/ABI-L2-LSTC/08/OR_ABI-L2-LSTC-M6_G16_s20201100801132_e20201100803505_c20201100804369.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 09 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201100901132_e20201100903505_c20201100904378.nc: [##########] 100% 2.069201 MB/2.069201 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/09\n", + "goes16/2020/4/19/ABI-L2-LSTC/09/OR_ABI-L2-LSTC-M6_G16_s20201100901132_e20201100903505_c20201100904378.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 10 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101001132_e20201101003505_c20201101004356.nc: [##########] 100% 2.08223 MB/2.08223 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/10\n", + "goes16/2020/4/19/ABI-L2-LSTC/10/OR_ABI-L2-LSTC-M6_G16_s20201101001132_e20201101003505_c20201101004356.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 11 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101101132_e20201101103505_c20201101104373.nc: [##########] 100% 2.057548 MB/2.057548 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/11\n", + "goes16/2020/4/19/ABI-L2-LSTC/11/OR_ABI-L2-LSTC-M6_G16_s20201101101132_e20201101103505_c20201101104373.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 12 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101201132_e20201101203505_c20201101204347.nc: [##########] 100% 1.966647 MB/1.966647 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/12\n", + "goes16/2020/4/19/ABI-L2-LSTC/12/OR_ABI-L2-LSTC-M6_G16_s20201101201132_e20201101203505_c20201101204347.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 13 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101301132_e20201101303505_c20201101304353.nc: [##########] 100% 1.739812 MB/1.739812 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/13\n", + "goes16/2020/4/19/ABI-L2-LSTC/13/OR_ABI-L2-LSTC-M6_G16_s20201101301132_e20201101303505_c20201101304353.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 14 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101401132_e20201101403505_c20201101404378.nc: [##########] 100% 1.568007 MB/1.568007 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/14\n", + "goes16/2020/4/19/ABI-L2-LSTC/14/OR_ABI-L2-LSTC-M6_G16_s20201101401132_e20201101403505_c20201101404378.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 15 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101501132_e20201101503505_c20201101504372.nc: [##########] 100% 1.53152 MB/1.53152 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/15\n", + "goes16/2020/4/19/ABI-L2-LSTC/15/OR_ABI-L2-LSTC-M6_G16_s20201101501132_e20201101503505_c20201101504372.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 16 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101601132_e20201101603505_c20201101604343.nc: [##########] 100% 1.536471 MB/1.536471 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/16\n", + "goes16/2020/4/19/ABI-L2-LSTC/16/OR_ABI-L2-LSTC-M6_G16_s20201101601132_e20201101603505_c20201101604343.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 17 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101701132_e20201101703505_c20201101704397.nc: [##########] 100% 1.530618 MB/1.530618 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/17\n", + "goes16/2020/4/19/ABI-L2-LSTC/17/OR_ABI-L2-LSTC-M6_G16_s20201101701132_e20201101703505_c20201101704397.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 18 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101801132_e20201101803505_c20201101804367.nc: [##########] 100% 1.507082 MB/1.507082 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/18\n", + "goes16/2020/4/19/ABI-L2-LSTC/18/OR_ABI-L2-LSTC-M6_G16_s20201101801132_e20201101803505_c20201101804367.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 19 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201101901132_e20201101903505_c20201101904334.nc: [##########] 100% 1.496716 MB/1.496716 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/19\n", + "goes16/2020/4/19/ABI-L2-LSTC/19/OR_ABI-L2-LSTC-M6_G16_s20201101901132_e20201101903505_c20201101904334.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 20 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201102001132_e20201102003505_c20201102004371.nc: [##########] 100% 1.467092 MB/1.467092 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/20\n", + "goes16/2020/4/19/ABI-L2-LSTC/20/OR_ABI-L2-LSTC-M6_G16_s20201102001132_e20201102003505_c20201102004371.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 21 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201102101132_e20201102103505_c20201102104445.nc: [##########] 100% 1.447728 MB/1.447728 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/21\n", + "goes16/2020/4/19/ABI-L2-LSTC/21/OR_ABI-L2-LSTC-M6_G16_s20201102101132_e20201102103505_c20201102104445.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 22 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201102201132_e20201102203505_c20201102204366.nc: [##########] 100% 1.461739 MB/1.461739 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/22\n", + "goes16/2020/4/19/ABI-L2-LSTC/22/OR_ABI-L2-LSTC-M6_G16_s20201102201132_e20201102203505_c20201102204366.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 23 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201102301132_e20201102303505_c20201102304375.nc: [##########] 100% 1.526796 MB/1.526796 MB\n", + "Subsetting files in...goes16/2020/4/19/ABI-L2-LSTC/23\n", + "goes16/2020/4/19/ABI-L2-LSTC/23/OR_ABI-L2-LSTC-M6_G16_s20201102301132_e20201102303505_c20201102304375.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 00 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110001132_e20201110003505_c20201110004359.nc: [##########] 100% 1.601177 MB/1.601177 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/00\n", + "goes16/2020/4/20/ABI-L2-LSTC/00/OR_ABI-L2-LSTC-M6_G16_s20201110001132_e20201110003505_c20201110004359.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 01 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110101132_e20201110103505_c20201110104364.nc: [##########] 100% 1.73089 MB/1.73089 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/01\n", + "goes16/2020/4/20/ABI-L2-LSTC/01/OR_ABI-L2-LSTC-M6_G16_s20201110101132_e20201110103505_c20201110104364.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 02 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110201132_e20201110203505_c20201110204344.nc: [##########] 100% 1.784311 MB/1.784311 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/02\n", + "goes16/2020/4/20/ABI-L2-LSTC/02/OR_ABI-L2-LSTC-M6_G16_s20201110201132_e20201110203505_c20201110204344.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 03 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110301132_e20201110303505_c20201110304351.nc: [##########] 100% 1.836626 MB/1.836626 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/03\n", + "goes16/2020/4/20/ABI-L2-LSTC/03/OR_ABI-L2-LSTC-M6_G16_s20201110301132_e20201110303505_c20201110304351.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 04 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110401132_e20201110403504_c20201110404358.nc: [##########] 100% 1.858458 MB/1.858458 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/04\n", + "goes16/2020/4/20/ABI-L2-LSTC/04/OR_ABI-L2-LSTC-M6_G16_s20201110401132_e20201110403504_c20201110404358.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 05 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110501131_e20201110503504_c20201110504389.nc: [##########] 100% 1.872962 MB/1.872962 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/05\n", + "goes16/2020/4/20/ABI-L2-LSTC/05/OR_ABI-L2-LSTC-M6_G16_s20201110501131_e20201110503504_c20201110504389.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 06 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110601131_e20201110603504_c20201110604343.nc: [##########] 100% 1.882345 MB/1.882345 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/06\n", + "goes16/2020/4/20/ABI-L2-LSTC/06/OR_ABI-L2-LSTC-M6_G16_s20201110601131_e20201110603504_c20201110604343.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 07 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110701131_e20201110703504_c20201110704364.nc: [##########] 100% 1.906022 MB/1.906022 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/07\n", + "goes16/2020/4/20/ABI-L2-LSTC/07/OR_ABI-L2-LSTC-M6_G16_s20201110701131_e20201110703504_c20201110704364.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 08 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110801131_e20201110803504_c20201110804329.nc: [##########] 100% 1.921032 MB/1.921032 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/08\n", + "goes16/2020/4/20/ABI-L2-LSTC/08/OR_ABI-L2-LSTC-M6_G16_s20201110801131_e20201110803504_c20201110804329.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 09 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201110901131_e20201110903504_c20201110904360.nc: [##########] 100% 1.954707 MB/1.954707 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/09\n", + "goes16/2020/4/20/ABI-L2-LSTC/09/OR_ABI-L2-LSTC-M6_G16_s20201110901131_e20201110903504_c20201110904360.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 10 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111001131_e20201111003504_c20201111004351.nc: [##########] 100% 1.975989 MB/1.975989 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/10\n", + "goes16/2020/4/20/ABI-L2-LSTC/10/OR_ABI-L2-LSTC-M6_G16_s20201111001131_e20201111003504_c20201111004351.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 11 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111101131_e20201111103504_c20201111104359.nc: [##########] 100% 1.984507 MB/1.984507 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/11\n", + "goes16/2020/4/20/ABI-L2-LSTC/11/OR_ABI-L2-LSTC-M6_G16_s20201111101131_e20201111103504_c20201111104359.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 12 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111201131_e20201111203503_c20201111204339.nc: [##########] 100% 1.967923 MB/1.967923 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/12\n", + "goes16/2020/4/20/ABI-L2-LSTC/12/OR_ABI-L2-LSTC-M6_G16_s20201111201131_e20201111203503_c20201111204339.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 13 UTC\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 14 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111401130_e20201111403503_c20201111404374.nc: [##########] 100% 1.810978 MB/1.810978 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/14\n", + "goes16/2020/4/20/ABI-L2-LSTC/14/OR_ABI-L2-LSTC-M6_G16_s20201111401130_e20201111403503_c20201111404374.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 15 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111501130_e20201111503503_c20201111504393.nc: [##########] 100% 1.880033 MB/1.880033 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/15\n", + "goes16/2020/4/20/ABI-L2-LSTC/15/OR_ABI-L2-LSTC-M6_G16_s20201111501130_e20201111503503_c20201111504393.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 16 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111601130_e20201111603503_c20201111604388.nc: [##########] 100% 1.912107 MB/1.912107 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/16\n", + "goes16/2020/4/20/ABI-L2-LSTC/16/OR_ABI-L2-LSTC-M6_G16_s20201111601130_e20201111603503_c20201111604388.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 17 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111701130_e20201111703503_c20201111704362.nc: [##########] 100% 1.920667 MB/1.920667 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/17\n", + "goes16/2020/4/20/ABI-L2-LSTC/17/OR_ABI-L2-LSTC-M6_G16_s20201111701130_e20201111703503_c20201111704362.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 18 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111801130_e20201111803503_c20201111804416.nc: [##########] 100% 1.845728 MB/1.845728 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/18\n", + "goes16/2020/4/20/ABI-L2-LSTC/18/OR_ABI-L2-LSTC-M6_G16_s20201111801130_e20201111803503_c20201111804416.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 19 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201111901130_e20201111903503_c20201111904383.nc: [##########] 100% 1.763694 MB/1.763694 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/19\n", + "goes16/2020/4/20/ABI-L2-LSTC/19/OR_ABI-L2-LSTC-M6_G16_s20201111901130_e20201111903503_c20201111904383.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 20 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201112001130_e20201112003503_c20201112004371.nc: [##########] 100% 1.727079 MB/1.727079 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/20\n", + "goes16/2020/4/20/ABI-L2-LSTC/20/OR_ABI-L2-LSTC-M6_G16_s20201112001130_e20201112003503_c20201112004371.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 21 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201112101130_e20201112103503_c20201112104424.nc: [##########] 100% 1.687915 MB/1.687915 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/21\n", + "goes16/2020/4/20/ABI-L2-LSTC/21/OR_ABI-L2-LSTC-M6_G16_s20201112101130_e20201112103503_c20201112104424.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 22 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201112201130_e20201112203503_c20201112204424.nc: [##########] 100% 1.691709 MB/1.691709 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/22\n", + "goes16/2020/4/20/ABI-L2-LSTC/22/OR_ABI-L2-LSTC-M6_G16_s20201112201130_e20201112203503_c20201112204424.nc\n", + "Downloading... the product ABI-L2-LSTC \n", + "Downloading dataset to... 23 UTC\n", + "OR_ABI-L2-LSTC-M6_G16_s20201112301130_e20201112303503_c20201112304429.nc: [##########] 100% 1.729744 MB/1.729744 MB\n", + "Subsetting files in...goes16/2020/4/20/ABI-L2-LSTC/23\n", + "goes16/2020/4/20/ABI-L2-LSTC/23/OR_ABI-L2-LSTC-M6_G16_s20201112301130_e20201112303503_c20201112304429.nc\n", + "Downloading... the product ABI-L2-LSTC \n", "Downloading dataset to... 00 UTC\n", - "OR_ABI-L1b-RadC-M6C02_G18_s20241100056172_e20241100058545_c20241100058573.nc: [##########] 100% 67.930455 MB/67.930455 MB\n", - "Subsetting files in...goes18/2024/4/19/ABI-L1b-RadC/00/C02\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100036172_e20241100038545_c20241100038570.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100006172_e20241100008545_c20241100008570.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100046172_e20241100048545_c20241100048571.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100021172_e20241100023545_c20241100023570.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100011172_e20241100013545_c20241100013573.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100016172_e20241100018545_c20241100018577.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100026172_e20241100028545_c20241100028580.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100031172_e20241100033545_c20241100033572.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100056172_e20241100058545_c20241100058573.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100001172_e20241100003545_c20241100003572.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100041172_e20241100043545_c20241100043570.nc\n", - "goes18/2024/4/19/ABI-L1b-RadC/00/C02/OR_ABI-L1b-RadC-M6C02_G18_s20241100051172_e20241100053545_c20241100053568.nc\n", + "OR_ABI-L2-LSTC-M6_G16_s20201120001130_e20201120003503_c20201120004384.nc: [##########] 100% 1.842908 MB/1.842908 MB\n", + "Subsetting files in...goes16/2020/4/21/ABI-L2-LSTC/00\n", + "goes16/2020/4/21/ABI-L2-LSTC/00/OR_ABI-L2-LSTC-M6_G16_s20201120001130_e20201120003503_c20201120004384.nc\n", "Done\n", - "CPU times: user 15.7 s, sys: 4.8 s, total: 20.5 s\n", - "Wall time: 51 s\n" + "CPU times: user 6.49 s, sys: 870 ms, total: 7.36 s\n", + "Wall time: 55.2 s\n" ] } ], "source": [ "%%time\n", - "filepaths = go.get_data.download_abi_goespy(f\"{satellite}-example-ACMC.json\")" + "filepaths = go.get_data.download_abi_goespy(request_filepath)" ] }, { "cell_type": "code", "execution_count": null, - "id": "bd893536-d241-4966-a49c-dd93729d6c43", + "id": "374725bb-d6fb-4efa-b194-dfe9bab317d0", "metadata": {}, "outputs": [], "source": [] diff --git a/docs/index.rst b/docs/index.rst index dc86a6c..b0f5e7e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,6 +7,7 @@ installation examples/quick_start + examples/download examples/subset_abi_netcdf_example examples/orthorectify_abi_example examples/orthorectify_abi_example2 diff --git a/pyproject.toml b/pyproject.toml index 0b9513b..c4b5294 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "aiohttp", "scipy", "geopandas", + "geojson", "folium", "distributed", "tqdm", diff --git a/src/goes_ortho/get_data.py b/src/goes_ortho/get_data.py index 5dc11b1..b28a330 100644 --- a/src/goes_ortho/get_data.py +++ b/src/goes_ortho/get_data.py @@ -1,6 +1,4 @@ -"""Get test data for tests and/or examples""" - -# based on https://github.com/GlacioHack/xdem/blob/d91bf1cc9b3f36d77f3729649bc8e9edc6b42f9f/xdem/examples.py#L33 +"""Functions for downloading data""" import datetime as dt import json @@ -13,29 +11,35 @@ import tarfile import urllib.request from pathlib import Path +from typing import List, Tuple, Union +import geojson import goes2go as g2g import numpy as np import xarray as xr import zarr from dateutil import parser, rrule +from shapely.geometry import shape from tqdm import tqdm import goes_ortho as go -def build_zarr(downloadRequest_filepath, downloader="goes2go"): +def build_zarr(download_request_filepath, downloader="goes2go"): + """ + For running through github actions. Download and build a zarr file from a download request json file + """ # download requested imagery logging.info("download requested imagery") if downloader == "goespy": - image_path_list = download_abi_goespy(downloadRequest_filepath) + image_path_list = download_abi_goespy(download_request_filepath) if downloader == "goes2go": - image_path_list = download_abi_goes2go(downloadRequest_filepath) + image_path_list = download_abi_goes2go(download_request_filepath) # parse json request file logging.info("parse json request file") _, _, bounds, _, _, _, _, variables, apiKey, _, outputFilepath = parse_json( - downloadRequest_filepath + download_request_filepath ) # orthorectify all images @@ -139,8 +143,10 @@ def make_request_json( band, variable, apiKey, -): - """For running through github actions, make a request json file from github user input to be read by the build_zarr function""" +) -> str: + """ + For running through github actions, make a request json file from github user input to be read by the build_zarr function + """ request_dict = { "dateRange": {"startDatetime": startDatetime, "endDatetime": endDatetime}, "bounds": { @@ -160,13 +166,33 @@ def make_request_json( filename = workflowName + ".json" with open(filename, "w") as f: json.dump(request_dict, f) + return filename -def get_start_date_from_abi_filename(s): - return s.split("_s")[1].split("_")[0] +def get_start_date_from_abi_filename(s: str) -> str: + """ + Read the start time from an ABI filename string. + + Parameters + ------------ + s : str + filename for an ABI image product + + Returns + ----------- + time_str : str + start time from the filename + """ + time_str = s.split("_s")[1].split("_")[0] + return time_str -def add_datetime_crs(files, variable, crs="EPSG:4326"): +def add_datetime_crs( + files: List[str], variable: str, crs: str = "EPSG:4326" +) -> Tuple[List[str], List[dt.datetime]]: + """ + Add datetime and CRS information to a GOES ABI image product file + """ print(files) print(variable) print(crs) @@ -206,29 +232,46 @@ def add_datetime_crs(files, variable, crs="EPSG:4326"): return new_files, datetimes -def parse_json(downloadRequest_filepath): +def parse_json(download_request_filepath: str) -> Tuple: + """ + Parses a json file that specifies what should be downloaded, created with make_request_json() + + Parameters + ------------ + download_request_filepath : str + file path to a download request json file, created with make_request_json() + + Returns + ----------- + parsed : Tuple + Tuple of parsed information (startDatetime, endDatetime, bounds, satellite, bucket, product, channels, variables, apiKey, outDir, outputFilepath) + + Examples + ------------ + (startDatetime, endDatetime, bounds, satellite, bucket, product, channels, variables, apiKey, outDir, outputFilepath) = parse_json("my_download_request.json") + """ # load json file that specifies what we'd like to download and parse its contents - with open(downloadRequest_filepath, "r") as f: - downloadRequest = json.load(f) + with open(download_request_filepath, "r") as f: + download_request = json.load(f) - startDatetime = parser.parse(downloadRequest["dateRange"]["startDatetime"]) - endDatetime = parser.parse(downloadRequest["dateRange"]["endDatetime"]) + startDatetime = parser.parse(download_request["dateRange"]["startDatetime"]) + endDatetime = parser.parse(download_request["dateRange"]["endDatetime"]) bounds = [ - downloadRequest["bounds"]["min_lon"], - downloadRequest["bounds"]["min_lat"], - downloadRequest["bounds"]["max_lon"], - downloadRequest["bounds"]["max_lat"], + download_request["bounds"]["min_lon"], + download_request["bounds"]["min_lat"], + download_request["bounds"]["max_lon"], + download_request["bounds"]["max_lat"], ] # bounds = [min_lon, min_lat, max_lon, max_lat] - satellite = downloadRequest["satellite"] + satellite = download_request["satellite"] bucket = "noaa-" + satellite - product = downloadRequest["product"] - channels = downloadRequest["bands"] - variables = downloadRequest["variables"] - apiKey = downloadRequest["apiKey"] - outDir = downloadRequest["downloadDirectory"] - outputFilepath = downloadRequest["outputFilepath"] - - return ( + product = download_request["product"] + channels = download_request["bands"] + variables = download_request["variables"] + apiKey = download_request["apiKey"] + outDir = download_request["downloadDirectory"] + outputFilepath = download_request["outputFilepath"] + + parsed = ( startDatetime, endDatetime, bounds, @@ -242,9 +285,27 @@ def parse_json(downloadRequest_filepath): outputFilepath, ) + return parsed + -def download_abi_goespy(downloadRequest_filepath): - """Download GOES ABI imagery as specified by an input JSON file. (this function wraps around goespy.ABIDownloader())""" +def download_abi_goespy(download_request_filepath: str) -> List[Path]: + """ + Download GOES ABI imagery as specified by an input JSON file using the goespy package for downloading. + + Parameters + ------------ + download_request_filepath : str + file path to a download request json file, created with make_request_json() + + Returns + ----------- + output_filepaths : List[Path] + path to output DEM (useful if the downloaded DEM is reprojected to custom proj) + + Examples + ------------ + filepaths = download_abi_goespy("my_download_request.json") + """ ( startDatetime, @@ -258,7 +319,7 @@ def download_abi_goespy(downloadRequest_filepath): _, outDir, _, - ) = parse_json(downloadRequest_filepath) + ) = parse_json(download_request_filepath) output_filepaths = [] @@ -336,8 +397,24 @@ def download_abi_goespy(downloadRequest_filepath): return output_filepaths -def download_abi_goes2go(downloadRequest_filepath): - """Download GOES ABI imagery as specified by an input JSON file. (this function wraps around goes2go functions)""" +def download_abi_goes2go(download_request_filepath: str) -> List[Path]: + """ + Download GOES ABI imagery as specified by an input JSON file using the goes2go package for downloading. + + Parameters + ------------ + download_request_filepath : str + file path to a download request json file, created with make_request_json() + + Returns + ----------- + output_filepaths : List[Path] + path to output DEM (useful if the downloaded DEM is reprojected to custom proj) + + Examples + ------------ + filepaths = download_abi_goes2go("my_download_request.json") + """ ( start_datetime, @@ -351,7 +428,7 @@ def download_abi_goes2go(downloadRequest_filepath): _, outDir, _, - ) = parse_json(downloadRequest_filepath) + ) = parse_json(download_request_filepath) output_filepaths = [] @@ -388,11 +465,20 @@ def download_abi_goes2go(downloadRequest_filepath): # see what is available to download # df = G.df(start=startDatetime.strftime("%Y-%m-%d %H:%M"), end=endDatetime.strftime("%Y-%m-%d %H:%M")) - # download this batch - df = G.timerange( - start=this_start_datetime.strftime("%Y-%m-%d %H:%M"), - end=this_end_datetime.strftime("%Y-%m-%d %H:%M"), - ) + try: + # download this batch + df = G.timerange( + start=this_start_datetime.strftime("%Y-%m-%d %H:%M"), + end=this_end_datetime.strftime("%Y-%m-%d %H:%M"), + ) + except FileNotFoundError as e: + print( + f"FileNotFoundError encountered. The requested image may not exist. Because this searched a time window of {n} hours, there may be some valid imagery within the time window. Try a smaller time window to search for valid imagery.\n{e}" + ) + # set the next start datetime to the end of this batch + this_start_datetime = this_end_datetime + batch_number += 1 + continue # get the filepaths of this batch batch_filepaths = [ @@ -431,7 +517,7 @@ def download_abi_goes2go(downloadRequest_filepath): def get_dem(demtype, bounds, api_key, out_fn=None, proj="EPSG:4326"): """ - download a DEM of choice from OpenTopography World DEM (modified by Shashank Bhushan, first written by David Shean) + download a DEM of choice from OpenTopography World DEM Parameters ------------ @@ -457,7 +543,8 @@ def get_dem(demtype, bounds, api_key, out_fn=None, proj="EPSG:4326"): import requests - ### From David Shean + # written by Shashank Bhushan, David Shean + base_url = "https://portal.opentopography.org/API/globaldem?demtype={}&west={}&south={}&east={}&north={}&outputFormat=GTiff&API_Key={}" if out_fn is None: out_fn = "{}.tif".format(demtype) @@ -488,8 +575,9 @@ def get_dem(demtype, bounds, api_key, out_fn=None, proj="EPSG:4326"): def run_bash_command(cmd): # written by Scott Henderson - # move to asp_binder_utils - """Call a system command through the subprocess python module.""" + """ + Call a system command through the subprocess python module. + """ print(cmd) try: retcode = subprocess.call(cmd, shell=True) @@ -505,7 +593,7 @@ def download_example_data() -> None: """ Fetch the GOES ABI example files. """ - + # based on https://github.com/GlacioHack/xdem/blob/d91bf1cc9b3f36d77f3729649bc8e9edc6b42f9f/xdem/examples.py#L33 # Static commit hash (to be updated as needed) # commit = "16756d3aff6ca41ebb0be999a82d2f66930e7851" # The URL from which to download the tarball @@ -535,5 +623,31 @@ def download_example_data() -> None: def remove_example_data() -> None: + """ + Remove example files downloaded by download_example_data(). + """ tmp_dir = "./tests/resources/" shutil.rmtree(tmp_dir) + + +def bounds_from_geojson(geojson_filepath: str) -> Union[List[List[float]], List[float]]: + """ + Get a bounding box around a polygon from a geojson file + """ + + with open(geojson_filepath) as f: + geojson_data = geojson.load(f) + bounds = [] + for feature in geojson_data["features"]: + bounds.append(shape(feature["geometry"]).bounds) # [minx, miny, maxx, maxy] + + if len(bounds) > 1: + print( + "geojson file contains more than one feature, returning a list of bounds, one set of bounds per feature" + ) + if len(bounds) == 1: + bounds = bounds[ + 0 + ] # if there is only one set of bounds, remove it from the list + + return bounds