Merge pull request #3 from limnoliver/add_data

Add data

.gitignore

@@ -4,5 +4,6 @@
 .Ruserdata
 
 out_data/*
+in_data/*
 .remake/*
 tmp/*

1_spatial.yml

@@ -6,31 +6,35 @@ packages:
   - rgdal
   - sf
   - zip
-
+  - readr
+  - stringr
+
 sources:
   - src/spatial_functions.R
+  - src/munge_functions.R
 
 targets:
   1_spatial:
     depends:
       - out_data/01_spatial.zip
 
-  modeled_network_sf:
-    command: retrieve_network(network_sf_fl = '')
-
-  network_vertices_sf:
-    command: retrieve_vertices(network_sf_fl = '')
-
+  study_sites:
+    command: retrieve_ids(in_file = 'in_data/Data - ERL paper/Forcing_attrFiles/no_dam_attr_temp60__days118sites.csv')
+
+  gage_locations:
+    command: subset_points(
+      points_url = I('https://water.usgs.gov/GIS/dsdl/gagesII_9322_point_shapefile.zip'), sites = study_sites)
+
   # include map of network, maybe with HRUs?
   #out_data/modeling_domain_map.png:
    # command: plot_domain_map(target_name,
     #  network_sf = modeled_network_sf,
       #plot_crs = I("+init=epsg:2811"))
-    
+
   spatial_metadata:
-    command: extract_feature(network_vertices_sf)
-    
-  out_data/01_spatial_network.zip:
+    command: extract_feature(gage_locations)
+
+  out_data/01_gage_locations.zip:
     command: sf_to_zip(target_name,
-      sf_object = modeled_network_sf,
-      layer_name = I('study_stream_reaches'))
+      sf_object = gage_locations,
+      layer_name = I('gage_locations'))

2_observations.yml

@@ -6,16 +6,23 @@ packages:
 
 sources:
   - src/file_functions.R
+  - src/munge_functions.R
 
 targets:
   2_observations:
     depends:
-      - out_data/temperature_observations.zip
-      - out_data/flow_observations.zip
+      - out_data/temperature_observations.csv
+      - out_data/flow_observations.csv
 
   # daily flow and temperature data
-  out_data/temperature_observations.zip:
-    command: zip_obs(out_file = target_name, in_file = '')
-
-  out_data/flow_observations.zip:
-    command: zip_obs(out_file = target_name, in_file = '')
+  out_data/temperature_observations.csv:
+    command: extract_obs(
+      out_file = target_name,
+      var = I('wtemp(C)'),
+      in_file = 'in_data/Data - ERL paper/Forcing_attrFiles/no_dam_forcing_60__days118sites.csv')
+
+  out_data/flow_observations.csv:
+    command: extract_obs(
+      out_file = target_name,
+      var = I('discharge(cfs)'),
+      in_file = 'in_data/Data - ERL paper/Forcing_attrFiles/no_dam_forcing_60__days118sites.csv')

3_inputs.yml

@@ -9,14 +9,28 @@ packages:
 
 sources:
   - src/file_functions.R
+  - src/munge_functions.R
 
 targets:
   3_inputs:
     depends:
+      - out_data/AT_basin_attributes.csv
+      - out_data/weather_drivers.zip
+      - out_data/pred_discharge.csv
 
-  #weather_drivers:
-   # command: extract_weather_drivers()
-
-  #out_data/weather_drivers.zip:
-   # command: zip_this(out_file = target_name, in_file = weather_drivers)
-
+  out_data/AT_basin_attributes.csv:
+    command: extract_AT_attributes(
+      out_file = target_name,
+      in_file = 'in_data/Data - ERL paper/Forcing_attrFiles/no_dam_attr_temp60__days118sites.csv')
+
+  weather_drivers:
+    command: subset_weather_drivers(in_file = 'in_data/Data - ERL paper/Forcing_attrFiles/no_dam_forcing_60__days118sites.csv')
+
+  out_data/weather_drivers.zip:
+    command: zip_this(out_file = target_name, weather_drivers)
+
+
+  out_data/pred_discharge.csv:
+    command: subset_pred_discharge(
+      out_file = target_name,
+      in_file = 'in_data/Data - ERL paper/Forcing_attrFiles/no_dam_forcing_60__days118sites.csv')

5_predictions.yml

@@ -1,13 +1,37 @@
 target_default: 5_predictions
 
-include:
-
 packages:
+  - readr
+  - reticulate
 
 sources:
+  - src/munge_functions.R
 
 targets:
   5_predictions:
     depends:
+      - out_data/model_lr_predictions.csv
+      - out_data/model_obsq_predictions.csv
+      - out_data/model_simq_predictions.csv
+      - out_data/model_noq_predictions.csv
+
+
+  out_data/model_lr_predictions.csv:
+    command: npy_to_csv(
+      out_file = target_name, sites = study_sites,
+      in_file = 'in_data/Data - ERL paper/LR/pred.npy')
+
+  out_data/model_obsq_predictions.csv:
+    command: npy_to_csv(
+      out_file = target_name, sites = study_sites,
+      in_file = 'in_data/Data - ERL paper/obsQ/pred.npy')
+
+  out_data/model_simq_predictions.csv:
+    command: npy_to_csv(
+      out_file = target_name, sites = study_sites,
+      in_file = 'in_data/Data - ERL paper/simQ/pred.npy')
 
-
+  out_data/model_noq_predictions.csv:
+    command: npy_to_csv(
+      out_file = target_name, sites = study_sites,
+      in_file = 'in_data/Data - ERL paper/noQ/pred.npy')

6_evaluation.yml

@@ -3,11 +3,45 @@ target_default: 6_evaluation
 include:
 
 packages:
+  - dplyr
+  - reticulate
 
 sources:
+  - src/munge_functions.R
 
 targets:
   6_evaluation:
     depends:
+      - out_data/model_lr_evaluation.csv
+      - out_data/model_obsq_evaluation.csv
+      - out_data/model_simq_evaluation.csv
+      - out_data/model_noq_evaluation.csv
+
+  out_data/model_obsq_evaluation.csv:
+    command: extract_metrics(
+      out_file = target_name,
+      mod_index = I(1),
+      sites = study_sites,
+      in_file = "in_data/Data - ERL paper/model_evaluation_file/statDictLst_res.npz")
+
+  out_data/model_simq_evaluation.csv:
+    command: extract_metrics(
+      out_file = target_name,
+      mod_index = I(2),
+      sites = study_sites,
+      in_file = "in_data/Data - ERL paper/model_evaluation_file/statDictLst_res.npz")
+
+  out_data/model_noq_evaluation.csv:
+    command: extract_metrics(
+      out_file = target_name,
+      mod_index = I(3),
+      sites = study_sites,
+      in_file = "in_data/Data - ERL paper/model_evaluation_file/statDictLst_res.npz")
+
+  out_data/model_lr_evaluation.csv:
+    command: extract_metrics(
+      out_file = target_name,
+      mod_index = I(4),
+      sites = study_sites,
+      in_file = "in_data/Data - ERL paper/model_evaluation_file/statDictLst_res.npz")
 
-

in_text/text_00_parent.yml

@@ -3,17 +3,17 @@ title: >-
 
 
 abstract: >-
-  <p>This data release provides all data and code used in Rahmani et al. 2020. Briefly, this project used the CAMELS dataset as a test case for temperature prediction using deep learning methods. XX
+  <p>This data release provides all data and code used in Rahmani et al. 2020. Briefly, this project used the Gages II dataset as a test case for temperature prediction using deep learning methods. XX
   <br/>The data are organized into these items:</p>
   <ol>
-  <li><a href="https://www.sciencebase.gov/catalog/item/5f908db182ce720ee2d0fef9">Spatial Information</a> - Basin polygons and pour points for 118 river basins used in this study</li>
+  <li><a href="https://www.sciencebase.gov/catalog/item/5f908db182ce720ee2d0fef9">Spatial Information</a> - Pour points for 118 river basins used in this study</li>
   <li><a href="https://www.sciencebase.gov/catalog/item/5f986594d34e198cb77ff084">Observations</a> - Water temperature observations and flow observations for the 118 river reaches used in this study</li>
   <li><a href="https://www.sciencebase.gov/catalog/item/5f9865abd34e198cb77ff086">Model Inputs</a> - Model inputs, including basin characteristics and weather drivers. </li>
   <li><a href="https://www.sciencebase.gov/catalog/item/5f9865cfd34e198cb77ff088">Models</a> - Code and configuration for stream temperature models.</li>
-  <li><a href="https://www.sciencebase.gov/catalog/item/5f9865e5d34e198cb77ff08a">Model Predictions</a> - Predictions of water temperature</li>
-  <li><a href="https://www.sciencebase.gov/catalog/item/5f9865fbd34e198cb77ff08c">Model Evaluation</a> - Performance of models</li>
+  <li><a href="https://www.sciencebase.gov/catalog/item/5f9865e5d34e198cb77ff08a">Model Predictions</a> - Predictions of stream water temperature and discharge</li>
+  <li><a href="https://www.sciencebase.gov/catalog/item/5f9865fbd34e198cb77ff08c">Model Evaluation</a> - Performance metrics of each stream temperature model</li>
   <br/>
-  <p>This research was funded by the USGS, XX.</p>
+  <p>This research was funded by the Integrated Water Prediction Program at the US Geological Survey.</p>
 
 build-environment: Multiple computer systems were used to generate these data, including linux, OSX. The open source languages R and Python were used on all systems. XX
 

in_text/text_01_spatial.yml

@@ -6,29 +6,53 @@ abstract: >-
 
 cross-cites:
   -
-    authors: ['A. Newman; K. Sampson; M. P. Clark; A. Bock; R. J. Viger; D. Blodgett']
+    authors: ['J.A. Falcone']
     title: >-
-       A large-sample watershed-scale hydrometeorological dataset for the contiguous USA
-    pubdate: 2014
-    link: https://dx.doi.org/10.5065/D6MW2F4D
+      GAGES-II: Geospatial attributes of gages for evaluating streamflow
+    pubdate: 2011
+    link: https://doi.org/10.3133/70046617
 
 entities:
   -
-    data-name: 01_spatial.zip
-    data-description: GIS polygons for 118 river catchments included in this study. XX.
+    data-name: 01_gage_locations.zip
+    data-description: GIS point locations for the 118 river catchments included in this study. These sites are a subset from the Gages II dataset.
     attributes:
     -
-      attr-label: XX
+      attr-label: site_no
       attr-def: >-
-        XX
+        USGS unique site identifier.
+      attr-defs: NA
+      data-min: NA
+      data-max: NA
+      data-units: NA
+    -
+      attr-label: site_name
+      attr-def: >-
+        Name of the site, which often includes river name.
+      attr-defs: NA
+      data-min: NA
+      data-max: NA
+      data-units: NA
+    -
+      attr-label: lat
+      attr-def: >-
+        Latitude of the site location.
+      attr-defs: NA
+      data-min: NA
+      data-max: NA
+      data-units: NA
+    -
+      attr-label: long
+      attr-def: >-
+        Longitude of the site location.
       attr-defs: NA
       data-min: NA
       data-max: NA
       data-units: NA
 
 
-data-name: Polygons for river catchments used in this study
-data-description: Shapefile of river catchments included in this data release.
+data-name: GIS points of sites used in this study.
+data-description: Location of USGS river gages used in this study.
 
 file-format: Shapefile Data Set