This repository has been archived by the owner on Jun 2, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathSnakefile_fetch_munge
91 lines (79 loc) · 3.99 KB
/
Snakefile_fetch_munge
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
import importlib
configfile: "01_fetch/wildcards_fetch_config.yaml"
usgs_nwis_sites = config["fetch_usgs_nwis.py"]["sites"]
noaa_nos_sites = config["fetch_noaa_nos.py"]["sites"]
noaa_nos_products = config["fetch_noaa_nos.py"]["products"]
noaa_nerrs_site = config["fetch_noaa_nerrs.py"]["sites"]
noaa_nerrs_years = [2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022]
wildcard_constraints:
noaa_nos_sites="\d+"
rule all_fetch_munge:
input:
#usgs nwis
expand("01_fetch/out/usgs_nwis_{sites}.txt", sites = usgs_nwis_sites),
expand("02_munge/out/D/usgs_nwis_{usgs_nwis_site}.csv", usgs_nwis_site = usgs_nwis_sites),
#noaa nos
expand("01_fetch/out/noaa_nos_{noaa_nos_sites}_{noaa_nos_products}.csv", noaa_nos_sites = noaa_nos_sites, noaa_nos_products = noaa_nos_products),
expand("02_munge/out/H/noaa_nos_{noaa_nos_sites}.csv", noaa_nos_sites = noaa_nos_sites),
expand("02_munge/out/daily_summaries/noaa_nos_{noaa_nos_sites}.csv", noaa_nos_sites = noaa_nos_sites),
#noaa nerr
expand("01_fetch/out/noaa_nerrs_{noaa_nerrs_site}_{years}.csv", noaa_nerrs_site = noaa_nerrs_site, years = noaa_nerrs_years),
expand("02_munge/out/D/noaa_nerrs_{noaa_nerrs_site}.csv", noaa_nerrs_site = noaa_nerrs_site)
rule fetch_usgs_nwis:
input:
"01_fetch/params_config_fetch_usgs_nwis.yaml"
output:
"01_fetch/out/usgs_nwis_{usgs_nwis_sites}.txt"
run:
fetch = importlib.import_module('01_fetch.src.fetch_usgs_nwis')
fetch.fetch_single_site_data({wildcards.usgs_nwis_sites})
fetch.fetch_param_file()
rule fetch_noaa_nos:
input:
"01_fetch/params_config_fetch_noaa_nos.yaml"
output:
"01_fetch/out/noaa_nos_{noaa_nos_sites}_{noaa_nos_products}.csv"
run:
fetch = importlib.import_module('01_fetch.src.fetch_noaa_nos')
fetch.fetch_single_site_data({wildcards.noaa_nos_sites}, {wildcards.noaa_nos_products})
fetch.fetch_site_metadata_file({wildcards.noaa_nos_sites})
rule fetch_noaa_nerrs:
input:
"01_fetch/params_config_fetch_noaa_nerrs.yaml"
output:
expand("01_fetch/out/noaa_nerrs_{noaa_nerrs_site}_{years}.csv", noaa_nerrs_site = noaa_nerrs_site, years = noaa_nerrs_years)
run:
fetch = importlib.import_module('01_fetch.src.fetch_noaa_nerrs')
fetch.fetch_single_site_data('delsjmet','.\\01_fetch\\in\\953860.zip')
rule munge_usgs_nwis:
input:
"02_munge/params_config_munge_usgs_nwis.yaml",
"01_fetch/out/usgs_nwis_{usgs_nwis_sites}.txt",
"02_munge/params_config_fill_discharge_prms.yaml",
output:
"02_munge/out/D/usgs_nwis_{usgs_nwis_sites}.csv"
run:
munge = importlib.import_module('02_munge.src.munge_usgs_nwis')
munge.munge_single_site_data({wildcards.usgs_nwis_sites})
# fill discharge data with prms data
fill = importlib.import_module('02_munge.src.fill_discharge_prms')
fill.fill_single_site_data({wildcards.usgs_nwis_sites})
rule munge_noaa_nos:
input:
"02_munge/params_config_munge_noaa_nos.yaml",
expand("01_fetch/out/noaa_nos_{noaa_nos_site}_{noaa_nos_product}.csv", noaa_nos_site = noaa_nos_sites, noaa_nos_product=noaa_nos_products)
output:
"02_munge/out/H/noaa_nos_{noaa_nos_sites}.csv",
"02_munge/out/daily_summaries/noaa_nos_{noaa_nos_sites}.csv"
run:
munge = importlib.import_module('02_munge.src.munge_noaa_nos')
munge.munge_single_site_data({wildcards.noaa_nos_sites})
rule munge_noaa_nerrs:
input:
"02_munge/params_config_munge_noaa_nerrs.yaml",
expand("01_fetch/out/noaa_nerrs_{noaa_nerrs_site}_{years}.csv", noaa_nerrs_site = noaa_nerrs_site, years = noaa_nerrs_years)
output:
expand("02_munge/out/D/noaa_nerrs_{noaa_nerrs_site}.csv", noaa_nerrs_site = noaa_nerrs_site)
run:
munge = importlib.import_module('02_munge.src.munge_noaa_nerrs')
munge.munge_single_site_data('delsjmet')