From 7aea2dfc6e4147863701c8ac29d8e5484b50efb2 Mon Sep 17 00:00:00 2001 From: crvernon Date: Mon, 8 Apr 2024 15:15:51 -0400 Subject: [PATCH] pre-commit fixes to nbs --- ...Demand_WaterRights_File_Modification.ipynb | 290 ++++++++++-------- .../N2_Evaporation_File_Modification.ipynb | 192 ++++++------ .../N3_Reservoir_File_Modification.ipynb | 137 +++++---- .../N4_Streamflow_File_Modification.ipynb | 124 ++++---- notebooks/N5_Batch_Modification.ipynb | 82 +++-- 5 files changed, 429 insertions(+), 396 deletions(-) diff --git a/notebooks/N1_Demand_WaterRights_File_Modification.ipynb b/notebooks/N1_Demand_WaterRights_File_Modification.ipynb index 9169a91..63dd110 100644 --- a/notebooks/N1_Demand_WaterRights_File_Modification.ipynb +++ b/notebooks/N1_Demand_WaterRights_File_Modification.ipynb @@ -77,12 +77,13 @@ "import logging\n", "import os\n", "import pickle\n", - "from string import Template\n", "import subprocess\n", + "from string import Template\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import pandas as pd \n", + "import pandas as pd\n", + "\n", "import statemodify as stm" ] }, @@ -122,11 +123,7 @@ "\n", "# data directory and root name for the target basin\n", "data_dir = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"sj2015_StateMod_modified\",\n", - " \"sj2015_StateMod_modified\",\n", - " \"StateMod\"\n", + " home_dir, \"data\", \"sj2015_StateMod_modified\", \"sj2015_StateMod_modified\", \"StateMod\"\n", ")\n", "\n", "# directory to the target basin input files with root name for the basin\n", @@ -141,17 +138,9 @@ "parquet_dir_ddr = os.path.join(data_dir, \"parquet_ddr\")\n", "\n", "# path to ddm and ddr template file\n", - "ddm_template_file = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"sj2015B_template_ddm.rsp\"\n", - ")\n", + "ddm_template_file = os.path.join(home_dir, \"data\", \"sj2015B_template_ddm.rsp\")\n", "\n", - "ddr_template_file = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"sj2015B_template_ddr.rsp\"\n", - ")" + "ddr_template_file = os.path.join(home_dir, \"data\", \"sj2015B_template_ddr.rsp\")" ] }, { @@ -165,7 +154,7 @@ "outputs": [], "source": [ "# run statemod\n", - "subprocess.call([statemod_exe, basin_path, \"-simulate\"])\n" + "subprocess.call([statemod_exe, basin_path, \"-simulate\"])" ] }, { @@ -185,9 +174,9 @@ }, "outputs": [], "source": [ - "#Extract shortages using statemodify convert_xdd() function \n", + "# Extract shortages using statemodify convert_xdd() function\n", "\n", - "# create a directory to store the historical shortages \n", + "# create a directory to store the historical shortages\n", "output_dir = os.path.join(data_dir, \"historic_shortages\")\n", "\n", "# create a directory to store the new files in if it does not exist\n", @@ -203,10 +192,10 @@ " # path, glob, or a list of paths/globs to the .xdd files you want to convert\n", " xdd_files=os.path.join(data_dir, \"*.xdd\"),\n", " # if the output .parquet files should only contain a subset of structure ids, list them here; None for all\n", - " id_subset=['2900501','2900519','2900555'],\n", + " id_subset=[\"2900501\", \"2900519\", \"2900555\"],\n", " # how many .xdd files to convert in parallel; optimally you will want 2-4 CPUs per parallel process\n", " parallel_jobs=4,\n", - ")\n" + ")" ] }, { @@ -218,7 +207,7 @@ }, "outputs": [], "source": [ - "data=pd.read_parquet(os.path.join(output_dir,'sj2015B.parquet'),engine='pyarrow')\n", + "data = pd.read_parquet(os.path.join(output_dir, \"sj2015B.parquet\"), engine=\"pyarrow\")\n", "data" ] }, @@ -249,14 +238,13 @@ }, "outputs": [], "source": [ - "data['shortage_total']=data['shortage_total'].astype(float)\n", - "data['year']=data['year'].astype(int)\n", + "data[\"shortage_total\"] = data[\"shortage_total\"].astype(float)\n", + "data[\"year\"] = data[\"year\"].astype(int)\n", "\n", "fig, ax = plt.subplots()\n", "\n", - "for name, group in data.groupby('structure_id'):\n", - " ax.scatter(\n", - " group['year'], group['shortage_total'], label=name)\n", + "for name, group in data.groupby(\"structure_id\"):\n", + " ax.scatter(group[\"year\"], group[\"shortage_total\"], label=name)\n", "\n", "plt.xlabel(\"Year\")\n", "plt.ylabel(\"Shortage (AF)\")\n", @@ -301,10 +289,7 @@ "outputs": [], "source": [ "# a dictionary to describe what users you want to modify and the bounds for the LHS\n", - "setup_dict = {\n", - " \"ids\": [\"2900501\", \"2900519\",\"2900555\"],\n", - " \"bounds\": [0.5, 1.5]\n", - "}\n", + "setup_dict = {\"ids\": [\"2900501\", \"2900519\", \"2900555\"], \"bounds\": [0.5, 1.5]}\n", "\n", "output_directory = output_dir = os.path.join(data_dir, \"input_files\")\n", "\n", @@ -329,22 +314,24 @@ "basin_name = \"San_Juan\"\n", "\n", "# generate a batch of files using generated LHS\n", - "stm.modify_ddm(modify_dict=setup_dict,\n", - " query_field=query_field,\n", - " output_dir=output_directory,\n", - " scenario=scenario,\n", - " basin_name=basin_name,\n", - " sampling_method=\"LHS\",\n", - " n_samples=n_samples,\n", - " skip_rows=skip_rows,\n", - " n_jobs=n_jobs,\n", - " seed_value=seed_value,\n", - " template_file=None,\n", - " factor_method=\"multiply\",\n", - " data_specification_file=None,\n", - " min_bound_value=-0.5,\n", - " max_bound_value=1.5,\n", - " save_sample=True)" + "stm.modify_ddm(\n", + " modify_dict=setup_dict,\n", + " query_field=query_field,\n", + " output_dir=output_directory,\n", + " scenario=scenario,\n", + " basin_name=basin_name,\n", + " sampling_method=\"LHS\",\n", + " n_samples=n_samples,\n", + " skip_rows=skip_rows,\n", + " n_jobs=n_jobs,\n", + " seed_value=seed_value,\n", + " template_file=None,\n", + " factor_method=\"multiply\",\n", + " data_specification_file=None,\n", + " min_bound_value=-0.5,\n", + " max_bound_value=1.5,\n", + " save_sample=True,\n", + ")" ] }, { @@ -365,7 +352,8 @@ "outputs": [], "source": [ "import numpy as np\n", - "sample_array=np.load(output_directory+'/ddm_2-samples_scenario-1.npy')\n", + "\n", + "sample_array = np.load(output_directory + \"/ddm_2-samples_scenario-1.npy\")\n", "sample_array" ] }, @@ -405,51 +393,62 @@ "\n", "# read RSP template\n", "with open(ddm_template_file) as template_obj:\n", - " \n", " # read in file\n", " template_rsp = Template(template_obj.read())\n", "\n", " for i in sample:\n", - " \n", " # create scenario name\n", " scenario = f\"S{i}_{realization}\"\n", - " \n", + "\n", " # dictionary holding search keys and replacement values to update the template file\n", " d = {\"DDM\": f\"../../input_files/sj2015B_{scenario}.ddm\"}\n", - " \n", + "\n", " # update the template\n", " new_rsp = template_rsp.safe_substitute(d)\n", - " \n", + "\n", " # construct simulated scenario directory\n", " simulated_scenario_dir = os.path.join(scenarios_dir_ddm, scenario)\n", " if not os.path.exists(simulated_scenario_dir):\n", " os.makedirs(simulated_scenario_dir)\n", - " \n", + "\n", " # target rsp file\n", " rsp_file = os.path.join(simulated_scenario_dir, f\"sj2015B_{scenario}.rsp\")\n", - " \n", + "\n", " # write updated rsp file\n", " with open(rsp_file, \"w\") as f1:\n", " f1.write(new_rsp)\n", - " \n", + "\n", " # construct simulated basin path\n", - " simulated_basin_path = os.path.join(simulated_scenario_dir, f\"sj2015B_{scenario}\")\n", + " simulated_basin_path = os.path.join(\n", + " simulated_scenario_dir, f\"sj2015B_{scenario}\"\n", + " )\n", "\n", " # run StateMod\n", " print(f\"Running: {scenario}\")\n", " os.chdir(simulated_scenario_dir)\n", "\n", " subprocess.call([statemod_exe, simulated_basin_path, \"-simulate\"])\n", - " \n", - " #Save output to parquet files \n", - " print('creating parquet for ' + scenario)\n", - " \n", - " output_directory = os.path.join(parquet_dir_ddm+\"/scenario/\"+ scenario)\n", - " \n", + "\n", + " # Save output to parquet files\n", + " print(\"creating parquet for \" + scenario)\n", + "\n", + " output_directory = os.path.join(parquet_dir_ddm + \"/scenario/\" + scenario)\n", + "\n", " if not os.path.exists(output_directory):\n", " os.makedirs(output_directory)\n", - " \n", - " stm.xdd.convert_xdd(output_path=output_directory,allow_overwrite=False,xdd_files=scenarios_dir_ddm + \"/\"+ scenario + \"/sj2015B_\"+scenario+\".xdd\",id_subset=['2900501','2900519','2900555'],parallel_jobs=4)" + "\n", + " stm.xdd.convert_xdd(\n", + " output_path=output_directory,\n", + " allow_overwrite=False,\n", + " xdd_files=scenarios_dir_ddm\n", + " + \"/\"\n", + " + scenario\n", + " + \"/sj2015B_\"\n", + " + scenario\n", + " + \".xdd\",\n", + " id_subset=[\"2900501\", \"2900519\", \"2900555\"],\n", + " parallel_jobs=4,\n", + " )" ] }, { @@ -482,7 +481,7 @@ "outputs": [], "source": [ "output_directory = os.path.join(data_dir, \"input_files\")\n", - "sample_array = np.load(output_directory+'/ddm_2-samples_scenario-1.npy')\n", + "sample_array = np.load(output_directory + \"/ddm_2-samples_scenario-1.npy\")\n", "sample_array" ] }, @@ -503,27 +502,41 @@ }, "outputs": [], "source": [ - "#Read in raw parquet files\n", - "baseline=pd.read_parquet(data_dir+'/historic_shortages/sj2015B.parquet',engine='pyarrow')\n", - "SOW_1=pd.read_parquet(parquet_dir_ddm+'/scenario/S0_1/sj2015B_S0_1.parquet',engine='pyarrow')\n", - "SOW_2=pd.read_parquet(parquet_dir_ddm+'/scenario/S1_1/sj2015B_S1_1.parquet',engine='pyarrow')\n", + "# Read in raw parquet files\n", + "baseline = pd.read_parquet(\n", + " data_dir + \"/historic_shortages/sj2015B.parquet\", engine=\"pyarrow\"\n", + ")\n", + "SOW_1 = pd.read_parquet(\n", + " parquet_dir_ddm + \"/scenario/S0_1/sj2015B_S0_1.parquet\", engine=\"pyarrow\"\n", + ")\n", + "SOW_2 = pd.read_parquet(\n", + " parquet_dir_ddm + \"/scenario/S1_1/sj2015B_S1_1.parquet\", engine=\"pyarrow\"\n", + ")\n", "\n", - "#Subtract shortages with respect to the baseline\n", - "subset_df=pd.concat([baseline['year'],baseline['shortage_total'],SOW_1['shortage_total'],SOW_2['shortage_total']],axis=1)\n", - "subset_df = subset_df.set_axis(['Year', 'Baseline', 'SOW_1','SOW_2'], axis=1)\n", - "subset_df['Baseline']=subset_df['Baseline'].astype(float)\n", - "subset_df['SOW_1']=subset_df['SOW_1'].astype(float)\n", - "subset_df['SOW_2']=subset_df['SOW_2'].astype(float)\n", - "subset_df['Year']=subset_df['Year'].astype(int)\n", - "subset_df['SOW_1_diff']=subset_df['SOW_1']-subset_df['Baseline']\n", - "subset_df['SOW_2_diff']=subset_df['SOW_2']-subset_df['Baseline']\n", + "# Subtract shortages with respect to the baseline\n", + "subset_df = pd.concat(\n", + " [\n", + " baseline[\"year\"],\n", + " baseline[\"shortage_total\"],\n", + " SOW_1[\"shortage_total\"],\n", + " SOW_2[\"shortage_total\"],\n", + " ],\n", + " axis=1,\n", + ")\n", + "subset_df = subset_df.set_axis([\"Year\", \"Baseline\", \"SOW_1\", \"SOW_2\"], axis=1)\n", + "subset_df[\"Baseline\"] = subset_df[\"Baseline\"].astype(float)\n", + "subset_df[\"SOW_1\"] = subset_df[\"SOW_1\"].astype(float)\n", + "subset_df[\"SOW_2\"] = subset_df[\"SOW_2\"].astype(float)\n", + "subset_df[\"Year\"] = subset_df[\"Year\"].astype(int)\n", + "subset_df[\"SOW_1_diff\"] = subset_df[\"SOW_1\"] - subset_df[\"Baseline\"]\n", + "subset_df[\"SOW_2_diff\"] = subset_df[\"SOW_2\"] - subset_df[\"Baseline\"]\n", "\n", "\n", - "#Plot shortages\n", + "# Plot shortages\n", "fig, ax = plt.subplots()\n", "\n", - "ax.scatter(subset_df['Year'], subset_df['SOW_1_diff'],label='Decreased Demand')\n", - "ax.scatter(subset_df['Year'], subset_df['SOW_2_diff'],label='Increased Demand')\n", + "ax.scatter(subset_df[\"Year\"], subset_df[\"SOW_1_diff\"], label=\"Decreased Demand\")\n", + "ax.scatter(subset_df[\"Year\"], subset_df[\"SOW_2_diff\"], label=\"Increased Demand\")\n", "\n", "plt.xlabel(\"Year\")\n", "plt.ylabel(\"Shortage (AF)\")\n", @@ -568,11 +581,9 @@ "setup_dict = {\n", " # ids can either be 'struct' or 'id' values\n", " \"ids\": [\"2900501\"],\n", - "\n", " # turn id on or off completely or for a given period\n", " # if 0 = off, 1 = on, YYYY = on for years >= YYYY, -YYYY = off for years > YYYY; see file header\n", " \"on_off\": [1],\n", - "\n", " # apply rank of administrative order where 0 is lowest (senior) and n is highest (junior); None is no change\n", " \"admin\": [1],\n", "}\n", @@ -599,22 +610,24 @@ "basin_name = \"San_Juan\"\n", "\n", "# generate a batch of files using generated LHS\n", - "stm.modify_ddr(modify_dict=setup_dict,\n", - " query_field=query_field,\n", - " output_dir=output_directory,\n", - " scenario=scenario,\n", - " basin_name=basin_name,\n", - " sampling_method=\"LHS\",\n", - " n_samples=n_samples,\n", - " skip_rows=skip_rows,\n", - " n_jobs=n_jobs,\n", - " seed_value=seed_value,\n", - " template_file=None,\n", - " factor_method=\"multiply\",\n", - " data_specification_file=None,\n", - " min_bound_value=-0.5,\n", - " max_bound_value=1.5,\n", - " save_sample=True)" + "stm.modify_ddr(\n", + " modify_dict=setup_dict,\n", + " query_field=query_field,\n", + " output_dir=output_directory,\n", + " scenario=scenario,\n", + " basin_name=basin_name,\n", + " sampling_method=\"LHS\",\n", + " n_samples=n_samples,\n", + " skip_rows=skip_rows,\n", + " n_jobs=n_jobs,\n", + " seed_value=seed_value,\n", + " template_file=None,\n", + " factor_method=\"multiply\",\n", + " data_specification_file=None,\n", + " min_bound_value=-0.5,\n", + " max_bound_value=1.5,\n", + " save_sample=True,\n", + ")" ] }, { @@ -641,51 +654,62 @@ "\n", "# read RSP template\n", "with open(ddr_template_file) as template_obj:\n", - " \n", " # read in file\n", " template_rsp = Template(template_obj.read())\n", "\n", " for i in sample:\n", - " \n", " # create scenario name\n", " scenario = f\"S{i}_{realization}\"\n", - " \n", + "\n", " # dictionary holding search keys and replacement values to update the template file\n", " d = {\"DDR\": f\"../../input_files/sj2015B_{scenario}.ddr\"}\n", - " \n", + "\n", " # update the template\n", " new_rsp = template_rsp.safe_substitute(d)\n", - " \n", + "\n", " # construct simulated scenario directory\n", " simulated_scenario_dir = os.path.join(scenarios_dir_ddr, scenario)\n", " if not os.path.exists(simulated_scenario_dir):\n", " os.makedirs(simulated_scenario_dir)\n", - " \n", + "\n", " # target rsp file\n", " rsp_file = os.path.join(simulated_scenario_dir, f\"sj2015B_{scenario}.rsp\")\n", - " \n", + "\n", " # write updated rsp file\n", " with open(rsp_file, \"w\") as f1:\n", " f1.write(new_rsp)\n", - " \n", + "\n", " # construct simulated basin path\n", - " simulated_basin_path = os.path.join(simulated_scenario_dir, f\"sj2015B_{scenario}\")\n", + " simulated_basin_path = os.path.join(\n", + " simulated_scenario_dir, f\"sj2015B_{scenario}\"\n", + " )\n", "\n", " # run StateMod\n", " print(f\"Running: {scenario}\")\n", " os.chdir(simulated_scenario_dir)\n", "\n", " subprocess.call([statemod_exe, simulated_basin_path, \"-simulate\"])\n", - " \n", - " #Save output to parquet files \n", - " print('creating parquet for ' + scenario)\n", - " \n", - " output_directory = os.path.join(parquet_dir_ddr+\"/scenario/\"+ scenario)\n", - " \n", + "\n", + " # Save output to parquet files\n", + " print(\"creating parquet for \" + scenario)\n", + "\n", + " output_directory = os.path.join(parquet_dir_ddr + \"/scenario/\" + scenario)\n", + "\n", " if not os.path.exists(output_directory):\n", " os.makedirs(output_directory)\n", - " \n", - " stm.xdd.convert_xdd(output_path=output_directory,allow_overwrite=False,xdd_files=scenarios_dir_ddr + \"/\"+ scenario + \"/sj2015B_\"+scenario+\".xdd\",id_subset=['2900501'],parallel_jobs=2)" + "\n", + " stm.xdd.convert_xdd(\n", + " output_path=output_directory,\n", + " allow_overwrite=False,\n", + " xdd_files=scenarios_dir_ddr\n", + " + \"/\"\n", + " + scenario\n", + " + \"/sj2015B_\"\n", + " + scenario\n", + " + \".xdd\",\n", + " id_subset=[\"2900501\"],\n", + " parallel_jobs=2,\n", + " )" ] }, { @@ -705,22 +729,28 @@ }, "outputs": [], "source": [ - "#Read in raw parquet files\n", - "baseline=pd.read_parquet(data_dir+'/historic_shortages/sj2015B.parquet',engine='pyarrow')\n", - "SOW_1=pd.read_parquet(parquet_dir_ddr+ '/scenario/S0_1/sj2015B_S0_1.parquet',engine='pyarrow')\n", + "# Read in raw parquet files\n", + "baseline = pd.read_parquet(\n", + " data_dir + \"/historic_shortages/sj2015B.parquet\", engine=\"pyarrow\"\n", + ")\n", + "SOW_1 = pd.read_parquet(\n", + " parquet_dir_ddr + \"/scenario/S0_1/sj2015B_S0_1.parquet\", engine=\"pyarrow\"\n", + ")\n", "\n", - "#Subtract shortages with respect to the baseline\n", - "subset_df=pd.concat([baseline['year'],baseline['shortage_total'],SOW_1['shortage_total']],axis=1)\n", - "subset_df = subset_df.set_axis(['Year', 'Baseline', 'SOW_1'], axis=1)\n", - "subset_df['Baseline']=subset_df['Baseline'].astype(float)\n", - "subset_df['SOW_1']=subset_df['SOW_1'].astype(float)\n", - "subset_df['Year']=subset_df['Year'].astype(int)\n", - "subset_df['diff']=subset_df['SOW_1']-subset_df['Baseline']\n", + "# Subtract shortages with respect to the baseline\n", + "subset_df = pd.concat(\n", + " [baseline[\"year\"], baseline[\"shortage_total\"], SOW_1[\"shortage_total\"]], axis=1\n", + ")\n", + "subset_df = subset_df.set_axis([\"Year\", \"Baseline\", \"SOW_1\"], axis=1)\n", + "subset_df[\"Baseline\"] = subset_df[\"Baseline\"].astype(float)\n", + "subset_df[\"SOW_1\"] = subset_df[\"SOW_1\"].astype(float)\n", + "subset_df[\"Year\"] = subset_df[\"Year\"].astype(int)\n", + "subset_df[\"diff\"] = subset_df[\"SOW_1\"] - subset_df[\"Baseline\"]\n", "\n", - "#Plot shortages\n", + "# Plot shortages\n", "fig, ax = plt.subplots()\n", "\n", - "ax.scatter(subset_df['Year'], subset_df['diff'])\n", + "ax.scatter(subset_df[\"Year\"], subset_df[\"diff\"])\n", "\n", "plt.xlabel(\"Year\")\n", "plt.ylabel(\"Shortage (AF)\")\n", diff --git a/notebooks/N2_Evaporation_File_Modification.ipynb b/notebooks/N2_Evaporation_File_Modification.ipynb index 4b39ff8..924ee6a 100644 --- a/notebooks/N2_Evaporation_File_Modification.ipynb +++ b/notebooks/N2_Evaporation_File_Modification.ipynb @@ -47,12 +47,13 @@ "import logging\n", "import os\n", "import pickle\n", - "from string import Template\n", "import subprocess\n", + "from string import Template\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import pandas as pd \n", + "import pandas as pd\n", + "\n", "import statemodify as stm" ] }, @@ -96,11 +97,7 @@ "\n", "# data directory and root name for the target basin\n", "data_dir = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"gm2015_StateMod_modified\",\n", - " \"gm2015_StateMod_modified\",\n", - " \"StateMod\"\n", + " home_dir, \"data\", \"gm2015_StateMod_modified\", \"gm2015_StateMod_modified\", \"StateMod\"\n", ")\n", "\n", "# directory to the target basin input files with root name for the basin\n", @@ -110,11 +107,7 @@ "scenarios_dir = os.path.join(data_dir, \"scenarios\")\n", "\n", "# path to eva template file\n", - "eva_template_file = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"gm2015B_template_eva.rsp\"\n", - ")" + "eva_template_file = os.path.join(home_dir, \"data\", \"gm2015B_template_eva.rsp\")" ] }, { @@ -128,7 +121,7 @@ "outputs": [], "source": [ "# run statemod\n", - "subprocess.call([statemod_exe, basin_path, \"-simulate\"])\n" + "subprocess.call([statemod_exe, basin_path, \"-simulate\"])" ] }, { @@ -148,7 +141,7 @@ }, "outputs": [], "source": [ - "# create a directory to store the historical reservoir levels at Blue Mesa \n", + "# create a directory to store the historical reservoir levels at Blue Mesa\n", "output_dir = os.path.join(data_dir, \"historic_reservoir_levels\")\n", "\n", "if not os.path.exists(output_dir):\n", @@ -158,19 +151,20 @@ "xre_file = os.path.join(data_dir, \"gm2015B.xre\")\n", "\n", "# structure ID for reservoir of interest\n", - "structure_ID = '6203532' \n", + "structure_ID = \"6203532\"\n", "\n", "# name of the reservoir\n", - "structure_name = 'Blue_Mesa' \n", + "structure_name = \"Blue_Mesa\"\n", "\n", "# extract the target info into a Pandas data frame\n", - "df = stm.extract_xre_data(structure_name=structure_name,\n", - " structure_id=structure_ID,\n", - " input_file=xre_file,\n", - " basin_name=None,\n", - " output_directory=output_dir,\n", - " write_csv=True,\n", - " write_parquet=None\n", + "df = stm.extract_xre_data(\n", + " structure_name=structure_name,\n", + " structure_id=structure_ID,\n", + " input_file=xre_file,\n", + " basin_name=None,\n", + " output_directory=output_dir,\n", + " write_csv=True,\n", + " write_parquet=None,\n", ")" ] }, @@ -194,15 +188,12 @@ "output_xre_file = os.path.join(output_dir, \"Blue_Mesa_xre_data.csv\")\n", "\n", "# read output data into a data frame\n", - "df = pd.read_csv(\n", - " output_xre_file, \n", - " usecols=['Year','Init. Storage'],\n", - " index_col=False) \n", + "df = pd.read_csv(output_xre_file, usecols=[\"Year\", \"Init. Storage\"], index_col=False)\n", "\n", "# calculate the annual average\n", - "df = df.groupby('Year').mean().reset_index()\n", + "df = df.groupby(\"Year\").mean().reset_index()\n", "\n", - "df\n" + "df" ] }, { @@ -224,11 +215,11 @@ "source": [ "fig, ax = plt.subplots()\n", "\n", - "plt.plot(df['Year'], df['Init. Storage'])\n", + "plt.plot(df[\"Year\"], df[\"Init. Storage\"])\n", "\n", "plt.title(\"Blue Mesa Storage\")\n", "plt.xlabel(\"Year\")\n", - "plt.ylabel(\"Reservoir Storage (AF)\")\n" + "plt.ylabel(\"Reservoir Storage (AF)\")" ] }, { @@ -256,11 +247,8 @@ }, "outputs": [], "source": [ - "# a dictionary to describe what you want to modify and the bounds for the Latin hypercube sample. \n", - "setup_dict = {\n", - " \"ids\": ['10011'],\n", - " \"bounds\": [-0.5, 1.0]\n", - "}\n", + "# a dictionary to describe what you want to modify and the bounds for the Latin hypercube sample.\n", + "setup_dict = {\"ids\": [\"10011\"], \"bounds\": [-0.5, 1.0]}\n", "\n", "# create a directory to store the new files in if it does not exist\n", "output_directory = os.path.join(data_dir, \"input_files\")\n", @@ -289,22 +277,24 @@ "basin_name = \"Gunnison\"\n", "\n", "# generate a batch of files using generated LHS\n", - "stm.modify_eva(modify_dict=setup_dict,\n", - " query_field=query_field,\n", - " output_dir=output_directory,\n", - " scenario=scenario,\n", - " basin_name=basin_name,\n", - " sampling_method=\"LHS\",\n", - " n_samples=n_samples,\n", - " skip_rows=skip_rows,\n", - " n_jobs=n_jobs,\n", - " seed_value=seed_value,\n", - " template_file=None,\n", - " factor_method=\"add\",\n", - " data_specification_file=None,\n", - " min_bound_value=-0.5,\n", - " max_bound_value=1.0,\n", - " save_sample=True)\n" + "stm.modify_eva(\n", + " modify_dict=setup_dict,\n", + " query_field=query_field,\n", + " output_dir=output_directory,\n", + " scenario=scenario,\n", + " basin_name=basin_name,\n", + " sampling_method=\"LHS\",\n", + " n_samples=n_samples,\n", + " skip_rows=skip_rows,\n", + " n_jobs=n_jobs,\n", + " seed_value=seed_value,\n", + " template_file=None,\n", + " factor_method=\"add\",\n", + " data_specification_file=None,\n", + " min_bound_value=-0.5,\n", + " max_bound_value=1.0,\n", + " save_sample=True,\n", + ")" ] }, { @@ -327,10 +317,10 @@ "# path to the numpy file containing the samples\n", "eva_samples_file = os.path.join(output_directory, \"eva_2-samples_scenario-1.npy\")\n", "\n", - "# load samples \n", + "# load samples\n", "sample_array = np.load(eva_samples_file)\n", "\n", - "sample_array\n" + "sample_array" ] }, { @@ -365,41 +355,41 @@ "\n", "# read RSP template\n", "with open(eva_template_file) as template_obj:\n", - " \n", " # read in file\n", " template_rsp = Template(template_obj.read())\n", "\n", " for i in sample:\n", - " \n", " # create scenario name\n", " scenario = f\"S{i}_{realization}\"\n", - " \n", + "\n", " # dictionary holding search keys and replacement values to update the template file\n", " d = {\"EVA\": f\"../../input_files/gm2015B_{scenario}.eva\"}\n", - " \n", + "\n", " # update the template\n", " new_rsp = template_rsp.safe_substitute(d)\n", - " \n", + "\n", " # construct simulated scenario directory\n", " simulated_scenario_dir = os.path.join(scenarios_dir, scenario)\n", " if not os.path.exists(simulated_scenario_dir):\n", " os.makedirs(simulated_scenario_dir)\n", - " \n", + "\n", " # target rsp file\n", " rsp_file = os.path.join(simulated_scenario_dir, f\"gm2015B_{scenario}.rsp\")\n", - " \n", + "\n", " # write updated rsp file\n", " with open(rsp_file, \"w\") as f1:\n", " f1.write(new_rsp)\n", - " \n", + "\n", " # construct simulated basin path\n", - " simulated_basin_path = os.path.join(simulated_scenario_dir, f\"gm2015B_{scenario}\")\n", + " simulated_basin_path = os.path.join(\n", + " simulated_scenario_dir, f\"gm2015B_{scenario}\"\n", + " )\n", "\n", " # run StateMod\n", " print(f\"Running: {scenario}\")\n", " os.chdir(simulated_scenario_dir)\n", "\n", - " subprocess.call([statemod_exe, simulated_basin_path, \"-simulate\"])\n" + " subprocess.call([statemod_exe, simulated_basin_path, \"-simulate\"])" ] }, { @@ -428,43 +418,45 @@ "outputs": [], "source": [ "# SOW 1\n", - "output_dir= os.path.join(scenarios_dir, \"S0_1\")\n", + "output_dir = os.path.join(scenarios_dir, \"S0_1\")\n", "\n", "# path the the xre file\n", - "xre_file = os.path.join(output_dir, \"gm2015B_S0_1.xre\") \n", + "xre_file = os.path.join(output_dir, \"gm2015B_S0_1.xre\")\n", "\n", "# structure ID for reservoir of interest\n", - "structure_ID = '6203532' \n", + "structure_ID = \"6203532\"\n", "\n", "# name of the reservoir\n", - "structure_name = 'Blue_Mesa' \n", + "structure_name = \"Blue_Mesa\"\n", "\n", "# extract the target info into a Pandas data frame\n", - "df = stm.extract_xre_data(structure_name=structure_name,\n", - " structure_id=structure_ID,\n", - " input_file=xre_file,\n", - " basin_name=None,\n", - " output_directory=output_dir,\n", - " write_csv=True,\n", - " write_parquet=None\n", + "df = stm.extract_xre_data(\n", + " structure_name=structure_name,\n", + " structure_id=structure_ID,\n", + " input_file=xre_file,\n", + " basin_name=None,\n", + " output_directory=output_dir,\n", + " write_csv=True,\n", + " write_parquet=None,\n", ")\n", "\n", "# SOW 2\n", - "output_dir= os.path.join(scenarios_dir, \"S1_1\")\n", + "output_dir = os.path.join(scenarios_dir, \"S1_1\")\n", "\n", "# path the the xre file\n", - "xre_file = os.path.join(output_dir, \"gm2015B_S1_1.xre\") \n", + "xre_file = os.path.join(output_dir, \"gm2015B_S1_1.xre\")\n", "\n", "\n", "# extract the target info into a Pandas data frame\n", - "df = stm.extract_xre_data(structure_name=structure_name,\n", - " structure_id=structure_ID,\n", - " input_file=xre_file,\n", - " basin_name=None,\n", - " output_directory=output_dir,\n", - " write_csv=True,\n", - " write_parquet=None\n", - ")\n" + "df = stm.extract_xre_data(\n", + " structure_name=structure_name,\n", + " structure_id=structure_ID,\n", + " input_file=xre_file,\n", + " basin_name=None,\n", + " output_directory=output_dir,\n", + " write_csv=True,\n", + " write_parquet=None,\n", + ")" ] }, { @@ -488,32 +480,34 @@ "historic_res_dir = os.path.join(data_dir, \"historic_reservoir_levels\")\n", "blue_mesa_file = os.path.join(historic_res_dir, \"Blue_Mesa_xre_data.csv\")\n", "\n", - "# Import baseline dataframe \n", - "baseline = pd.read_csv(blue_mesa_file, index_col=False, usecols=['Year','Init. Storage']) \n", - "baseline = baseline.groupby('Year').mean().reset_index()\n", + "# Import baseline dataframe\n", + "baseline = pd.read_csv(\n", + " blue_mesa_file, index_col=False, usecols=[\"Year\", \"Init. Storage\"]\n", + ")\n", + "baseline = baseline.groupby(\"Year\").mean().reset_index()\n", "\n", "# Import SOW1\n", "s0_1_file = os.path.join(scenarios_dir, \"S0_1\", \"Blue_Mesa_xre_data.csv\")\n", - "SOW1 = pd.read_csv(s0_1_file, index_col=False, usecols=['Year','Init. Storage']) \n", - "SOW1 = SOW1.groupby('Year').mean().reset_index()\n", - " \n", + "SOW1 = pd.read_csv(s0_1_file, index_col=False, usecols=[\"Year\", \"Init. Storage\"])\n", + "SOW1 = SOW1.groupby(\"Year\").mean().reset_index()\n", + "\n", "# Import SOW2\n", "s1_1_file = os.path.join(scenarios_dir, \"S1_1\", \"Blue_Mesa_xre_data.csv\")\n", - "SOW2 = pd.read_csv(s1_1_file, index_col=False, usecols=['Year','Init. Storage']) \n", - "SOW2 = SOW2.groupby('Year').mean().reset_index()\n", - " \n", - "# Plot reservoir levels \n", + "SOW2 = pd.read_csv(s1_1_file, index_col=False, usecols=[\"Year\", \"Init. Storage\"])\n", + "SOW2 = SOW2.groupby(\"Year\").mean().reset_index()\n", + "\n", + "# Plot reservoir levels\n", "fig, ax = plt.subplots()\n", "\n", - "plt.plot(baseline['Year'], baseline['Init. Storage'],label='Baseline')\n", - "plt.plot(SOW1['Year'], SOW1['Init. Storage'],label='Reduced Evaporation')\n", - "plt.plot(SOW2['Year'], SOW2['Init. Storage'],label='Increased Evaporation')\n", + "plt.plot(baseline[\"Year\"], baseline[\"Init. Storage\"], label=\"Baseline\")\n", + "plt.plot(SOW1[\"Year\"], SOW1[\"Init. Storage\"], label=\"Reduced Evaporation\")\n", + "plt.plot(SOW2[\"Year\"], SOW2[\"Init. Storage\"], label=\"Increased Evaporation\")\n", "\n", "plt.title(\"Blue Mesa Storage\")\n", "plt.xlabel(\"Year\")\n", "plt.ylabel(\"Reservoir Storage (AF)\")\n", "\n", - "plt.legend()\n" + "plt.legend()" ] }, { diff --git a/notebooks/N3_Reservoir_File_Modification.ipynb b/notebooks/N3_Reservoir_File_Modification.ipynb index dbca4e1..2798178 100644 --- a/notebooks/N3_Reservoir_File_Modification.ipynb +++ b/notebooks/N3_Reservoir_File_Modification.ipynb @@ -37,12 +37,13 @@ "import logging\n", "import os\n", "import pickle\n", - "from string import Template\n", "import subprocess\n", + "from string import Template\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import pandas as pd \n", + "import pandas as pd\n", + "\n", "import statemodify as stm" ] }, @@ -88,12 +89,7 @@ "statemod_exe = os.path.join(root_dir, \"statemod\")\n", "\n", "# data directory and root name for the target basin\n", - "data_dir = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"cm2015_StateMod\",\n", - " \"StateMod\"\n", - ")\n", + "data_dir = os.path.join(home_dir, \"data\", \"cm2015_StateMod\", \"StateMod\")\n", "\n", "# directory to the target basin input files with root name for the basin\n", "basin_path = os.path.join(data_dir, \"cm2015B\")\n", @@ -106,11 +102,7 @@ "\n", "\n", "# path to res template file\n", - "res_template_file = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"cm2015B_template_res.rsp\"\n", - ")" + "res_template_file = os.path.join(home_dir, \"data\", \"cm2015B_template_res.rsp\")" ] }, { @@ -135,9 +127,11 @@ }, "outputs": [], "source": [ - "# Change directories first \n", - "os.chdir(data_dir) #This is needed specific to the Upper Colorado model as the path name is too long for the model to accept\n", - "subprocess.call([statemod_exe, \"cm2015B\", \"-simulate\"])\n" + "# Change directories first\n", + "os.chdir(\n", + " data_dir\n", + ") # This is needed specific to the Upper Colorado model as the path name is too long for the model to accept\n", + "subprocess.call([statemod_exe, \"cm2015B\", \"-simulate\"])" ] }, { @@ -159,9 +153,9 @@ }, "outputs": [], "source": [ - "#Extract shortages using statemodify convert_xdd() function \n", + "# Extract shortages using statemodify convert_xdd() function\n", "\n", - "# create a directory to store the historic shortages \n", + "# create a directory to store the historic shortages\n", "output_dir = os.path.join(data_dir, \"historic_shortages\")\n", "\n", "# create a directory to store the new files in if it does not exist\n", @@ -177,7 +171,7 @@ " # path, glob, or a list of paths/globs to the .xdd files you want to convert\n", " xdd_files=os.path.join(data_dir, \"*.xdd\"),\n", " # if the output .parquet files should only contain a subset of structure ids, list them here; None for all\n", - " id_subset=['3601008'],\n", + " id_subset=[\"3601008\"],\n", " # how many .xdd files to convert in parallel; optimally you will want 2-4 CPUs per parallel process\n", " parallel_jobs=2,\n", ")" @@ -200,17 +194,16 @@ }, "outputs": [], "source": [ - "data=pd.read_parquet(output_dir +'/cm2015B.parquet',engine='pyarrow')\n", + "data = pd.read_parquet(output_dir + \"/cm2015B.parquet\", engine=\"pyarrow\")\n", "\n", "\n", - "data['shortage_total']=data['shortage_total'].astype(float)\n", - "data['year']=data['year'].astype(int)\n", + "data[\"shortage_total\"] = data[\"shortage_total\"].astype(float)\n", + "data[\"year\"] = data[\"year\"].astype(int)\n", "\n", "fig, ax = plt.subplots()\n", "\n", - "for name, group in data.groupby('structure_id'):\n", - " ax.scatter(\n", - " group['year'], group['shortage_total'], label=name)\n", + "for name, group in data.groupby(\"structure_id\"):\n", + " ax.scatter(group[\"year\"], group[\"shortage_total\"], label=name)\n", "\n", "plt.xlabel(\"Year\")\n", "plt.ylabel(\"Shortage (AF)\")\n", @@ -265,14 +258,16 @@ "# number of samples to generate\n", "n_samples = 1\n", "\n", - "stm.modify_res(output_dir=output_directory,\n", - " scenario=scenario,\n", - " basin_name=basin_name,\n", - " target_structure_id_list=['3603575','3604512'],\n", - " seed_value=seed_value,\n", - " n_jobs=n_jobs,\n", - " n_samples=n_samples,\n", - " save_sample=True)\n" + "stm.modify_res(\n", + " output_dir=output_directory,\n", + " scenario=scenario,\n", + " basin_name=basin_name,\n", + " target_structure_id_list=[\"3603575\", \"3604512\"],\n", + " seed_value=seed_value,\n", + " n_jobs=n_jobs,\n", + " n_samples=n_samples,\n", + " save_sample=True,\n", + ")" ] }, { @@ -293,7 +288,8 @@ "outputs": [], "source": [ "import numpy as np\n", - "sample_array = np.load(output_directory+'/res_1-samples_scenario-1.npy')\n", + "\n", + "sample_array = np.load(output_directory + \"/res_1-samples_scenario-1.npy\")\n", "sample_array" ] }, @@ -329,33 +325,31 @@ "\n", "# read RSP template\n", "with open(res_template_file) as template_obj:\n", - " \n", " # read in file\n", " template_rsp = Template(template_obj.read())\n", "\n", " for i in sample:\n", - " \n", " # create scenario name\n", " scenario = f\"S{i}_{realization}\"\n", - " \n", + "\n", " # dictionary holding search keys and replacement values to update the template file\n", " d = {\"RES\": f\"../../input_files/cm2015B_{scenario}.res\"}\n", - " \n", + "\n", " # update the template\n", " new_rsp = template_rsp.safe_substitute(d)\n", - " \n", + "\n", " # construct simulated scenario directory\n", " simulated_scenario_dir = os.path.join(scenarios_dir_res, scenario)\n", " if not os.path.exists(simulated_scenario_dir):\n", " os.makedirs(simulated_scenario_dir)\n", - " \n", + "\n", " # target rsp file\n", " rsp_file = os.path.join(simulated_scenario_dir, f\"cm2015B_{scenario}.rsp\")\n", - " \n", + "\n", " # write updated rsp file\n", " with open(rsp_file, \"w\") as f1:\n", " f1.write(new_rsp)\n", - " \n", + "\n", " # construct simulated basin path\n", " simulated_basin_path = f\"cm2015B_{scenario}\"\n", "\n", @@ -364,16 +358,27 @@ " os.chdir(simulated_scenario_dir)\n", "\n", " subprocess.call([statemod_exe, simulated_basin_path, \"-simulate\"])\n", - " \n", - " #Save output to parquet files \n", - " print('creating parquet for ' + scenario)\n", - " \n", - " output_directory = os.path.join(parquet_dir_res+\"/scenario/\"+ scenario)\n", - " \n", + "\n", + " # Save output to parquet files\n", + " print(\"creating parquet for \" + scenario)\n", + "\n", + " output_directory = os.path.join(parquet_dir_res + \"/scenario/\" + scenario)\n", + "\n", " if not os.path.exists(output_directory):\n", " os.makedirs(output_directory)\n", - " \n", - " stm.xdd.convert_xdd(output_path=output_directory,allow_overwrite=True,xdd_files=scenarios_dir_res + \"/\"+ scenario + \"/cm2015B_\"+scenario+\".xdd\",id_subset=['3601008'],parallel_jobs=2)" + "\n", + " stm.xdd.convert_xdd(\n", + " output_path=output_directory,\n", + " allow_overwrite=True,\n", + " xdd_files=scenarios_dir_res\n", + " + \"/\"\n", + " + scenario\n", + " + \"/cm2015B_\"\n", + " + scenario\n", + " + \".xdd\",\n", + " id_subset=[\"3601008\"],\n", + " parallel_jobs=2,\n", + " )" ] }, { @@ -393,21 +398,27 @@ }, "outputs": [], "source": [ - "baseline=pd.read_parquet(data_dir+'/'+'historic_shortages/cm2015B.parquet',engine='pyarrow')\n", - "SOW_1=pd.read_parquet(parquet_dir_res+'/scenario/S0_1/cm2015B_S0_1.parquet',engine='pyarrow')\n", - "\n", - "#Subtract shortages with respect to the baseline\n", - "subset_df=pd.concat([baseline['year'],baseline['shortage_total'],SOW_1['shortage_total']],axis=1)\n", - "subset_df = subset_df.set_axis(['Year', 'Baseline', 'SOW_1'], axis=1)\n", - "subset_df['Baseline']=subset_df['Baseline'].astype(float)\n", - "subset_df['SOW_1']=subset_df['SOW_1'].astype(float)\n", - "subset_df['Year']=subset_df['Year'].astype(int)\n", - "subset_df['SOW_1_diff']=subset_df['SOW_1']-subset_df['Baseline']\n", - "\n", - "#Plot shortages\n", + "baseline = pd.read_parquet(\n", + " data_dir + \"/\" + \"historic_shortages/cm2015B.parquet\", engine=\"pyarrow\"\n", + ")\n", + "SOW_1 = pd.read_parquet(\n", + " parquet_dir_res + \"/scenario/S0_1/cm2015B_S0_1.parquet\", engine=\"pyarrow\"\n", + ")\n", + "\n", + "# Subtract shortages with respect to the baseline\n", + "subset_df = pd.concat(\n", + " [baseline[\"year\"], baseline[\"shortage_total\"], SOW_1[\"shortage_total\"]], axis=1\n", + ")\n", + "subset_df = subset_df.set_axis([\"Year\", \"Baseline\", \"SOW_1\"], axis=1)\n", + "subset_df[\"Baseline\"] = subset_df[\"Baseline\"].astype(float)\n", + "subset_df[\"SOW_1\"] = subset_df[\"SOW_1\"].astype(float)\n", + "subset_df[\"Year\"] = subset_df[\"Year\"].astype(int)\n", + "subset_df[\"SOW_1_diff\"] = subset_df[\"SOW_1\"] - subset_df[\"Baseline\"]\n", + "\n", + "# Plot shortages\n", "fig, ax = plt.subplots()\n", "\n", - "ax.scatter(subset_df['Year'], subset_df['SOW_1_diff'])\n", + "ax.scatter(subset_df[\"Year\"], subset_df[\"SOW_1_diff\"])\n", "\n", "plt.xlabel(\"Year\")\n", "plt.ylabel(\"Shortage (AF)\")\n", diff --git a/notebooks/N4_Streamflow_File_Modification.ipynb b/notebooks/N4_Streamflow_File_Modification.ipynb index b341b94..5aecfd7 100644 --- a/notebooks/N4_Streamflow_File_Modification.ipynb +++ b/notebooks/N4_Streamflow_File_Modification.ipynb @@ -67,12 +67,13 @@ "import logging\n", "import os\n", "import pickle\n", - "from string import Template\n", "import subprocess\n", + "from string import Template\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import pandas as pd \n", + "import pandas as pd\n", + "\n", "import statemodify as stm" ] }, @@ -106,12 +107,7 @@ "statemod_exe = os.path.join(root_dir, \"statemod\")\n", "\n", "# data directory and root name for the target basin\n", - "data_dir = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"cm2015_StateMod\",\n", - " \"StateMod\"\n", - ")\n", + "data_dir = os.path.join(home_dir, \"data\", \"cm2015_StateMod\", \"StateMod\")\n", "\n", "# directory to the target basin input files with root name for the basin\n", "basin_path = os.path.join(data_dir, \"cm2015B\")\n", @@ -119,12 +115,8 @@ "# scenarios output directory\n", "scenarios_dir = os.path.join(data_dir, \"scenarios\")\n", "\n", - "# path to iwr/xbm file \n", - "xbm_iwr_template_file = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"cm2015B_template_xbm_iwr.rsp\"\n", - ")" + "# path to iwr/xbm file\n", + "xbm_iwr_template_file = os.path.join(home_dir, \"data\", \"cm2015B_template_xbm_iwr.rsp\")" ] }, { @@ -144,7 +136,7 @@ }, "outputs": [], "source": [ - "#Make directory to store HMM parameters\n", + "# Make directory to store HMM parameters\n", "\n", "output_dir = os.path.join(data_dir, \"HMM_parameters\")\n", "\n", @@ -156,9 +148,9 @@ "# choice to save parameters to NumPy array files\n", "save_parameters = True\n", "\n", - "fit_array_dict = stm.hmm_multisite_fit(n_basins=n_basins,\n", - " save_parameters=save_parameters,\n", - " output_directory=output_dir)\n", + "fit_array_dict = stm.hmm_multisite_fit(\n", + " n_basins=n_basins, save_parameters=save_parameters, output_directory=output_dir\n", + ")\n", "\n", "# unpack output dictionary\n", "unconditional_dry = fit_array_dict[\"unconditional_dry\"]\n", @@ -168,7 +160,7 @@ "covariance_matrix_wet = fit_array_dict[\"covariance_matrix_wet\"]\n", "covariance_matrix_dry = fit_array_dict[\"covariance_matrix_dry\"]\n", "wet_state_means = fit_array_dict[\"wet_state_means\"]\n", - "dry_state_means = fit_array_dict[\"dry_state_means\"]\n" + "dry_state_means = fit_array_dict[\"dry_state_means\"]" ] }, { @@ -196,7 +188,7 @@ }, "outputs": [], "source": [ - "#Create a folder to store the runs\n", + "# Create a folder to store the runs\n", "\n", "output_dir = os.path.join(data_dir, \"HMM_Runs\")\n", "\n", @@ -204,15 +196,17 @@ " os.makedirs(output_dir)\n", "\n", "# using the outputs of the fit function above; this function write output sample files to the output directory\n", - "stm.hmm_multisite_sample(logAnnualQ_h,\n", - " transition_matrix,\n", - " unconditional_dry,\n", - " dry_state_means,\n", - " wet_state_means,\n", - " covariance_matrix_dry,\n", - " covariance_matrix_wet,\n", - " n_basins=n_basins,\n", - " output_directory=output_dir)" + "stm.hmm_multisite_sample(\n", + " logAnnualQ_h,\n", + " transition_matrix,\n", + " unconditional_dry,\n", + " dry_state_means,\n", + " wet_state_means,\n", + " covariance_matrix_dry,\n", + " covariance_matrix_wet,\n", + " n_basins=n_basins,\n", + " output_directory=output_dir,\n", + ")" ] }, { @@ -232,10 +226,13 @@ }, "outputs": [], "source": [ - "stm.plot_flow_duration_curves(flow_realizations_directory=output_dir,\n", - " save_figure=True,output_directory=output_dir,\n", - " figure_name= 'FDC',\n", - " dpi= 300)" + "stm.plot_flow_duration_curves(\n", + " flow_realizations_directory=output_dir,\n", + " save_figure=True,\n", + " output_directory=output_dir,\n", + " figure_name=\"FDC\",\n", + " dpi=300,\n", + ")" ] }, { @@ -265,16 +262,16 @@ }, "outputs": [], "source": [ - "#Make directory to store input files \n", + "# Make directory to store input files\n", "\n", "output_dir = os.path.join(data_dir, \"input_files\")\n", "\n", "if not os.path.exists(output_dir):\n", " os.makedirs(output_dir)\n", - " \n", + "\n", "\n", "flow_realizations_directory = os.path.join(data_dir, \"HMM_Runs\")\n", - " \n", + "\n", "scenario = \"1\"\n", "\n", "# basin name to process\n", @@ -290,15 +287,17 @@ "n_samples = 1\n", "\n", "# generate a batch of files using generated LHS\n", - "stm.modify_xbm_iwr(output_dir=output_dir,\n", - " flow_realizations_directory=flow_realizations_directory,\n", - " scenario=scenario,\n", - " basin_name=basin_name,\n", - " seed_value=seed_value,\n", - " n_jobs=n_jobs,\n", - " n_samples=n_samples,\n", - " save_sample=True,\n", - " randomly_select_flow_sample=True) " + "stm.modify_xbm_iwr(\n", + " output_dir=output_dir,\n", + " flow_realizations_directory=flow_realizations_directory,\n", + " scenario=scenario,\n", + " basin_name=basin_name,\n", + " seed_value=seed_value,\n", + " n_jobs=n_jobs,\n", + " n_samples=n_samples,\n", + " save_sample=True,\n", + " randomly_select_flow_sample=True,\n", + ")" ] }, { @@ -347,33 +346,34 @@ "\n", "# read RSP template\n", "with open(xbm_iwr_template_file) as template_obj:\n", - " \n", " # read in file\n", " template_rsp = Template(template_obj.read())\n", "\n", " for i in sample:\n", - " \n", " # create scenario name\n", " scenario = f\"S{i}_{realization}\"\n", - " \n", + "\n", " # dictionary holding search keys and replacement values to update the template file\n", - " d = {\"XBM\": f\"../../input_files/cm2015B_{scenario}.xbm\",\"IWR\": f\"../../input_files/cm2015B_{scenario}.iwr\"}\n", - " \n", + " d = {\n", + " \"XBM\": f\"../../input_files/cm2015B_{scenario}.xbm\",\n", + " \"IWR\": f\"../../input_files/cm2015B_{scenario}.iwr\",\n", + " }\n", + "\n", " # update the template\n", " new_rsp = template_rsp.safe_substitute(d)\n", - " \n", + "\n", " # construct simulated scenario directory\n", " simulated_scenario_dir = os.path.join(scenarios_dir, scenario)\n", " if not os.path.exists(simulated_scenario_dir):\n", " os.makedirs(simulated_scenario_dir)\n", - " \n", + "\n", " # target rsp file\n", " rsp_file = os.path.join(simulated_scenario_dir, f\"cm2015B_{scenario}.rsp\")\n", - " \n", + "\n", " # write updated rsp file\n", " with open(rsp_file, \"w\") as f1:\n", " f1.write(new_rsp)\n", - " \n", + "\n", " # construct simulated basin path\n", " simulated_basin_path = f\"cm2015B_{scenario}\"\n", "\n", @@ -381,7 +381,7 @@ " print(f\"Running: {scenario}\")\n", " os.chdir(simulated_scenario_dir)\n", "\n", - " subprocess.call([statemod_exe, simulated_basin_path, \"-simulate\"])\n" + " subprocess.call([statemod_exe, simulated_basin_path, \"-simulate\"])" ] }, { @@ -402,14 +402,16 @@ "outputs": [], "source": [ "# Example with Granby Lake\n", - "zip_file_path = os.path.join(home_dir, 'data', 'Granby_Dataset.zip')\n", - "final_directory = os.path.join(home_dir, 'data/')\n", + "zip_file_path = os.path.join(home_dir, \"data\", \"Granby_Dataset.zip\")\n", + "final_directory = os.path.join(home_dir, \"data/\")\n", "\n", - "!unzip $zip_file_path -d $final_directory \n", - "granby_hmm, granby_hist, granby_hist_mean, granby_hist_1p = stm.read_xre(os.path.join(home_dir,\"data/Upper_Colorado/\"), 'Granby')\n", + "!unzip $zip_file_path -d $final_directory\n", + "granby_hmm, granby_hist, granby_hist_mean, granby_hist_1p = stm.read_xre(\n", + " os.path.join(home_dir, \"data/Upper_Colorado/\"), \"Granby\"\n", + ")\n", "\n", "# Plot quantiles\n", - "stm.plot_res_quantiles(granby_hmm, granby_hist_mean, 'Lake Granby')" + "stm.plot_res_quantiles(granby_hmm, granby_hist_mean, \"Lake Granby\")" ] }, { @@ -438,7 +440,7 @@ "outputs": [], "source": [ "# using the output of the above `read_xre` function as inputs\n", - "stm.plot_reservoir_boxes(granby_hmm, granby_hist, 'Lake Granby')" + "stm.plot_reservoir_boxes(granby_hmm, granby_hist, \"Lake Granby\")" ] }, { diff --git a/notebooks/N5_Batch_Modification.ipynb b/notebooks/N5_Batch_Modification.ipynb index 9099c0a..0bec3e9 100644 --- a/notebooks/N5_Batch_Modification.ipynb +++ b/notebooks/N5_Batch_Modification.ipynb @@ -33,12 +33,13 @@ "import logging\n", "import os\n", "import pickle\n", - "from string import Template\n", "import subprocess\n", + "from string import Template\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import pandas as pd \n", + "import pandas as pd\n", + "\n", "import statemodify as stm" ] }, @@ -80,12 +81,7 @@ "statemod_exe = os.path.join(root_dir, \"statemod\")\n", "\n", "# data directory and root name for the target basin\n", - "data_dir = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"cm2015_StateMod\",\n", - " \"StateMod\"\n", - ")\n", + "data_dir = os.path.join(home_dir, \"data\", \"cm2015_StateMod\", \"StateMod\")\n", "\n", "# directory to the target basin input files with root name for the basin\n", "basin_path = os.path.join(data_dir, \"cm2015B\")\n", @@ -93,16 +89,12 @@ "# scenarios output directory\n", "scenarios_dir = os.path.join(data_dir, \"scenarios\")\n", "\n", - "#parquet output directory\n", - "parquet_dir=os.path.join(data_dir, \"parquet\")\n", + "# parquet output directory\n", + "parquet_dir = os.path.join(data_dir, \"parquet\")\n", "\n", "\n", "# path to template file\n", - "multi_template_file = os.path.join(\n", - " home_dir,\n", - " \"data\",\n", - " \"cm2015B_template_multi.rsp\"\n", - ")" + "multi_template_file = os.path.join(home_dir, \"data\", \"cm2015B_template_multi.rsp\")" ] }, { @@ -169,13 +161,9 @@ "# problem dictionary\n", "problem_dict = {\n", " \"n_samples\": 1,\n", - " 'num_vars': 3,\n", - " 'names': ['modify_eva', 'modify_ddr', 'modify_ddm'],\n", - " 'bounds': [\n", - " [-0.5, 1.0],\n", - " None,\n", - " [0.5, 1.0]\n", - " ],\n", + " \"num_vars\": 3,\n", + " \"names\": [\"modify_eva\", \"modify_ddr\", \"modify_ddm\"],\n", + " \"bounds\": [[-0.5, 1.0], None, [0.5, 1.0]],\n", " # additional settings for each function\n", " \"modify_eva\": {\n", " \"seed_value\": seed_value,\n", @@ -183,7 +171,7 @@ " \"scenario\": scenario,\n", " \"basin_name\": basin_name,\n", " \"query_field\": \"id\",\n", - " \"ids\": [\"10008\", \"10009\"]\n", + " \"ids\": [\"10008\", \"10009\"],\n", " },\n", " \"modify_ddr\": {\n", " \"seed_value\": seed_value,\n", @@ -193,7 +181,7 @@ " \"query_field\": \"id\",\n", " \"ids\": [\"3600507.01\", \"3600507.02\"],\n", " \"admin\": [1, None],\n", - " \"on_off\": [1, 1]\n", + " \"on_off\": [1, 1],\n", " },\n", " \"modify_ddm\": {\n", " \"seed_value\": seed_value,\n", @@ -201,8 +189,8 @@ " \"scenario\": scenario,\n", " \"basin_name\": basin_name,\n", " \"query_field\": \"id\",\n", - " \"ids\": [\"3600507\", \"3600603\"]\n", - " }\n", + " \"ids\": [\"3600507\", \"3600603\"],\n", + " },\n", "}\n", "\n", "# run in batch\n", @@ -3028,33 +3016,35 @@ "\n", "# read RSP template\n", "with open(multi_template_file) as template_obj:\n", - " \n", " # read in file\n", " template_rsp = Template(template_obj.read())\n", "\n", " for i in sample:\n", - " \n", " # create scenario name\n", " scenario = f\"S{i}_{realization}\"\n", - " \n", + "\n", " # dictionary holding search keys and replacement values to update the template file\n", - " d = {\"EVA\": f\"../../input_files/cm2015B_{scenario}.eva\",\"DDM\": f\"../../input_files/cm2015B_{scenario}.ddm\",\"DDR\": f\"../../input_files/cm2015B_{scenario}.ddr\"}\n", - " \n", + " d = {\n", + " \"EVA\": f\"../../input_files/cm2015B_{scenario}.eva\",\n", + " \"DDM\": f\"../../input_files/cm2015B_{scenario}.ddm\",\n", + " \"DDR\": f\"../../input_files/cm2015B_{scenario}.ddr\",\n", + " }\n", + "\n", " # update the template\n", " new_rsp = template_rsp.safe_substitute(d)\n", - " \n", + "\n", " # construct simulated scenario directory\n", " simulated_scenario_dir = os.path.join(scenarios_dir, scenario)\n", " if not os.path.exists(simulated_scenario_dir):\n", " os.makedirs(simulated_scenario_dir)\n", - " \n", + "\n", " # target rsp file\n", " rsp_file = os.path.join(simulated_scenario_dir, f\"cm2015B_{scenario}.rsp\")\n", - " \n", + "\n", " # write updated rsp file\n", " with open(rsp_file, \"w\") as f1:\n", " f1.write(new_rsp)\n", - " \n", + "\n", " # construct simulated basin path\n", " simulated_basin_path = f\"cm2015B_{scenario}\"\n", "\n", @@ -3063,16 +3053,22 @@ " os.chdir(simulated_scenario_dir)\n", "\n", " subprocess.call([statemod_exe, simulated_basin_path, \"-simulate\"])\n", - " \n", - " #Save output to parquet files \n", - " print('creating parquet for ' + scenario)\n", - " \n", - " output_directory = os.path.join(parquet_dir+ \"/scenario/\"+ scenario)\n", - " \n", + "\n", + " # Save output to parquet files\n", + " print(\"creating parquet for \" + scenario)\n", + "\n", + " output_directory = os.path.join(parquet_dir + \"/scenario/\" + scenario)\n", + "\n", " if not os.path.exists(output_directory):\n", " os.makedirs(output_directory)\n", - " \n", - " stm.xdd.convert_xdd(output_path=output_directory,allow_overwrite=False,xdd_files=scenarios_dir + \"/\"+ scenario + \"/cm2015B_\"+scenario+\".xdd\",id_subset=['3601008'],parallel_jobs=2)\n" + "\n", + " stm.xdd.convert_xdd(\n", + " output_path=output_directory,\n", + " allow_overwrite=False,\n", + " xdd_files=scenarios_dir + \"/\" + scenario + \"/cm2015B_\" + scenario + \".xdd\",\n", + " id_subset=[\"3601008\"],\n", + " parallel_jobs=2,\n", + " )" ] }, {