From bd2caa8994dd515d1b9b9f306e62bb66418bedc2 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Mon, 7 Oct 2024 11:10:38 -0400 Subject: [PATCH 01/27] checkpoint: add scaling to cstr but didn't note any affect on BSM2-P solver iterations; also replacing aeration tank on bsm2-p instead of cstr_injection with flowsheet level vars and constraints --- .../BSM2_P_extension.py | 19 ++++++++++++++----- watertap/unit_models/cstr.py | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/watertap/flowsheets/full_water_resource_recovery_facility/BSM2_P_extension.py b/watertap/flowsheets/full_water_resource_recovery_facility/BSM2_P_extension.py index 51ea6ec635..8742c652a6 100644 --- a/watertap/flowsheets/full_water_resource_recovery_facility/BSM2_P_extension.py +++ b/watertap/flowsheets/full_water_resource_recovery_facility/BSM2_P_extension.py @@ -22,7 +22,7 @@ import pyomo.environ as pyo from pyomo.network import Arc, SequentialDecomposition - +from pyomo.util.calc_var_value import calculate_variable_from_constraint from idaes.core import ( FlowsheetBlock, UnitModelCostingBlock, @@ -36,6 +36,7 @@ Mixer, PressureChanger, ) +from idaes.core.util.model_diagnostics import DiagnosticsToolbox from idaes.models.unit_models.separator import SplittingType from watertap.core.solvers import get_solver from idaes.core.util.model_statistics import degrees_of_freedom @@ -46,6 +47,8 @@ stream_table_dataframe_to_string, ) from watertap.unit_models.cstr_injection import CSTR_Injection + +from watertap.unit_models.aeration_tank import AerationTank from watertap.unit_models.clarifier import Clarifier from watertap.property_models.unit_specific.anaerobic_digestion.modified_adm1_properties import ( ModifiedADM1ParameterBlock, @@ -91,10 +94,11 @@ # Set up logger _log = idaeslog.getLogger(__name__) - +_log.debug def main(bio_P=False): m = build(bio_P=bio_P) + dt = DiagnosticsToolbox(m) set_operating_conditions(m) for mx in m.fs.mixers: @@ -144,7 +148,7 @@ def main(bio_P=False): display_costing(m) display_performance_metrics(m) - return m, results + return m, results, dt def build(bio_P=False): @@ -372,6 +376,9 @@ def build(bio_P=False): mutable=True, doc="Dissolved oxygen concentration at equilibrium", ) + m.fs.R5.KLa = 240 + m.fs.R6.KLa = 240 + m.fs.R7.KLa = 84 m.fs.aerobic_reactors = (m.fs.R5, m.fs.R6, m.fs.R7) for R in m.fs.aerobic_reactors: @@ -480,7 +487,9 @@ def set_operating_conditions(m): m.fs.R5.outlet.conc_mass_comp[:, "S_O2"].fix(1.91e-3) m.fs.R6.outlet.conc_mass_comp[:, "S_O2"].fix(2.60e-3) m.fs.R7.outlet.conc_mass_comp[:, "S_O2"].fix(3.20e-3) - + # for R in m.fs.aerobic_reactors: + # calculate_variable_from_constraint(R.KLa, R.eq_mass_transfer[0]) + # Set fraction of outflow from reactor 5 that goes to recycle m.fs.SP1.split_fraction[:, "underflow"].fix(0.60) @@ -952,7 +961,7 @@ def display_performance_metrics(m): if __name__ == "__main__": - m, results = main(bio_P=False) + m, results, dt = main(bio_P=False) stream_table = create_stream_table_dataframe( { diff --git a/watertap/unit_models/cstr.py b/watertap/unit_models/cstr.py index d77ebb8b7d..e31ffb65b2 100644 --- a/watertap/unit_models/cstr.py +++ b/watertap/unit_models/cstr.py @@ -21,6 +21,7 @@ from idaes.models.unit_models.cstr import CSTRData as CSTRIDAESData import idaes.logger as idaeslog +import idaes.core.util.scaling as iscale from pyomo.environ import ( Constraint, @@ -81,3 +82,19 @@ def CSTR_retention_time_rule(self, t): @property def default_costing_method(self): return cost_cstr + + def calculate_scaling_factors(self): + + super().calculate_scaling_factors() + + for t in self.flowsheet().time: + if iscale.get_scaling_factor(self.hydraulic_retention_time[t]) is None: + iscale.set_variable_scaling_from_current_value(self.hydraulic_retention_time[t]) + if iscale.get_scaling_factor(self.volume[t]) is None: + iscale.set_variable_scaling_from_current_value(self.volume[t]) + + sf = iscale.get_scaling_factor(self.hydraulic_retention_time[t]) + iscale.constraint_scaling_transform(self.CSTR_retention_time[t], sf) + + + From 35c42dc7205518f75cb6bdad3917c699a1d70e28 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 18 Oct 2024 12:05:46 -0400 Subject: [PATCH 02/27] fix base engine url to mitigate bad request error --- watertap/tools/oli_api/credentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watertap/tools/oli_api/credentials.py b/watertap/tools/oli_api/credentials.py index 0c944caa7c..002230de76 100644 --- a/watertap/tools/oli_api/credentials.py +++ b/watertap/tools/oli_api/credentials.py @@ -191,7 +191,7 @@ def _manage_credentials(self, username, password, root_url, auth_url, access_key self.dbs_url = self.credentials["root_url"] + "/channel/dbs" self.upload_dbs_url = self.credentials["root_url"] + "/channel/upload/dbs" self.access_key_url = self.credentials["root_url"] + "/user/api-key" - self.engine_url = self.credentials["root_url"] + "/engine/" + self.engine_url = self.credentials["root_url"] + "/engine" self._delete_dbs_url = self.credentials["root_url"] + "/channel/file/" def _decrypt_credentials(self): From e9ad11927e884561a05ff954c82e6451f76ec1ef Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 18 Oct 2024 12:05:54 -0400 Subject: [PATCH 03/27] added this to mitigate error that arose while trying composition surveys; revisit this change --- watertap/tools/oli_api/flash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watertap/tools/oli_api/flash.py b/watertap/tools/oli_api/flash.py index 8a44b784a1..5353ab2437 100644 --- a/watertap/tools/oli_api/flash.py +++ b/watertap/tools/oli_api/flash.py @@ -972,7 +972,7 @@ def get_clone(self, flash_method, json_input, index, survey=None): pass elif k in d["inflows"]["values"]: d = d["inflows"]["values"] - elif k in d["corrosionParameters"]: + elif hasattr(d, "corrosionParameters") and (k in d["corrosionParameters"]): d = d["corrosionParameters"] else: _logger.warning(f"Survey key {k} not found in JSON input.") From 6a2d5c9e857aff4326ecdbbcdc7f6388f60bff60 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Wed, 30 Oct 2024 16:34:03 -0400 Subject: [PATCH 04/27] Revert "checkpoint: add scaling to cstr but didn't note any affect on BSM2-P solver iterations; also replacing aeration tank on bsm2-p instead of cstr_injection with flowsheet level vars and constraints" This reverts commit bd2caa8994dd515d1b9b9f306e62bb66418bedc2. --- .../BSM2_P_extension.py | 19 +++++-------------- watertap/unit_models/cstr.py | 17 ----------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/watertap/flowsheets/full_water_resource_recovery_facility/BSM2_P_extension.py b/watertap/flowsheets/full_water_resource_recovery_facility/BSM2_P_extension.py index 8742c652a6..51ea6ec635 100644 --- a/watertap/flowsheets/full_water_resource_recovery_facility/BSM2_P_extension.py +++ b/watertap/flowsheets/full_water_resource_recovery_facility/BSM2_P_extension.py @@ -22,7 +22,7 @@ import pyomo.environ as pyo from pyomo.network import Arc, SequentialDecomposition -from pyomo.util.calc_var_value import calculate_variable_from_constraint + from idaes.core import ( FlowsheetBlock, UnitModelCostingBlock, @@ -36,7 +36,6 @@ Mixer, PressureChanger, ) -from idaes.core.util.model_diagnostics import DiagnosticsToolbox from idaes.models.unit_models.separator import SplittingType from watertap.core.solvers import get_solver from idaes.core.util.model_statistics import degrees_of_freedom @@ -47,8 +46,6 @@ stream_table_dataframe_to_string, ) from watertap.unit_models.cstr_injection import CSTR_Injection - -from watertap.unit_models.aeration_tank import AerationTank from watertap.unit_models.clarifier import Clarifier from watertap.property_models.unit_specific.anaerobic_digestion.modified_adm1_properties import ( ModifiedADM1ParameterBlock, @@ -94,11 +91,10 @@ # Set up logger _log = idaeslog.getLogger(__name__) -_log.debug + def main(bio_P=False): m = build(bio_P=bio_P) - dt = DiagnosticsToolbox(m) set_operating_conditions(m) for mx in m.fs.mixers: @@ -148,7 +144,7 @@ def main(bio_P=False): display_costing(m) display_performance_metrics(m) - return m, results, dt + return m, results def build(bio_P=False): @@ -376,9 +372,6 @@ def build(bio_P=False): mutable=True, doc="Dissolved oxygen concentration at equilibrium", ) - m.fs.R5.KLa = 240 - m.fs.R6.KLa = 240 - m.fs.R7.KLa = 84 m.fs.aerobic_reactors = (m.fs.R5, m.fs.R6, m.fs.R7) for R in m.fs.aerobic_reactors: @@ -487,9 +480,7 @@ def set_operating_conditions(m): m.fs.R5.outlet.conc_mass_comp[:, "S_O2"].fix(1.91e-3) m.fs.R6.outlet.conc_mass_comp[:, "S_O2"].fix(2.60e-3) m.fs.R7.outlet.conc_mass_comp[:, "S_O2"].fix(3.20e-3) - # for R in m.fs.aerobic_reactors: - # calculate_variable_from_constraint(R.KLa, R.eq_mass_transfer[0]) - + # Set fraction of outflow from reactor 5 that goes to recycle m.fs.SP1.split_fraction[:, "underflow"].fix(0.60) @@ -961,7 +952,7 @@ def display_performance_metrics(m): if __name__ == "__main__": - m, results, dt = main(bio_P=False) + m, results = main(bio_P=False) stream_table = create_stream_table_dataframe( { diff --git a/watertap/unit_models/cstr.py b/watertap/unit_models/cstr.py index e31ffb65b2..d77ebb8b7d 100644 --- a/watertap/unit_models/cstr.py +++ b/watertap/unit_models/cstr.py @@ -21,7 +21,6 @@ from idaes.models.unit_models.cstr import CSTRData as CSTRIDAESData import idaes.logger as idaeslog -import idaes.core.util.scaling as iscale from pyomo.environ import ( Constraint, @@ -82,19 +81,3 @@ def CSTR_retention_time_rule(self, t): @property def default_costing_method(self): return cost_cstr - - def calculate_scaling_factors(self): - - super().calculate_scaling_factors() - - for t in self.flowsheet().time: - if iscale.get_scaling_factor(self.hydraulic_retention_time[t]) is None: - iscale.set_variable_scaling_from_current_value(self.hydraulic_retention_time[t]) - if iscale.get_scaling_factor(self.volume[t]) is None: - iscale.set_variable_scaling_from_current_value(self.volume[t]) - - sf = iscale.get_scaling_factor(self.hydraulic_retention_time[t]) - iscale.constraint_scaling_transform(self.CSTR_retention_time[t], sf) - - - From a6f6d4304a97f3620f8e6d8652bdac30141af916 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Wed, 30 Oct 2024 16:36:04 -0400 Subject: [PATCH 05/27] blk --- watertap/tools/oli_api/flash.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/watertap/tools/oli_api/flash.py b/watertap/tools/oli_api/flash.py index 5353ab2437..2e4d1c9284 100644 --- a/watertap/tools/oli_api/flash.py +++ b/watertap/tools/oli_api/flash.py @@ -972,7 +972,9 @@ def get_clone(self, flash_method, json_input, index, survey=None): pass elif k in d["inflows"]["values"]: d = d["inflows"]["values"] - elif hasattr(d, "corrosionParameters") and (k in d["corrosionParameters"]): + elif hasattr(d, "corrosionParameters") and ( + k in d["corrosionParameters"] + ): d = d["corrosionParameters"] else: _logger.warning(f"Survey key {k} not found in JSON input.") From 37e9a7aebcc6e6cd1a138fab4e8e23de26d56f52 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Thu, 31 Oct 2024 11:59:15 -0400 Subject: [PATCH 06/27] update logger messages to be more clear on uploading/generating/deleting files; fix issue related to session dbs file deletion on exit --- watertap/tools/oli_api/client.py | 18 ++++++++++++------ watertap/tools/oli_api/tests/test_client.py | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index 8dd557e68c..a14cb37d28 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -56,7 +56,7 @@ from pyomo.common.dependencies import attempt_import requests, requests_available = attempt_import("requests", defer_check=False) - +from requests.exceptions import JSONDecodeError from watertap.tools.oli_api.util.watertap_to_oli_helper_functions import get_oli_name @@ -103,6 +103,7 @@ def __enter__(self): # return False if no exceptions raised def __exit__(self, exc_type=None, exc_value=None, traceback=None): # delete all .dbs files created during session + _logger.info("Exiting: deleting all DBS files created during the session.") self.dbs_file_cleanup(self.session_dbs_files) return False @@ -133,7 +134,7 @@ def upload_dbs_file(self, dbs_file_path, keep_file=False): if bool(dbs_file_id): if not keep_file: self.session_dbs_files.append(dbs_file_id) - _logger.info(f"DBS file ID is {dbs_file_id}") + _logger.info(f"Uploaded DBS file ID is {dbs_file_id}") return dbs_file_id else: raise RuntimeError("Unexpected failure getting DBS file ID.") @@ -230,7 +231,7 @@ def generate_dbs_file( if bool(dbs_file_id): if not keep_file: self.session_dbs_files.append(dbs_file_id) - _logger.info(f"DBS file ID is {dbs_file_id}") + _logger.info(f"Generated DBS file ID is {dbs_file_id}") return dbs_file_id else: raise RuntimeError("Unexpected failure getting DBS file ID.") @@ -302,8 +303,14 @@ def dbs_file_cleanup(self, dbs_file_ids=None): headers=self.credential_manager.headers, ) req = _request_status_test(req, ["SUCCESS"]) + if req["status"] == "SUCCESS": - _logger.info(f"File deleted") + # Remove the file from session_dbs_files list if it is there, otherwise an error will occur upon exit when this method is called again and already deleted files will remain on the list for deletion. Thus, an error can occur if there is no existing ID to delete. + if dbs_file_id in self.session_dbs_files: + self.session_dbs_files.remove(dbs_file_id) + _logger.info( + f"File {dbs_file_id} deleted and removed from session_dbs_files list." + ) def get_corrosion_contact_surfaces(self, dbs_file_id): """ @@ -454,7 +461,7 @@ def _get_result_link(req_json): if "resultsLink" in req_json["data"]: result_link = req_json["data"]["resultsLink"] if not result_link: - raise RuntimeError(f"Failed to get 'resultsLink'. Response: {req.json()}") + raise RuntimeError(f"Failed to get 'resultsLink'. Response: {req_json.json()}") return result_link @@ -467,7 +474,6 @@ def _request_status_test(req, target_keys): :return req_json: response object converted to JSON """ - req_json = req.json() func_name = sys._getframe().f_back.f_code.co_name _logger.debug(f"{func_name} response: {req_json}") diff --git a/watertap/tools/oli_api/tests/test_client.py b/watertap/tools/oli_api/tests/test_client.py index 2d3d1e07ab..973de63d33 100644 --- a/watertap/tools/oli_api/tests/test_client.py +++ b/watertap/tools/oli_api/tests/test_client.py @@ -58,6 +58,7 @@ def test_dbs_file_available_for_testing(local_dbs_file: Path): @pytest.mark.unit def test_dbs_file_cleanup(oliapi_instance: OLIApi, local_dbs_file: Path): + # The following line will return 3 DBS file IDs. Note, the same file is uploaded three times, but each time a new ID is assigned. ids = [oliapi_instance.upload_dbs_file(str(local_dbs_file)) for i in range(3)] oliapi_instance.dbs_file_cleanup(ids) From b6357515e922b73f3738e846b8e9f5bb3e918de4 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Thu, 31 Oct 2024 12:25:16 -0400 Subject: [PATCH 07/27] remove import --- watertap/tools/oli_api/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index a14cb37d28..d8c124cae8 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -56,7 +56,6 @@ from pyomo.common.dependencies import attempt_import requests, requests_available = attempt_import("requests", defer_check=False) -from requests.exceptions import JSONDecodeError from watertap.tools.oli_api.util.watertap_to_oli_helper_functions import get_oli_name From 71cc4e73d9cc17050956221b92f846d52b1c014e Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Thu, 31 Oct 2024 13:02:58 -0400 Subject: [PATCH 08/27] try inspecting notebook --- tutorials/oli_calculations.ipynb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tutorials/oli_calculations.ipynb b/tutorials/oli_calculations.ipynb index 78190e698a..76d2ab5f62 100644 --- a/tutorials/oli_calculations.ipynb +++ b/tutorials/oli_calculations.ipynb @@ -411,13 +411,14 @@ " dbs_files = oliapi.get_user_dbs_file_ids()\n", " \n", " # uncomment the following lines to view results:\n", - " #for idx, file in enumerate(dbs_files):\n", - " # print(f\"{idx+1}\\t{file}\")\n", + " for idx, file in enumerate(dbs_files):\n", + " print(f\"{idx+1}\\t{file}\")\n", " \n", " # if a DBS file has been retained from a previous session,\n", " # its flash history and chemistry information can be summarized.\n", " file_summary = oliapi.get_dbs_file_summary(dbs_file_id)\n", - "\n", + " print(file_summary)\n", + " \n", " # save chemistry information\n", " chemistry_info = file_summary[\"chemistry_info\"]\n", " write_output(chemistry_info[\"result\"], \"chemistry_info.json\")\n", From 5aa1e95973e8fe4844ef2f2b9820220166a90963 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Thu, 31 Oct 2024 13:55:45 -0400 Subject: [PATCH 09/27] attempt to resolve notebook error, clear up logger messages --- tutorials/oli_calculations.ipynb | 3 ++- watertap/tools/oli_api/client.py | 21 ++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tutorials/oli_calculations.ipynb b/tutorials/oli_calculations.ipynb index 76d2ab5f62..4883955da3 100644 --- a/tutorials/oli_calculations.ipynb +++ b/tutorials/oli_calculations.ipynb @@ -421,7 +421,8 @@ " \n", " # save chemistry information\n", " chemistry_info = file_summary[\"chemistry_info\"]\n", - " write_output(chemistry_info[\"result\"], \"chemistry_info.json\")\n", + " # Uncomment next line to save as json file\n", + " # write_output(chemistry_info[\"result\"], \"chemistry_info.json\")\n", " \n", " # DBS files can also be deleted.\n", " oliapi.dbs_file_cleanup(dbs_files)\n", diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index d8c124cae8..076578759e 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -102,14 +102,16 @@ def __enter__(self): # return False if no exceptions raised def __exit__(self, exc_type=None, exc_value=None, traceback=None): # delete all .dbs files created during session - _logger.info("Exiting: deleting all DBS files created during the session.") + _logger.info(f"Exiting: deleting {len(self.session_dbs_files)} remaining DBS files created during the session that were not marked by keep_file=True.") self.dbs_file_cleanup(self.session_dbs_files) return False def _prompt(self, msg, default=""): if self.interactive_mode: + msg = msg + "Enter [y]/n to proceed." return input(msg) else: + msg = msg + "To choose [y]/n to this action, set interactive_mode=True." _logger.info(msg) return default @@ -289,9 +291,14 @@ def dbs_file_cleanup(self, dbs_file_ids=None): """ if dbs_file_ids is None: + _logger.info("No DBS file IDs were provided to the dbs_file_cleanup method. Checking user's cloud account for DBS file IDs.") dbs_file_ids = self.get_user_dbs_file_ids() + if not len(dbs_file_ids): + _logger.info("No DBS file IDs were found on the user's cloud account.") + return + r = self._prompt( - f"WaterTAP will delete {len(dbs_file_ids)} DBS files [y]/n ", "y" + f"WaterTAP will delete {len(dbs_file_ids)} DBS files. ", "y" ) if (r.lower() == "y") or (r == ""): for dbs_file_id in dbs_file_ids: @@ -307,9 +314,13 @@ def dbs_file_cleanup(self, dbs_file_ids=None): # Remove the file from session_dbs_files list if it is there, otherwise an error will occur upon exit when this method is called again and already deleted files will remain on the list for deletion. Thus, an error can occur if there is no existing ID to delete. if dbs_file_id in self.session_dbs_files: self.session_dbs_files.remove(dbs_file_id) - _logger.info( - f"File {dbs_file_id} deleted and removed from session_dbs_files list." - ) + _logger.info( + f"File {dbs_file_id} deleted and removed from session_dbs_files list." + ) + else: + _logger.info( + f"File {dbs_file_id} deleted." + ) def get_corrosion_contact_surfaces(self, dbs_file_id): """ From 7b5288bc4bd5a66624ab7dbddc90bbd2a920451b Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Thu, 31 Oct 2024 14:35:12 -0400 Subject: [PATCH 10/27] temporarily comment out failing survey tests --- watertap/tools/oli_api/tests/test_flash.py | 96 +++++++++++----------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/watertap/tools/oli_api/tests/test_flash.py b/watertap/tools/oli_api/tests/test_flash.py index 706896d176..6a7dc406f7 100644 --- a/watertap/tools/oli_api/tests/test_flash.py +++ b/watertap/tools/oli_api/tests/test_flash.py @@ -72,28 +72,28 @@ def test_water_analysis_single_point( ) -@pytest.mark.unit -def test_water_analysis_survey( - flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path -): - dbs_file_id = oliapi_instance.session_dbs_files[-1] - survey = build_survey( - { - "Na_+": linspace(0, 1e4, 2), - "temperature": linspace(0, 70, 2), - }, - get_oli_names=True, - file_name=tmp_path / "test_survey", - ) - stream_input = flash_instance.configure_water_analysis(source_water) - stream_output = flash_instance.run_flash( - "wateranalysis", - oliapi_instance, - dbs_file_id, - stream_input, - survey, - file_name=tmp_path / "test_wa_survey", - ) +# @pytest.mark.unit +# def test_water_analysis_survey( +# flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path +# ): +# dbs_file_id = oliapi_instance.session_dbs_files[-1] +# survey = build_survey( +# { +# "Na_+": linspace(0, 1e4, 2), +# "temperature": linspace(0, 70, 2), +# }, +# get_oli_names=True, +# file_name=tmp_path / "test_survey", +# ) +# stream_input = flash_instance.configure_water_analysis(source_water) +# stream_output = flash_instance.run_flash( +# "wateranalysis", +# oliapi_instance, +# dbs_file_id, +# stream_input, +# survey, +# file_name=tmp_path / "test_wa_survey", +# ) @pytest.mark.unit @@ -116,32 +116,32 @@ def test_isothermal_flash_single_point( ) -def test_isothermal_flash_survey( - flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path -): - dbs_file_id = oliapi_instance.session_dbs_files[-1] - survey = build_survey( - { - "NaCl": linspace(0, 1e4, 2), - "temperature": linspace(0, 70, 2), - }, - get_oli_names=True, - file_name=tmp_path / "test_survey", - ) - dbs_file_id = oliapi_instance.session_dbs_files[-1] - stream_input = flash_instance.configure_water_analysis(source_water) - inflows = flash_instance.get_apparent_species_from_true( - stream_input, - oliapi_instance, - dbs_file_id, - ) - isothermal_input = flash_instance.configure_flash_analysis(inflows, "isothermal") - isothermal_output = flash_instance.run_flash( - "isothermal", - oliapi_instance, - dbs_file_id, - isothermal_input, - ) +# def test_isothermal_flash_survey( +# flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path +# ): +# dbs_file_id = oliapi_instance.session_dbs_files[-1] +# survey = build_survey( +# { +# "NaCl": linspace(0, 1e4, 2), +# "temperature": linspace(0, 70, 2), +# }, +# get_oli_names=True, +# file_name=tmp_path / "test_survey", +# ) +# dbs_file_id = oliapi_instance.session_dbs_files[-1] +# stream_input = flash_instance.configure_water_analysis(source_water) +# inflows = flash_instance.get_apparent_species_from_true( +# stream_input, +# oliapi_instance, +# dbs_file_id, +# ) +# isothermal_input = flash_instance.configure_flash_analysis(inflows, "isothermal") +# isothermal_output = flash_instance.run_flash( +# "isothermal", +# oliapi_instance, +# dbs_file_id, +# isothermal_input, +# ) @pytest.mark.unit From 116fcdb10401e3daa56415c9233eca7945e93ac3 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Thu, 31 Oct 2024 16:10:53 -0400 Subject: [PATCH 11/27] unclear why survey tests fail on GH when they pass locally with same credentials used; bring tests back --- watertap/tools/oli_api/tests/test_flash.py | 96 +++++++++++----------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/watertap/tools/oli_api/tests/test_flash.py b/watertap/tools/oli_api/tests/test_flash.py index 6a7dc406f7..706896d176 100644 --- a/watertap/tools/oli_api/tests/test_flash.py +++ b/watertap/tools/oli_api/tests/test_flash.py @@ -72,28 +72,28 @@ def test_water_analysis_single_point( ) -# @pytest.mark.unit -# def test_water_analysis_survey( -# flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path -# ): -# dbs_file_id = oliapi_instance.session_dbs_files[-1] -# survey = build_survey( -# { -# "Na_+": linspace(0, 1e4, 2), -# "temperature": linspace(0, 70, 2), -# }, -# get_oli_names=True, -# file_name=tmp_path / "test_survey", -# ) -# stream_input = flash_instance.configure_water_analysis(source_water) -# stream_output = flash_instance.run_flash( -# "wateranalysis", -# oliapi_instance, -# dbs_file_id, -# stream_input, -# survey, -# file_name=tmp_path / "test_wa_survey", -# ) +@pytest.mark.unit +def test_water_analysis_survey( + flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path +): + dbs_file_id = oliapi_instance.session_dbs_files[-1] + survey = build_survey( + { + "Na_+": linspace(0, 1e4, 2), + "temperature": linspace(0, 70, 2), + }, + get_oli_names=True, + file_name=tmp_path / "test_survey", + ) + stream_input = flash_instance.configure_water_analysis(source_water) + stream_output = flash_instance.run_flash( + "wateranalysis", + oliapi_instance, + dbs_file_id, + stream_input, + survey, + file_name=tmp_path / "test_wa_survey", + ) @pytest.mark.unit @@ -116,32 +116,32 @@ def test_isothermal_flash_single_point( ) -# def test_isothermal_flash_survey( -# flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path -# ): -# dbs_file_id = oliapi_instance.session_dbs_files[-1] -# survey = build_survey( -# { -# "NaCl": linspace(0, 1e4, 2), -# "temperature": linspace(0, 70, 2), -# }, -# get_oli_names=True, -# file_name=tmp_path / "test_survey", -# ) -# dbs_file_id = oliapi_instance.session_dbs_files[-1] -# stream_input = flash_instance.configure_water_analysis(source_water) -# inflows = flash_instance.get_apparent_species_from_true( -# stream_input, -# oliapi_instance, -# dbs_file_id, -# ) -# isothermal_input = flash_instance.configure_flash_analysis(inflows, "isothermal") -# isothermal_output = flash_instance.run_flash( -# "isothermal", -# oliapi_instance, -# dbs_file_id, -# isothermal_input, -# ) +def test_isothermal_flash_survey( + flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path +): + dbs_file_id = oliapi_instance.session_dbs_files[-1] + survey = build_survey( + { + "NaCl": linspace(0, 1e4, 2), + "temperature": linspace(0, 70, 2), + }, + get_oli_names=True, + file_name=tmp_path / "test_survey", + ) + dbs_file_id = oliapi_instance.session_dbs_files[-1] + stream_input = flash_instance.configure_water_analysis(source_water) + inflows = flash_instance.get_apparent_species_from_true( + stream_input, + oliapi_instance, + dbs_file_id, + ) + isothermal_input = flash_instance.configure_flash_analysis(inflows, "isothermal") + isothermal_output = flash_instance.run_flash( + "isothermal", + oliapi_instance, + dbs_file_id, + isothermal_input, + ) @pytest.mark.unit From 8041b20685d5971f854f2347d5701623775f58d3 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 09:22:27 -0400 Subject: [PATCH 12/27] temporarily comment out dbs file cleanup upon exit --- watertap/tools/oli_api/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index 076578759e..a86bd44352 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -103,7 +103,7 @@ def __enter__(self): def __exit__(self, exc_type=None, exc_value=None, traceback=None): # delete all .dbs files created during session _logger.info(f"Exiting: deleting {len(self.session_dbs_files)} remaining DBS files created during the session that were not marked by keep_file=True.") - self.dbs_file_cleanup(self.session_dbs_files) + # self.dbs_file_cleanup(self.session_dbs_files) return False def _prompt(self, msg, default=""): From 7deba9a7c024eb31129763088d947e009c01826c Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 12:29:52 -0400 Subject: [PATCH 13/27] bring back session_dbs_files deletion --- watertap/tools/oli_api/client.py | 2 +- watertap/tools/oli_api/credentials.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index a86bd44352..076578759e 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -103,7 +103,7 @@ def __enter__(self): def __exit__(self, exc_type=None, exc_value=None, traceback=None): # delete all .dbs files created during session _logger.info(f"Exiting: deleting {len(self.session_dbs_files)} remaining DBS files created during the session that were not marked by keep_file=True.") - # self.dbs_file_cleanup(self.session_dbs_files) + self.dbs_file_cleanup(self.session_dbs_files) return False def _prompt(self, msg, default=""): diff --git a/watertap/tools/oli_api/credentials.py b/watertap/tools/oli_api/credentials.py index 002230de76..3e2e815fc1 100644 --- a/watertap/tools/oli_api/credentials.py +++ b/watertap/tools/oli_api/credentials.py @@ -89,6 +89,7 @@ def __init__( access_keys=[], test=False, interactive_mode=True, + refresh=False, ): """ Manages credentials for OLIApi authentication requests. @@ -102,6 +103,7 @@ def __init__( :param access_keys: list of access keys generated by user :param test: bool switch for automation during tests :param interactive_mode: bool to switch level of logging display from info to debug only + :param refresh: bool to use refresh token in login method """ self.test = test @@ -121,6 +123,8 @@ def __init__( _logger.setLevel(logging.DEBUG) self.set_headers() + self.refresh = refresh + def set_headers(self): """ Creates OLI Cloud API headers and performs initial login. @@ -336,12 +340,10 @@ def delete_oliapi_access_key(self, api_key): _logger.info(response.text) return response.text - def login(self, refresh=False): + def login(self): """ Log in to OLI Cloud using access key or credentials. - :param refresh: bool to get refresh token - :return status: bool indicating success or failure """ @@ -356,7 +358,7 @@ def login(self, refresh=False): ) else: _logger.info("Logging into OLI API using username and password") - if refresh: + if self.refresh: body = { "refresh_token": self.refresh_token, "grant_type": "refresh_token", From 8acaddad12a9c0f9ebddc8da11048fb78e0072b4 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 13:15:38 -0400 Subject: [PATCH 14/27] update tests with suspected fix regarding dbs file usage and also try to troubleshoot errors after switch to sequential test deployment --- watertap/tools/oli_api/client.py | 3 ++- watertap/tools/oli_api/conftest.py | 6 ++++-- watertap/tools/oli_api/tests/test_client.py | 5 +++++ watertap/tools/oli_api/tests/test_flash.py | 11 +++++------ 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index 076578759e..8c907987c4 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -487,6 +487,7 @@ def _request_status_test(req, target_keys): req_json = req.json() func_name = sys._getframe().f_back.f_code.co_name _logger.debug(f"{func_name} response: {req_json}") + if req.status_code == 200: if target_keys: if "status" in req_json: @@ -494,7 +495,7 @@ def _request_status_test(req, target_keys): return req_json else: return req_json - raise RuntimeError(f"Failure in {func_name}. Response: {req_json}") + raise RuntimeError(f"Failure in {func_name}. Response: {req_json}. Status Code: {req.status_code}, Response Object Keys: {req_json.keys()}") def _poll_result_link(result_link, headers, max_request, poll_time): diff --git a/watertap/tools/oli_api/conftest.py b/watertap/tools/oli_api/conftest.py index 223c3e6ab5..e97fb8fb94 100644 --- a/watertap/tools/oli_api/conftest.py +++ b/watertap/tools/oli_api/conftest.py @@ -93,11 +93,13 @@ def oliapi_instance( credentials = { **auth_credentials, "config_file": cred_file_path, + "refresh": True, + "interactive_mode": False, } credential_manager = CredentialManager(**credentials, test=True) with OLIApi(credential_manager, interactive_mode=False) as oliapi: - oliapi.upload_dbs_file(str(local_dbs_file)) - oliapi.generate_dbs_file(source_water) + # oliapi.upload_dbs_file(str(local_dbs_file)) + # oliapi.generate_dbs_file(source_water) yield oliapi with contextlib.suppress(FileNotFoundError): cred_file_path.unlink() diff --git a/watertap/tools/oli_api/tests/test_client.py b/watertap/tools/oli_api/tests/test_client.py index 973de63d33..e34605492a 100644 --- a/watertap/tools/oli_api/tests/test_client.py +++ b/watertap/tools/oli_api/tests/test_client.py @@ -55,9 +55,14 @@ def test_dbs_file_available_for_testing(local_dbs_file: Path): assert local_dbs_file.is_file() +@pytest.mark.unit +def test_generate_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) + assert len(dbs_file_id) > 0 @pytest.mark.unit def test_dbs_file_cleanup(oliapi_instance: OLIApi, local_dbs_file: Path): + # This test checks both the upload_dbs_file method and dbs_file_cleanup method # The following line will return 3 DBS file IDs. Note, the same file is uploaded three times, but each time a new ID is assigned. ids = [oliapi_instance.upload_dbs_file(str(local_dbs_file)) for i in range(3)] oliapi_instance.dbs_file_cleanup(ids) diff --git a/watertap/tools/oli_api/tests/test_flash.py b/watertap/tools/oli_api/tests/test_flash.py index 706896d176..4dc85337c7 100644 --- a/watertap/tools/oli_api/tests/test_flash.py +++ b/watertap/tools/oli_api/tests/test_flash.py @@ -58,7 +58,7 @@ def test_water_analysis_single_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) stream_input = flash_instance.configure_water_analysis( source_water, file_name=tmp_path / "test_wa_input", @@ -76,7 +76,7 @@ def test_water_analysis_single_point( def test_water_analysis_survey( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) survey = build_survey( { "Na_+": linspace(0, 1e4, 2), @@ -100,7 +100,7 @@ def test_water_analysis_survey( def test_isothermal_flash_single_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( stream_input, @@ -119,7 +119,7 @@ def test_isothermal_flash_single_point( def test_isothermal_flash_survey( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) survey = build_survey( { "NaCl": linspace(0, 1e4, 2), @@ -128,7 +128,6 @@ def test_isothermal_flash_survey( get_oli_names=True, file_name=tmp_path / "test_survey", ) - dbs_file_id = oliapi_instance.session_dbs_files[-1] stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( stream_input, @@ -148,7 +147,7 @@ def test_isothermal_flash_survey( def test_bubble_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( From ea309a1d6b6e6f6ecdb1cf7d6e704f92bb1c9fa6 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 13:42:22 -0400 Subject: [PATCH 15/27] revise tests --- watertap/tools/oli_api/client.py | 5 +++-- watertap/tools/oli_api/tests/test_client.py | 6 +++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index 8c907987c4..b2f8f68562 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -93,6 +93,9 @@ def __init__(self, credential_manager, interactive_mode=True, debug_level="INFO" _logger.setLevel(logging.INFO) else: _logger.setLevel(logging.DEBUG) + + # TODO: unknown bug where only "liquid1" phase is found in Flash analysis + self.valid_phases = ["liquid1", "vapor", "solid", "liquid2"] # binds OLIApi instance to context manager def __enter__(self): @@ -193,8 +196,6 @@ def generate_dbs_file( else: dbs_file_inputs["modelName"] = "OLI_analysis" - # TODO: unknown bug where only "liquid1" phase is found in Flash analysis - self.valid_phases = ["liquid1", "vapor", "solid", "liquid2"] if phases is not None: invalid_phases = [p for p in phases if p not in self.valid_phases] if invalid_phases: diff --git a/watertap/tools/oli_api/tests/test_client.py b/watertap/tools/oli_api/tests/test_client.py index e34605492a..0cc812447e 100644 --- a/watertap/tools/oli_api/tests/test_client.py +++ b/watertap/tools/oli_api/tests/test_client.py @@ -60,6 +60,11 @@ def test_generate_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source dbs_file_id = oliapi_instance.generate_dbs_file(source_water) assert len(dbs_file_id) > 0 +@pytest.mark.unit +def test_upload_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): + dbs_file_id = oliapi_instance.upload_dbs_file(str(local_dbs_file)) + assert len(dbs_file_id) > 0 + @pytest.mark.unit def test_dbs_file_cleanup(oliapi_instance: OLIApi, local_dbs_file: Path): # This test checks both the upload_dbs_file method and dbs_file_cleanup method @@ -81,7 +86,6 @@ def test_valid_phases(oliapi_instance: OLIApi): for v in oliapi_instance.valid_phases: assert v in valid_phases - @pytest.mark.unit def test_invalid_phases(oliapi_instance_with_invalid_phase: OLIApi): oliapi_instance_with_invalid_phase From 59766c39105dced9ab68336efb4d13f51389f067 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 13:50:55 -0400 Subject: [PATCH 16/27] tests passed after pushing last commit! let's see if reverting this change would break them again to identify what the solution was --- watertap/tools/oli_api/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watertap/tools/oli_api/conftest.py b/watertap/tools/oli_api/conftest.py index e97fb8fb94..d5eed025ff 100644 --- a/watertap/tools/oli_api/conftest.py +++ b/watertap/tools/oli_api/conftest.py @@ -93,7 +93,7 @@ def oliapi_instance( credentials = { **auth_credentials, "config_file": cred_file_path, - "refresh": True, + "refresh": False, "interactive_mode": False, } credential_manager = CredentialManager(**credentials, test=True) From 24485e7670864e2ba40348e9438099e64ee1c213 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 13:59:43 -0400 Subject: [PATCH 17/27] bring back refresh since it seems that wasn't the solution (alone) and bring back a change that I ultimately want to remove but think might have been part of prob --- watertap/tools/oli_api/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/watertap/tools/oli_api/conftest.py b/watertap/tools/oli_api/conftest.py index d5eed025ff..e21b6cd60e 100644 --- a/watertap/tools/oli_api/conftest.py +++ b/watertap/tools/oli_api/conftest.py @@ -93,12 +93,12 @@ def oliapi_instance( credentials = { **auth_credentials, "config_file": cred_file_path, - "refresh": False, + "refresh": True, "interactive_mode": False, } credential_manager = CredentialManager(**credentials, test=True) with OLIApi(credential_manager, interactive_mode=False) as oliapi: - # oliapi.upload_dbs_file(str(local_dbs_file)) + oliapi.upload_dbs_file(str(local_dbs_file)) # oliapi.generate_dbs_file(source_water) yield oliapi with contextlib.suppress(FileNotFoundError): From 363d652f7f007ddfc378f0f2bcc748faa8b23d63 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 14:09:54 -0400 Subject: [PATCH 18/27] undoing more changes to see which lines resolved the issue --- watertap/tools/oli_api/conftest.py | 2 +- watertap/tools/oli_api/tests/test_flash.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/watertap/tools/oli_api/conftest.py b/watertap/tools/oli_api/conftest.py index e21b6cd60e..079708a578 100644 --- a/watertap/tools/oli_api/conftest.py +++ b/watertap/tools/oli_api/conftest.py @@ -99,7 +99,7 @@ def oliapi_instance( credential_manager = CredentialManager(**credentials, test=True) with OLIApi(credential_manager, interactive_mode=False) as oliapi: oliapi.upload_dbs_file(str(local_dbs_file)) - # oliapi.generate_dbs_file(source_water) + oliapi.generate_dbs_file(source_water) yield oliapi with contextlib.suppress(FileNotFoundError): cred_file_path.unlink() diff --git a/watertap/tools/oli_api/tests/test_flash.py b/watertap/tools/oli_api/tests/test_flash.py index 4dc85337c7..953b030f1a 100644 --- a/watertap/tools/oli_api/tests/test_flash.py +++ b/watertap/tools/oli_api/tests/test_flash.py @@ -147,7 +147,7 @@ def test_isothermal_flash_survey( def test_bubble_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.generate_dbs_file(source_water) + dbs_file_id = oliapi_instance.session_dbs_files[-1] stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( From 4a378e8c830ab2eb35485dcef399d407b62c96fc Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 14:27:16 -0400 Subject: [PATCH 19/27] bringing back changes that I suspect were the true culprit before --- watertap/tools/oli_api/tests/test_flash.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/watertap/tools/oli_api/tests/test_flash.py b/watertap/tools/oli_api/tests/test_flash.py index 953b030f1a..706896d176 100644 --- a/watertap/tools/oli_api/tests/test_flash.py +++ b/watertap/tools/oli_api/tests/test_flash.py @@ -58,7 +58,7 @@ def test_water_analysis_single_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.generate_dbs_file(source_water) + dbs_file_id = oliapi_instance.session_dbs_files[-1] stream_input = flash_instance.configure_water_analysis( source_water, file_name=tmp_path / "test_wa_input", @@ -76,7 +76,7 @@ def test_water_analysis_single_point( def test_water_analysis_survey( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.generate_dbs_file(source_water) + dbs_file_id = oliapi_instance.session_dbs_files[-1] survey = build_survey( { "Na_+": linspace(0, 1e4, 2), @@ -100,7 +100,7 @@ def test_water_analysis_survey( def test_isothermal_flash_single_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.generate_dbs_file(source_water) + dbs_file_id = oliapi_instance.session_dbs_files[-1] stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( stream_input, @@ -119,7 +119,7 @@ def test_isothermal_flash_single_point( def test_isothermal_flash_survey( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.generate_dbs_file(source_water) + dbs_file_id = oliapi_instance.session_dbs_files[-1] survey = build_survey( { "NaCl": linspace(0, 1e4, 2), @@ -128,6 +128,7 @@ def test_isothermal_flash_survey( get_oli_names=True, file_name=tmp_path / "test_survey", ) + dbs_file_id = oliapi_instance.session_dbs_files[-1] stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( stream_input, From a9b9fe66b5ce540a419fe9a0e5ab62441a751e29 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 14:37:13 -0400 Subject: [PATCH 20/27] set refresh False again --- watertap/tools/oli_api/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watertap/tools/oli_api/conftest.py b/watertap/tools/oli_api/conftest.py index 079708a578..e020154351 100644 --- a/watertap/tools/oli_api/conftest.py +++ b/watertap/tools/oli_api/conftest.py @@ -93,7 +93,7 @@ def oliapi_instance( credentials = { **auth_credentials, "config_file": cred_file_path, - "refresh": True, + "refresh": False, "interactive_mode": False, } credential_manager = CredentialManager(**credentials, test=True) From 8ca5f216557a5458b3e5b565e204ca7538a0a520 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 14:50:34 -0400 Subject: [PATCH 21/27] undoing more to identify issue --- watertap/tools/oli_api/tests/test_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/watertap/tools/oli_api/tests/test_client.py b/watertap/tools/oli_api/tests/test_client.py index 0cc812447e..d652366a92 100644 --- a/watertap/tools/oli_api/tests/test_client.py +++ b/watertap/tools/oli_api/tests/test_client.py @@ -55,15 +55,15 @@ def test_dbs_file_available_for_testing(local_dbs_file: Path): assert local_dbs_file.is_file() -@pytest.mark.unit -def test_generate_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): - dbs_file_id = oliapi_instance.generate_dbs_file(source_water) - assert len(dbs_file_id) > 0 +# @pytest.mark.unit +# def test_generate_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): +# dbs_file_id = oliapi_instance.generate_dbs_file(source_water) +# assert len(dbs_file_id) > 0 -@pytest.mark.unit -def test_upload_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): - dbs_file_id = oliapi_instance.upload_dbs_file(str(local_dbs_file)) - assert len(dbs_file_id) > 0 +# @pytest.mark.unit +# def test_upload_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): +# dbs_file_id = oliapi_instance.upload_dbs_file(str(local_dbs_file)) +# assert len(dbs_file_id) > 0 @pytest.mark.unit def test_dbs_file_cleanup(oliapi_instance: OLIApi, local_dbs_file: Path): From a5a72154700dc87d501bdac80f3eebf01d1e0230 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 14:52:41 -0400 Subject: [PATCH 22/27] seems that tests fail when refresh=False and conftest changes are reverted for oliapi_instance, along with point to last session dbs file in test_flash; now will set refresh=True on oliapi_instance in conftest --- watertap/tools/oli_api/conftest.py | 2 +- watertap/tools/oli_api/tests/test_client.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/watertap/tools/oli_api/conftest.py b/watertap/tools/oli_api/conftest.py index e020154351..079708a578 100644 --- a/watertap/tools/oli_api/conftest.py +++ b/watertap/tools/oli_api/conftest.py @@ -93,7 +93,7 @@ def oliapi_instance( credentials = { **auth_credentials, "config_file": cred_file_path, - "refresh": False, + "refresh": True, "interactive_mode": False, } credential_manager = CredentialManager(**credentials, test=True) diff --git a/watertap/tools/oli_api/tests/test_client.py b/watertap/tools/oli_api/tests/test_client.py index d652366a92..0cc812447e 100644 --- a/watertap/tools/oli_api/tests/test_client.py +++ b/watertap/tools/oli_api/tests/test_client.py @@ -55,15 +55,15 @@ def test_dbs_file_available_for_testing(local_dbs_file: Path): assert local_dbs_file.is_file() -# @pytest.mark.unit -# def test_generate_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): -# dbs_file_id = oliapi_instance.generate_dbs_file(source_water) -# assert len(dbs_file_id) > 0 +@pytest.mark.unit +def test_generate_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) + assert len(dbs_file_id) > 0 -# @pytest.mark.unit -# def test_upload_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): -# dbs_file_id = oliapi_instance.upload_dbs_file(str(local_dbs_file)) -# assert len(dbs_file_id) > 0 +@pytest.mark.unit +def test_upload_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): + dbs_file_id = oliapi_instance.upload_dbs_file(str(local_dbs_file)) + assert len(dbs_file_id) > 0 @pytest.mark.unit def test_dbs_file_cleanup(oliapi_instance: OLIApi, local_dbs_file: Path): From dd88d627314b940195adb3336875a89c26f31c80 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 15:35:11 -0400 Subject: [PATCH 23/27] bring back all changes that led to success --- watertap/tools/oli_api/conftest.py | 2 -- watertap/tools/oli_api/tests/test_flash.py | 11 +++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/watertap/tools/oli_api/conftest.py b/watertap/tools/oli_api/conftest.py index 079708a578..573152f7ba 100644 --- a/watertap/tools/oli_api/conftest.py +++ b/watertap/tools/oli_api/conftest.py @@ -98,8 +98,6 @@ def oliapi_instance( } credential_manager = CredentialManager(**credentials, test=True) with OLIApi(credential_manager, interactive_mode=False) as oliapi: - oliapi.upload_dbs_file(str(local_dbs_file)) - oliapi.generate_dbs_file(source_water) yield oliapi with contextlib.suppress(FileNotFoundError): cred_file_path.unlink() diff --git a/watertap/tools/oli_api/tests/test_flash.py b/watertap/tools/oli_api/tests/test_flash.py index 706896d176..4dc85337c7 100644 --- a/watertap/tools/oli_api/tests/test_flash.py +++ b/watertap/tools/oli_api/tests/test_flash.py @@ -58,7 +58,7 @@ def test_water_analysis_single_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) stream_input = flash_instance.configure_water_analysis( source_water, file_name=tmp_path / "test_wa_input", @@ -76,7 +76,7 @@ def test_water_analysis_single_point( def test_water_analysis_survey( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) survey = build_survey( { "Na_+": linspace(0, 1e4, 2), @@ -100,7 +100,7 @@ def test_water_analysis_survey( def test_isothermal_flash_single_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( stream_input, @@ -119,7 +119,7 @@ def test_isothermal_flash_single_point( def test_isothermal_flash_survey( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) survey = build_survey( { "NaCl": linspace(0, 1e4, 2), @@ -128,7 +128,6 @@ def test_isothermal_flash_survey( get_oli_names=True, file_name=tmp_path / "test_survey", ) - dbs_file_id = oliapi_instance.session_dbs_files[-1] stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( stream_input, @@ -148,7 +147,7 @@ def test_isothermal_flash_survey( def test_bubble_point( flash_instance: Flash, source_water: dict, oliapi_instance: OLIApi, tmp_path: Path ): - dbs_file_id = oliapi_instance.session_dbs_files[-1] + dbs_file_id = oliapi_instance.generate_dbs_file(source_water) stream_input = flash_instance.configure_water_analysis(source_water) inflows = flash_instance.get_apparent_species_from_true( From 0e108334cbeb0ebad11776a7ee16f466304e947f Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 16:07:01 -0400 Subject: [PATCH 24/27] undo change to exception message --- watertap/tools/oli_api/client.py | 2 +- watertap/tools/oli_api/conftest.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index b2f8f68562..6aa6726aee 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -496,7 +496,7 @@ def _request_status_test(req, target_keys): return req_json else: return req_json - raise RuntimeError(f"Failure in {func_name}. Response: {req_json}. Status Code: {req.status_code}, Response Object Keys: {req_json.keys()}") + raise RuntimeError(f"Failure in {func_name}. Response: {req_json}. Status Code: {req.status_code}.") def _poll_result_link(result_link, headers, max_request, poll_time): diff --git a/watertap/tools/oli_api/conftest.py b/watertap/tools/oli_api/conftest.py index 573152f7ba..ba39fdf090 100644 --- a/watertap/tools/oli_api/conftest.py +++ b/watertap/tools/oli_api/conftest.py @@ -82,8 +82,6 @@ def auth_credentials() -> dict: def oliapi_instance( tmp_path: Path, auth_credentials: dict, - local_dbs_file: Path, - source_water: dict, ) -> OLIApi: if not cryptography_available: From faf0d108a56f630a4f952fc2a74a5eb180e84f49 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Fri, 1 Nov 2024 16:07:44 -0400 Subject: [PATCH 25/27] run black --- watertap/tools/oli_api/client.py | 22 +++++++++++---------- watertap/tools/oli_api/tests/test_client.py | 12 +++++++++-- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/watertap/tools/oli_api/client.py b/watertap/tools/oli_api/client.py index 6aa6726aee..bd2746dd01 100644 --- a/watertap/tools/oli_api/client.py +++ b/watertap/tools/oli_api/client.py @@ -93,7 +93,7 @@ def __init__(self, credential_manager, interactive_mode=True, debug_level="INFO" _logger.setLevel(logging.INFO) else: _logger.setLevel(logging.DEBUG) - + # TODO: unknown bug where only "liquid1" phase is found in Flash analysis self.valid_phases = ["liquid1", "vapor", "solid", "liquid2"] @@ -105,7 +105,9 @@ def __enter__(self): # return False if no exceptions raised def __exit__(self, exc_type=None, exc_value=None, traceback=None): # delete all .dbs files created during session - _logger.info(f"Exiting: deleting {len(self.session_dbs_files)} remaining DBS files created during the session that were not marked by keep_file=True.") + _logger.info( + f"Exiting: deleting {len(self.session_dbs_files)} remaining DBS files created during the session that were not marked by keep_file=True." + ) self.dbs_file_cleanup(self.session_dbs_files) return False @@ -292,15 +294,15 @@ def dbs_file_cleanup(self, dbs_file_ids=None): """ if dbs_file_ids is None: - _logger.info("No DBS file IDs were provided to the dbs_file_cleanup method. Checking user's cloud account for DBS file IDs.") + _logger.info( + "No DBS file IDs were provided to the dbs_file_cleanup method. Checking user's cloud account for DBS file IDs." + ) dbs_file_ids = self.get_user_dbs_file_ids() if not len(dbs_file_ids): _logger.info("No DBS file IDs were found on the user's cloud account.") return - r = self._prompt( - f"WaterTAP will delete {len(dbs_file_ids)} DBS files. ", "y" - ) + r = self._prompt(f"WaterTAP will delete {len(dbs_file_ids)} DBS files. ", "y") if (r.lower() == "y") or (r == ""): for dbs_file_id in dbs_file_ids: _logger.info(f"Deleting {dbs_file_id} ...") @@ -319,9 +321,7 @@ def dbs_file_cleanup(self, dbs_file_ids=None): f"File {dbs_file_id} deleted and removed from session_dbs_files list." ) else: - _logger.info( - f"File {dbs_file_id} deleted." - ) + _logger.info(f"File {dbs_file_id} deleted.") def get_corrosion_contact_surfaces(self, dbs_file_id): """ @@ -496,7 +496,9 @@ def _request_status_test(req, target_keys): return req_json else: return req_json - raise RuntimeError(f"Failure in {func_name}. Response: {req_json}. Status Code: {req.status_code}.") + raise RuntimeError( + f"Failure in {func_name}. Response: {req_json}. Status Code: {req.status_code}." + ) def _poll_result_link(result_link, headers, max_request, poll_time): diff --git a/watertap/tools/oli_api/tests/test_client.py b/watertap/tools/oli_api/tests/test_client.py index 0cc812447e..33e648816e 100644 --- a/watertap/tools/oli_api/tests/test_client.py +++ b/watertap/tools/oli_api/tests/test_client.py @@ -55,16 +55,23 @@ def test_dbs_file_available_for_testing(local_dbs_file: Path): assert local_dbs_file.is_file() + @pytest.mark.unit -def test_generate_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): +def test_generate_dbs_file( + oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict +): dbs_file_id = oliapi_instance.generate_dbs_file(source_water) assert len(dbs_file_id) > 0 + @pytest.mark.unit -def test_upload_dbs_file(oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict): +def test_upload_dbs_file( + oliapi_instance: OLIApi, local_dbs_file: Path, source_water: dict +): dbs_file_id = oliapi_instance.upload_dbs_file(str(local_dbs_file)) assert len(dbs_file_id) > 0 + @pytest.mark.unit def test_dbs_file_cleanup(oliapi_instance: OLIApi, local_dbs_file: Path): # This test checks both the upload_dbs_file method and dbs_file_cleanup method @@ -86,6 +93,7 @@ def test_valid_phases(oliapi_instance: OLIApi): for v in oliapi_instance.valid_phases: assert v in valid_phases + @pytest.mark.unit def test_invalid_phases(oliapi_instance_with_invalid_phase: OLIApi): oliapi_instance_with_invalid_phase From e28de474438d5f23b5fc97895ed14cee9c1ab1ad Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Tue, 5 Nov 2024 10:12:15 -0500 Subject: [PATCH 26/27] move refresh attribute earlier; troubleshoot issue with unexpected response when shifting to alternate version of OLI used for NAWI --- watertap/tools/oli_api/credentials.py | 3 ++- watertap/tools/oli_api/flash.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/watertap/tools/oli_api/credentials.py b/watertap/tools/oli_api/credentials.py index 3e2e815fc1..754d728104 100644 --- a/watertap/tools/oli_api/credentials.py +++ b/watertap/tools/oli_api/credentials.py @@ -110,6 +110,7 @@ def __init__( self.access_key = "" self.encryption_key = encryption_key self.config_file = Path(config_file).resolve() + self.refresh = refresh self._manage_credentials( username, password, @@ -123,7 +124,7 @@ def __init__( _logger.setLevel(logging.DEBUG) self.set_headers() - self.refresh = refresh + def set_headers(self): """ diff --git a/watertap/tools/oli_api/flash.py b/watertap/tools/oli_api/flash.py index 2e4d1c9284..c9ba1d2950 100644 --- a/watertap/tools/oli_api/flash.py +++ b/watertap/tools/oli_api/flash.py @@ -1160,9 +1160,9 @@ def _create_input_dict(props, result): prop_tag = _get_nested_data(result, prop)["name"] else: _logger.warning( - f"Unexpected result:\n{result}\n\ninput_dict:\n{input_dict}" + f"Unexpected result:\n{result}\n\ninput_dict:\n{input_dict} from prop {prop}" ) - + continue label = f"{prop_tag}_{phase_tag}" if phase_tag else prop_tag input_dict[k][label] = _extract_values(result, prop) return input_dict From d2af67b2e1e5adf4addf00d65e010f1e6403c253 Mon Sep 17 00:00:00 2001 From: Adam Atia Date: Thu, 7 Nov 2024 14:28:45 -0500 Subject: [PATCH 27/27] blk --- watertap/tools/oli_api/credentials.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/watertap/tools/oli_api/credentials.py b/watertap/tools/oli_api/credentials.py index 754d728104..bba24f1366 100644 --- a/watertap/tools/oli_api/credentials.py +++ b/watertap/tools/oli_api/credentials.py @@ -124,8 +124,6 @@ def __init__( _logger.setLevel(logging.DEBUG) self.set_headers() - - def set_headers(self): """ Creates OLI Cloud API headers and performs initial login.