From db3c760a934e83ed350e1549ad9e65a46087006e Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Tue, 11 Jun 2024 15:42:21 -0500 Subject: [PATCH 1/2] adjust ParamFileStore for on-demand frontend imports --- yt/utilities/parameter_file_storage.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/yt/utilities/parameter_file_storage.py b/yt/utilities/parameter_file_storage.py index cdd006e4341..a466ed0ac0e 100644 --- a/yt/utilities/parameter_file_storage.py +++ b/yt/utilities/parameter_file_storage.py @@ -8,7 +8,16 @@ parallel_simple_proxy, ) -_field_names = ("hash", "bn", "fp", "tt", "ctid", "class_name", "last_seen") +_field_names = ( + "hash", + "bn", + "fp", + "tt", + "ctid", + "class_name", + "module_name", + "last_seen", +) class NoParameterShelf(Exception): @@ -104,6 +113,7 @@ def _adapt_ds(self, ds): "tt": ds.current_time, "ctid": ds.unique_identifier, "class_name": ds.__class__.__name__, + "module_name": ds.__module__, "last_seen": ds._instantiated, } @@ -114,7 +124,17 @@ def _convert_ds(self, ds_dict): fn = os.path.join(fp, bn) class_name = ds_dict["class_name"] if class_name not in output_type_registry: - raise UnknownDatasetType(class_name) + # first check that the relevant frontend is imported: + # the output_type_registry is refreshed between sessions as + # frontends are imported on-demand now. + import importlib + + module = ds_dict["module_name"] + fe_api = ".".join(module.split(".")[0:3]) + ".api" + _ = importlib.import_module(fe_api) + # if it is **still** not in the registry, error + if class_name not in output_type_registry: + raise UnknownDatasetType(class_name) mylog.info("Checking %s", fn) if os.path.exists(fn): ds = output_type_registry[class_name](os.path.join(fp, bn)) From 7ef8b6706d53296c09162171d66aaa897ef52751 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 26 Jun 2024 16:33:36 -0500 Subject: [PATCH 2/2] use exact param store file if exists, add test --- nose_ignores.txt | 1 + tests/tests.yaml | 1 + yt/utilities/parameter_file_storage.py | 2 + .../tests/test_parameter_file_storage.py | 55 +++++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 yt/utilities/tests/test_parameter_file_storage.py diff --git a/nose_ignores.txt b/nose_ignores.txt index a60796b5707..cb26e98682d 100644 --- a/nose_ignores.txt +++ b/nose_ignores.txt @@ -27,6 +27,7 @@ --ignore-file=test_normal_plot_api\.py --ignore-file=test_on_demand_imports\.py --ignore-file=test_outputs_pytest\.py +--ignore-file=test_parameter_file_storage\.py --ignore-file=test_profile_plots\.py --ignore-file=test_raw_field_slices\.py --ignore-file=test_registration\.py diff --git a/tests/tests.yaml b/tests/tests.yaml index 7a8a1826a38..cdfc6c4f646 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -206,6 +206,7 @@ other_tests: - "--ignore-file=test_normal_plot_api\\.py" - "--ignore-file=test_on_demand_imports\\.py" - "--ignore-file=test_outputs_pytest\\.py" + - "--ignore-file=test_parameter_file_storage\\.py" - "--ignore-file=test_profile_plots\\.py" - "--ignore-file=test_raw_field_slices\\.py" - "--ignore-file=test_registration\\.py" diff --git a/yt/utilities/parameter_file_storage.py b/yt/utilities/parameter_file_storage.py index a466ed0ac0e..7dc0bca6e3e 100644 --- a/yt/utilities/parameter_file_storage.py +++ b/yt/utilities/parameter_file_storage.py @@ -91,6 +91,8 @@ def init_db(self): def _get_db_name(self): base_file_name = ytcfg.get("yt", "parameter_file_store") + if os.path.isfile(base_file_name): + return os.path.abspath(base_file_name) if not os.access(os.path.expanduser("~/"), os.W_OK): return os.path.abspath(base_file_name) return os.path.expanduser(f"~/.yt/{base_file_name}") diff --git a/yt/utilities/tests/test_parameter_file_storage.py b/yt/utilities/tests/test_parameter_file_storage.py new file mode 100644 index 00000000000..c244fafd835 --- /dev/null +++ b/yt/utilities/tests/test_parameter_file_storage.py @@ -0,0 +1,55 @@ +import os + +import pytest + +import yt +from yt.config import ytcfg +from yt.utilities.parameter_file_storage import ParameterFileStore + + +@pytest.fixture +def store_parameter_files(): + spf_init = ytcfg.get("yt", "store_parameter_files") + ytcfg.set("yt", "store_parameter_files", True) + yield + ytcfg.set("yt", "store_parameter_files", spf_init) + + +@pytest.fixture +def set_parameter_file(tmp_path): + pfs_init_name = ytcfg.get("yt", "parameter_file_store") + pfs_path = tmp_path / "param_file_store.csv" + pfs_path.touch() + ytcfg.set("yt", "parameter_file_store", str(pfs_path)) + yield + ytcfg.set("yt", "parameter_file_store", pfs_init_name) + + +def test_on_disk_parameter_file_store( + monkeypatch, store_parameter_files, set_parameter_file +): + pfs = ParameterFileStore() + # patching _register to True forces a full re-initialization on next + # instantiation of a ParameterFileStore + monkeypatch.setattr(pfs, "_register", True) + monkeypatch.setattr(pfs, "_records", {}) + + # to force a re-initialization, set _register to True then get a new + # instance. This should read from disk. + pfs = ParameterFileStore() + pfs_path = ytcfg.get("yt", "parameter_file_store") + assert os.path.isfile(pfs_path) # on disk store should now exist + db_records = pfs.read_db() # force a read from disk + assert len(db_records) == 0 # and it should be empty + + # check that ds load is registered on disk + ds = yt.load_sample("IsolatedGalaxy") + db_records = pfs.read_db() + + assert len(db_records) == 1 + hash = ds._hash() + assert hash in db_records + ds_rec = db_records[hash] + assert ds_rec["bn"] == "galaxy0030" + assert ds_rec["class_name"] == "EnzoDataset" + assert ds_rec["module_name"] == "yt.frontends.enzo.data_structures"