From 56e748c7879d8ed9010b0569f0f08dc3729334c6 Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Sun, 4 Jun 2023 15:58:11 -0400 Subject: [PATCH 01/17] Two tweaks for updated GIZMO file snapshots dmetal check - gizmo runs with METALS enabled but all of COOL_METAL_LINES_BY_SPECIES, GALSF_FB_FIRE_RPROCESS, GALSF_FB_FIRE_AGE_TRACERS, and STARFORGE_FEEDBACK_TRACERS disabled will have a metallicity field (aka dmetal) a length 1. This change accounts for this possibility Overriding Gadget _parse_parameter_file - newer versions of GIZMO have changed some of the header fields used by yt for determining the cosmology of the simulation. Moving this function allows for improved cosmology detection. Now check if ComovingIntegrationOn flag exists - if so, use that to determine if cosmological run and pull cosmological parameters using updated names. Otherwise, fall back to checking for OmegaLambda and setting cosmological flag from that. --- yt/frontends/gizmo/data_structures.py | 112 +++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 63af45c8db..302038795d 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -1,6 +1,11 @@ import os +import numpy as np + from yt.frontends.gadget.data_structures import GadgetHDF5Dataset +from yt.funcs import only_on_root +from yt.utilities.cosmology import Cosmology +from yt.utilities.logger import ytLogger as mylog from yt.utilities.on_demand_imports import _h5py as h5py from .fields import GizmoFieldInfo @@ -39,7 +44,7 @@ def _is_valid(cls, filename, *args, **kwargs): vg in fh["/"] for vg in veto_groups ) dmetal = "/PartType0/Metallicity" - if dmetal not in fh or fh[dmetal].shape[1] < 11: + if dmetal not in fh or (fh[dmetal].ndim > 1 and fh[dmetal].shape[1] < 11): valid = False fh.close() except Exception: @@ -49,3 +54,108 @@ def _is_valid(cls, filename, *args, **kwargs): def _set_code_unit_attributes(self): super()._set_code_unit_attributes() + + def _parse_parameter_file(self): + hvals = self._get_hvals() + + self.dimensionality = 3 + self.refine_by = 2 + self.parameters["HydroMethod"] = "sph" + # Set standard values + + # We may have an overridden bounding box. + if self.domain_left_edge is None and hvals["BoxSize"] != 0: + self.domain_left_edge = np.zeros(3, "float64") + self.domain_right_edge = np.ones(3, "float64") * hvals["BoxSize"] + + self.domain_dimensions = np.ones(3, "int32") + self._periodicity = (True, True, True) + + self.cosmological_simulation = 1 + + try: + self.current_redshift = hvals["Redshift"] + except KeyError: + # Probably not a cosmological dataset, we should just set + # z = 0 and let the user know + self.current_redshift = 0.0 + only_on_root(mylog.info, "Redshift is not set in Header. Assuming z=0.") + + try: + # The current version of GIZMO has updated the names of the Omegas + # to include an _, added baryons and radiation and added the + # ComovingIntegrationOn field. ComovingIntegrationOn is always set, + # but the Omega's are only included if ComovingIntegrationOn is true + if "ComovingIntegrationOn" in hvals: + self.cosmological_simulation = hvals["ComovingIntegrationOn"] + self.omega_lambda = hvals["Omega_Lambda"] + self.omega_matter = hvals["Omega_Matter"] + self.omega_baryon = hvals["Omega_Baryon"] + self.omega_radiation = hvals["Omega_Radiation"] + self.hubble_constant = hvals["HubbleParam"] + else: + # Should still support old GIZMO versions too + self.omega_lambda = hvals["OmegaLambda"] + self.omega_matter = hvals["Omega0"] + self.hubble_constant = self.omega_lambda == 0.0 + except KeyError: + # If these are not set it is definitely not a cosmological dataset. + self.omega_lambda = 0.0 + self.omega_matter = 1.0 # Just in case somebody asks for it. + self.cosmological_simulation = 0 + # Hubble is set below for Omega Lambda = 0. + + # Since new versions of GIZMO output the ComovingIntegrationOn flag + # we will rely on its presence to determine if this is a cosmological + # dataset. + if not self.cosmological_simulation: + only_on_root( + mylog.info, + "ComovingIntegrationOn != 1 or (not found " + "and OmegaLambda is 0.0), so we are turning off Cosmology.", + ) + self.hubble_constant = 1.0 # So that scaling comes out correct + self.cosmological_simulation = 0 + self.current_redshift = 0.0 + # This may not be correct. + self.current_time = hvals["Time"] + else: + # Now we calculate our time based on the cosmology, because in + # ComovingIntegration hvals["Time"] will in fact be the expansion + # factor, not the actual integration time, so we re-calculate + # global time from our Cosmology. + cosmo = Cosmology( + hubble_constant=self.hubble_constant, + omega_matter=self.omega_matter, + omega_lambda=self.omega_lambda, + ) + self.current_time = cosmo.lookback_time(self.current_redshift, 1e6) + only_on_root( + mylog.info, + "Calculating time from %0.3e to be %0.3e seconds", + hvals["Time"], + self.current_time, + ) + self.parameters = hvals + + prefix = os.path.abspath( + os.path.join( + os.path.dirname(self.parameter_filename), + os.path.basename(self.parameter_filename).split(".", 1)[0], + ) + ) + + if hvals["NumFiles"] > 1: + for t in ( + f"{prefix}.%(num)s{self._suffix}", + f"{prefix}.gad.%(num)s{self._suffix}", + ): + if os.path.isfile(t % {"num": 0}): + self.filename_template = t + break + else: + raise RuntimeError("Could not determine correct data file template.") + else: + self.filename_template = self.parameter_filename + + self.file_count = hvals["NumFiles"] From 680236973011cf3abae2213599c5ca94dae0b5dd Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Sun, 4 Jun 2023 18:59:27 -0400 Subject: [PATCH 02/17] Added diagnostic information and bugfix I somehow deleted part of a line before the previous commit. This has been fixed. --- yt/frontends/gizmo/data_structures.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 302038795d..5ceaa74794 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -95,9 +95,14 @@ def _parse_parameter_file(self): self.hubble_constant = hvals["HubbleParam"] else: # Should still support old GIZMO versions too + only_on_root( + mylog.info, + "ComovingIntegrationOn does not exist, falling back to OmegaLambda" + ) self.omega_lambda = hvals["OmegaLambda"] self.omega_matter = hvals["Omega0"] - self.hubble_constant = self.omega_lambda == 0.0 + self.hubble_constant = hvals["HubbleParam"] + self.cosmological_simulation = self.omega_lambda == 0.0 except KeyError: # If these are not set it is definitely not a cosmological dataset. self.omega_lambda = 0.0 From dfafbe63aa3c7190942e145b4513f28c05962602 Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Sun, 4 Jun 2023 19:01:46 -0400 Subject: [PATCH 03/17] pre-commit changes --- yt/frontends/gizmo/data_structures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 5ceaa74794..4b3df7b76b 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -96,8 +96,8 @@ def _parse_parameter_file(self): else: # Should still support old GIZMO versions too only_on_root( - mylog.info, - "ComovingIntegrationOn does not exist, falling back to OmegaLambda" + mylog.info, + "ComovingIntegrationOn does not exist, falling back to OmegaLambda", ) self.omega_lambda = hvals["OmegaLambda"] self.omega_matter = hvals["Omega0"] From 08eaa986228a91aeaf9ff85fbc960f02f8ad4dea Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Mon, 26 Jun 2023 13:24:53 -0400 Subject: [PATCH 04/17] Updated comments Updated comments to point out when the switch occurred. --- yt/frontends/gizmo/data_structures.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 4b3df7b76b..e84ac60424 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -82,7 +82,7 @@ def _parse_parameter_file(self): only_on_root(mylog.info, "Redshift is not set in Header. Assuming z=0.") try: - # The current version of GIZMO has updated the names of the Omegas + # In 1d8479, Nov 2020, public GIZMO updated the names of the Omegas # to include an _, added baryons and radiation and added the # ComovingIntegrationOn field. ComovingIntegrationOn is always set, # but the Omega's are only included if ComovingIntegrationOn is true @@ -94,7 +94,7 @@ def _parse_parameter_file(self): self.omega_radiation = hvals["Omega_Radiation"] self.hubble_constant = hvals["HubbleParam"] else: - # Should still support old GIZMO versions too + # Should still support GIZMO versions prior to 1d8479 too only_on_root( mylog.info, "ComovingIntegrationOn does not exist, falling back to OmegaLambda", @@ -110,9 +110,6 @@ def _parse_parameter_file(self): self.cosmological_simulation = 0 # Hubble is set below for Omega Lambda = 0. - # Since new versions of GIZMO output the ComovingIntegrationOn flag - # we will rely on its presence to determine if this is a cosmological - # dataset. if not self.cosmological_simulation: only_on_root( mylog.info, From 3ff5fbf26c6a25c51c335dfdcdc0c7d87bdbd26b Mon Sep 17 00:00:00 2001 From: mtryan83 Date: Thu, 6 Jul 2023 13:52:08 -0400 Subject: [PATCH 05/17] Update yt/frontends/gizmo/data_structures.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/frontends/gizmo/data_structures.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index e84ac60424..a9bdd666de 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -140,12 +140,7 @@ def _parse_parameter_file(self): ) self.parameters = hvals - prefix = os.path.abspath( - os.path.join( - os.path.dirname(self.parameter_filename), - os.path.basename(self.parameter_filename).split(".", 1)[0], - ) - ) + prefix = os.path.join(ds.directory, ds.basename.split('.', 1)[0]) if hvals["NumFiles"] > 1: for t in ( From c1095d57021dd82024e3ecf26521c3d6fb5b165f Mon Sep 17 00:00:00 2001 From: mtryan83 Date: Thu, 6 Jul 2023 13:57:39 -0400 Subject: [PATCH 06/17] Update yt/frontends/gizmo/data_structures.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/frontends/gizmo/data_structures.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index a9bdd666de..58f93593f6 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -73,13 +73,9 @@ def _parse_parameter_file(self): self.cosmological_simulation = 1 - try: - self.current_redshift = hvals["Redshift"] - except KeyError: - # Probably not a cosmological dataset, we should just set - # z = 0 and let the user know - self.current_redshift = 0.0 - only_on_root(mylog.info, "Redshift is not set in Header. Assuming z=0.") + self.current_redshift = hvals.get("Redshift", 0.0) + if "Redshift" not in hvals; + mylog.info("Redshift is not set in Header. Assuming z=0.") try: # In 1d8479, Nov 2020, public GIZMO updated the names of the Omegas From 403ce1917daccd5473b6d2a0ac27e0fee1ed3428 Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Thu, 6 Jul 2023 14:59:49 -0400 Subject: [PATCH 07/17] Responding to comments --- yt/frontends/gizmo/data_structures.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 58f93593f6..16ac5e8f09 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -3,7 +3,6 @@ import numpy as np from yt.frontends.gadget.data_structures import GadgetHDF5Dataset -from yt.funcs import only_on_root from yt.utilities.cosmology import Cosmology from yt.utilities.logger import ytLogger as mylog from yt.utilities.on_demand_imports import _h5py as h5py @@ -74,7 +73,7 @@ def _parse_parameter_file(self): self.cosmological_simulation = 1 self.current_redshift = hvals.get("Redshift", 0.0) - if "Redshift" not in hvals; + if "Redshift" not in hvals: mylog.info("Redshift is not set in Header. Assuming z=0.") try: @@ -91,8 +90,7 @@ def _parse_parameter_file(self): self.hubble_constant = hvals["HubbleParam"] else: # Should still support GIZMO versions prior to 1d8479 too - only_on_root( - mylog.info, + mylog.info( "ComovingIntegrationOn does not exist, falling back to OmegaLambda", ) self.omega_lambda = hvals["OmegaLambda"] @@ -102,13 +100,12 @@ def _parse_parameter_file(self): except KeyError: # If these are not set it is definitely not a cosmological dataset. self.omega_lambda = 0.0 - self.omega_matter = 1.0 # Just in case somebody asks for it. + self.omega_matter = 0.0 # Just in case somebody asks for it. self.cosmological_simulation = 0 # Hubble is set below for Omega Lambda = 0. if not self.cosmological_simulation: - only_on_root( - mylog.info, + mylog.info( "ComovingIntegrationOn != 1 or (not found " "and OmegaLambda is 0.0), so we are turning off Cosmology.", ) @@ -128,15 +125,14 @@ def _parse_parameter_file(self): omega_lambda=self.omega_lambda, ) self.current_time = cosmo.lookback_time(self.current_redshift, 1e6) - only_on_root( - mylog.info, + mylog.info( "Calculating time from %0.3e to be %0.3e seconds", hvals["Time"], self.current_time, ) self.parameters = hvals - prefix = os.path.join(ds.directory, ds.basename.split('.', 1)[0]) + prefix = os.path.join(self.directory, self.basename.split('.', 1)[0]) if hvals["NumFiles"] > 1: for t in ( From 770da8267cb380ae78d026fec46672e4359eaa9e Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Thu, 6 Jul 2023 15:01:40 -0400 Subject: [PATCH 08/17] Removing unnecessary gadget 2 support --- yt/frontends/gizmo/data_structures.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 16ac5e8f09..f972f72055 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -135,15 +135,7 @@ def _parse_parameter_file(self): prefix = os.path.join(self.directory, self.basename.split('.', 1)[0]) if hvals["NumFiles"] > 1: - for t in ( - f"{prefix}.%(num)s{self._suffix}", - f"{prefix}.gad.%(num)s{self._suffix}", - ): - if os.path.isfile(t % {"num": 0}): - self.filename_template = t - break - else: - raise RuntimeError("Could not determine correct data file template.") + self.filename_template = f"{prefix}.%(num)s{self._suffix}" else: self.filename_template = self.parameter_filename From 044c3d0e07d86f946ed01ba9af5be6e542637b0f Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Thu, 6 Jul 2023 15:09:35 -0400 Subject: [PATCH 09/17] pre-commit check --- yt/frontends/gizmo/data_structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index f972f72055..4c1c0a85fe 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -132,7 +132,7 @@ def _parse_parameter_file(self): ) self.parameters = hvals - prefix = os.path.join(self.directory, self.basename.split('.', 1)[0]) + prefix = os.path.join(self.directory, self.basename.split(".", 1)[0]) if hvals["NumFiles"] > 1: self.filename_template = f"{prefix}.%(num)s{self._suffix}" From 296cc7bb1307e669b953d34fd72a3ec8c45271b4 Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Tue, 11 Jul 2023 17:19:49 -0400 Subject: [PATCH 10/17] Adding veto groups It looks like gizmo doesn't export any additional groups besides the Header, unlike Eagle/OWLS. So this should resolve those ambiguities. --- yt/frontends/gizmo/data_structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 4c1c0a85fe..7d13c93904 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -16,7 +16,7 @@ class GizmoDataset(GadgetHDF5Dataset): @classmethod def _is_valid(cls, filename, *args, **kwargs): need_groups = ["Header"] - veto_groups = ["FOF", "Group", "Subhalo"] + veto_groups = ["Config", "Constants", "FOF", "Group", "Subhalo"] valid = True valid_fname = filename # If passed arg is a directory, look for the .0 file in that dir From 21cf09bf892c4b81021f0d2e08148790668ca8bd Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Tue, 11 Jul 2023 23:54:52 -0400 Subject: [PATCH 11/17] Accidentally inverted fallback cosmological sim test and removed extraneous setting --- yt/frontends/gizmo/data_structures.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 7d13c93904..91cf1f124c 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -96,7 +96,7 @@ def _parse_parameter_file(self): self.omega_lambda = hvals["OmegaLambda"] self.omega_matter = hvals["Omega0"] self.hubble_constant = hvals["HubbleParam"] - self.cosmological_simulation = self.omega_lambda == 0.0 + self.cosmological_simulation = self.omega_lambda != 0.0 except KeyError: # If these are not set it is definitely not a cosmological dataset. self.omega_lambda = 0.0 @@ -110,7 +110,6 @@ def _parse_parameter_file(self): "and OmegaLambda is 0.0), so we are turning off Cosmology.", ) self.hubble_constant = 1.0 # So that scaling comes out correct - self.cosmological_simulation = 0 self.current_redshift = 0.0 # This may not be correct. self.current_time = hvals["Time"] From 82bd65d48515a8beac30953b39a6fe9b42997655 Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Fri, 14 Jul 2023 14:16:45 -0400 Subject: [PATCH 12/17] Switched from try/except to if/else structure. Also added some debugging information --- yt/frontends/gizmo/data_structures.py | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 91cf1f124c..b1f2ad848b 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -76,29 +76,35 @@ def _parse_parameter_file(self): if "Redshift" not in hvals: mylog.info("Redshift is not set in Header. Assuming z=0.") - try: + if "ComovingIntegrationOn" in hvals: # In 1d8479, Nov 2020, public GIZMO updated the names of the Omegas # to include an _, added baryons and radiation and added the # ComovingIntegrationOn field. ComovingIntegrationOn is always set, # but the Omega's are only included if ComovingIntegrationOn is true - if "ComovingIntegrationOn" in hvals: - self.cosmological_simulation = hvals["ComovingIntegrationOn"] + mylog.debug( + "Reading cosmological parameters using post-1d8479 format" + ) + self.cosmological_simulation = hvals["ComovingIntegrationOn"] + if self.cosmological_simulation: self.omega_lambda = hvals["Omega_Lambda"] self.omega_matter = hvals["Omega_Matter"] self.omega_baryon = hvals["Omega_Baryon"] self.omega_radiation = hvals["Omega_Radiation"] - self.hubble_constant = hvals["HubbleParam"] - else: - # Should still support GIZMO versions prior to 1d8479 too - mylog.info( - "ComovingIntegrationOn does not exist, falling back to OmegaLambda", - ) - self.omega_lambda = hvals["OmegaLambda"] - self.omega_matter = hvals["Omega0"] - self.hubble_constant = hvals["HubbleParam"] - self.cosmological_simulation = self.omega_lambda != 0.0 - except KeyError: + self.hubble_constant = hvals["HubbleParam"] + elif "OmegaLambda" in hvals: + # Should still support GIZMO versions prior to 1d8479 too + mylog.info( + "ComovingIntegrationOn does not exist, falling back to OmegaLambda", + ) + self.omega_lambda = hvals["OmegaLambda"] + self.omega_matter = hvals["Omega0"] + self.hubble_constant = hvals["HubbleParam"] + self.cosmological_simulation = self.omega_lambda != 0.0 + else: # If these are not set it is definitely not a cosmological dataset. + mylog.debug( + "No cosmological information found, assuming defaults" + ) self.omega_lambda = 0.0 self.omega_matter = 0.0 # Just in case somebody asks for it. self.cosmological_simulation = 0 From d6bf2c5534e796810fa18592bc2ea57d1f523df6 Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Fri, 14 Jul 2023 14:22:18 -0400 Subject: [PATCH 13/17] Pre-commit changes --- yt/frontends/gizmo/data_structures.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index b1f2ad848b..d2605366a0 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -81,9 +81,7 @@ def _parse_parameter_file(self): # to include an _, added baryons and radiation and added the # ComovingIntegrationOn field. ComovingIntegrationOn is always set, # but the Omega's are only included if ComovingIntegrationOn is true - mylog.debug( - "Reading cosmological parameters using post-1d8479 format" - ) + mylog.debug("Reading cosmological parameters using post-1d8479 format") self.cosmological_simulation = hvals["ComovingIntegrationOn"] if self.cosmological_simulation: self.omega_lambda = hvals["Omega_Lambda"] @@ -102,9 +100,7 @@ def _parse_parameter_file(self): self.cosmological_simulation = self.omega_lambda != 0.0 else: # If these are not set it is definitely not a cosmological dataset. - mylog.debug( - "No cosmological information found, assuming defaults" - ) + mylog.debug("No cosmological information found, assuming defaults") self.omega_lambda = 0.0 self.omega_matter = 0.0 # Just in case somebody asks for it. self.cosmological_simulation = 0 From 5252fc8dec4556f7cfbe2a44cbb528a5ed2e571f Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Fri, 14 Jul 2023 18:02:44 -0400 Subject: [PATCH 14/17] Updated _is_valid Newer (post Apr 2021) versions of gizmo export GIZMO_version=year_of_last_commit as a header variable. We should use this in preference to checking the gas metallicity, which might not exist. Since the names of the omegas have changed, we don't want to fall back to the gadget loader in the comoving case (in the non-cosmo case it shouldn't matter?) --- yt/frontends/gizmo/data_structures.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index d2605366a0..9b2d3e3e2e 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -42,9 +42,14 @@ def _is_valid(cls, filename, *args, **kwargs): valid = all(ng in fh["/"] for ng in need_groups) and not any( vg in fh["/"] for vg in veto_groups ) - dmetal = "/PartType0/Metallicity" - if dmetal not in fh or (fh[dmetal].ndim > 1 and fh[dmetal].shape[1] < 11): - valid = False + # From Apr 2021, 7f1f06f, public gizmo includes a header variable + # GIZMO_version, which is set to the year of the most recent commit + # We should prefer this to checking the metallicity, which might + # not exist + if "GIZMO_version" not in fh["/Header"].attrs: + dmetal = "/PartType0/Metallicity" + if dmetal not in fh or (fh[dmetal].ndim > 1 and fh[dmetal].shape[1] < 11): + valid = False fh.close() except Exception: valid = False From 883f8450ebe4560fcabb05e01f737922568ad440 Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Fri, 14 Jul 2023 18:09:44 -0400 Subject: [PATCH 15/17] pre-commit changes --- yt/frontends/gizmo/data_structures.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yt/frontends/gizmo/data_structures.py b/yt/frontends/gizmo/data_structures.py index 9b2d3e3e2e..4aeeac96e0 100644 --- a/yt/frontends/gizmo/data_structures.py +++ b/yt/frontends/gizmo/data_structures.py @@ -44,11 +44,13 @@ def _is_valid(cls, filename, *args, **kwargs): ) # From Apr 2021, 7f1f06f, public gizmo includes a header variable # GIZMO_version, which is set to the year of the most recent commit - # We should prefer this to checking the metallicity, which might + # We should prefer this to checking the metallicity, which might # not exist if "GIZMO_version" not in fh["/Header"].attrs: dmetal = "/PartType0/Metallicity" - if dmetal not in fh or (fh[dmetal].ndim > 1 and fh[dmetal].shape[1] < 11): + if dmetal not in fh or ( + fh[dmetal].ndim > 1 and fh[dmetal].shape[1] < 11 + ): valid = False fh.close() except Exception: From 4e39566423c884ff6e2f5e66264be9d68858a77d Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Mon, 17 Jul 2023 19:54:25 -0400 Subject: [PATCH 16/17] Added test for new _is_valid and _parse_parameter_file logic New test requires test files gizmo_zeldovich/snapshot_076_XX_gizver.hdf5 where XX is either wi or no. The files can be generated using the gizmo zeldovich pancake test problem with the additional config flags COOLING and METAL. The _no_gizver then has the GIZMO_version header field manually removed. Snapshot 076 was picked randomly. --- yt/frontends/gizmo/tests/test_outputs.py | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/yt/frontends/gizmo/tests/test_outputs.py b/yt/frontends/gizmo/tests/test_outputs.py index a19afc751e..31cd686fb3 100644 --- a/yt/frontends/gizmo/tests/test_outputs.py +++ b/yt/frontends/gizmo/tests/test_outputs.py @@ -3,7 +3,8 @@ import yt from yt.frontends.gizmo.api import GizmoDataset from yt.frontends.gizmo.fields import metal_elements -from yt.testing import requires_file, requires_module +from yt.testing import assert_allclose_units, requires_file, requires_module +from yt.units import Myr from yt.utilities.answer_testing.framework import requires_ds, sph_answer # This maps from field names to weight field names to use for projections @@ -20,6 +21,8 @@ g64 = "gizmo_64/output/snap_N64L16_135.hdf5" gmhd = "gizmo_mhd_mwdisk/gizmo_mhd_mwdisk.hdf5" gmhd_bbox = [[-400, 400]] * 3 +zeld_wg = "gizmo_zeldovich/snapshot_076_wi_gizver.hdf5" +zeld_ng = "gizmo_zeldovich/snapshot_076_no_gizver.hdf5" @requires_module("h5py") @@ -32,6 +35,32 @@ def test_gizmo_64(): yield test +@requires_module("h5py") +@requires_file(zeld_wg) +@requires_file(zeld_ng) +def test_gizmo_zeldovich(): + """ + Test loading a recent gizmo snapshot that doesn't have cooling/metallicity + + The gizmo_zeldovich file has no metallicity field on the gas particles + but is a cosmological dataset run using GIZMO_version=2022. There are two + versions of the file, with GIZMO_version (_wg) and without GIZMO_version + (_ng). Check that both load as gizmo datasets and correctly pull the + cosmological variables. This test should get simpler when the file switches + to pytest. + """ + for fn in [zeld_wg, zeld_ng]: + ds = yt.load(fn) + assert isinstance(ds, GizmoDataset) + + assert ds.cosmological_simulation + assert ds.omega_matter == 1.0 + assert ds.omega_lambda == 0.0 + # current_time is calculated from the cosmology so this checks if that + # was calculated correctly + assert_allclose_units(ds.current_time, 1672.0678 * Myr) + + @requires_module("h5py") @requires_file(gmhd) def test_gizmo_mhd(): From da25c87b6b3d7f55e60901e42f7580d9fd58a252 Mon Sep 17 00:00:00 2001 From: Michael Ryan Date: Tue, 25 Jul 2023 10:51:40 -0400 Subject: [PATCH 17/17] Updated sample data registry --- yt/sample_data_registry.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/yt/sample_data_registry.json b/yt/sample_data_registry.json index eedff08763..e0e7842a91 100644 --- a/yt/sample_data_registry.json +++ b/yt/sample_data_registry.json @@ -724,6 +724,12 @@ "load_name": "gizmo_mhd_mwdisk.hdf5", "url": "https://yt-project.org/data/gizmo_mhd_mwdisk.tar.gz" }, + "gizmo_zeldovich.tar.gz": { + "hash": "5f1a73ead736024ffb6c099f84e834ec927d2818c93d7c775d9ff625adaa7c51", + "load_kwargs": {}, + "load_name": "snapshot_076_wi_gizver.hdf5", + "url": "https://yt-project.org/data/gizmo_zeldovich.tar.gz" + }, "grs-50-cube.fits.gz": { "hash": "1aff152f616d2626c8515c1493e65f07843d9b5f2ba73b7e4271a2dc6a9e6da7", "load_kwargs": {},