ENH: Add a reader for nexrad level2 files

.github/workflows/ci.yml

@@ -22,9 +22,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install black black[jupyter] ruff
-      - name: Black style check
-        run: |
-          black --check .
       - name: Lint with ruff
         run: |
           ruff .

.github/workflows/upload_pypi.yml

@@ -30,4 +30,5 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
         run: |
           python -m build
+          python setup.py build_ext --inplace
           twine upload dist/*

MANIFEST.in

@@ -8,4 +8,5 @@ recursive-include tests *
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]
 
+global-include *.pyx *pxd
 recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif

docs/history.md

@@ -2,6 +2,7 @@
 
 ## Development Version (unreleased)
 
+* ENH: Add a reader for nexrad level2 files ({pull}`147`) by [@mgrover1](https://github.com/mgrover1)
 * MNT: Update GitHub actions, address DeprecationWarnings ({pull}`153`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
 * MNT: restructure odim.py/gamic.py, add test_odim.py/test_gamic.py ({pull}`154`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
 * MNT: use CODECOV token ({pull}`155`) by [@kmuehlbauer](https://github.com/kmuehlbauer).

pyproject.toml

@@ -39,12 +39,14 @@ gamic = "xradar.io.backends:GamicBackendEntrypoint"
 iris = "xradar.io.backends:IrisBackendEntrypoint"
 odim = "xradar.io.backends:OdimBackendEntrypoint"
 rainbow = "xradar.io.backends:RainbowBackendEntrypoint"
+nexradlevel2 = "xradar.io.backends:NexradLevel2BackendEntrypoint"
 
 [build-system]
 requires = [
     "setuptools>=45",
     "wheel",
     "setuptools_scm[toml]>=7.0",
+    "numpy"
 ]
 build-backend = "setuptools.build_meta"
 

tests/conftest.py

@@ -58,3 +58,8 @@ def iris0_file():
 @pytest.fixture(scope="session")
 def iris1_file():
     return DATASETS.fetch("SUR210819000227.RAWKPJV")
+
+
+@pytest.fixture(scope="session")
+def nexradlevel2_file():
+    return DATASETS.fetch("KATX20130717_195021_V06")

tests/io/backends/test_common.py

@@ -0,0 +1,9 @@
+from xradar.io.backends import common
+
+
+def test_lazy_dict():
+    d = common.LazyLoadDict({"key1": "value1", "key2": "value2"})
+    assert d["key1"] == "value1"
+    lazy_func = lambda: 999
+    d.set_lazy("lazykey1", lazy_func)
+    assert d["lazykey1"] == 999

tests/io/test_nexrad_level2.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# Copyright (c) 2024, openradar developers.
+# Distributed under the MIT License. See LICENSE for more info.
+
+"""Tests for `xradar.io.nexrad_archive` module."""
+
+import xarray as xr
+
+from xradar.io.backends import open_nexradlevel2_datatree
+
+
+def test_open_nexradlevel2_datatree(nexradlevel2_file):
+    dtree = open_nexradlevel2_datatree(nexradlevel2_file)
+    ds = dtree["sweep_0"]
+    assert ds.attrs["instrument_name"] == "KATX"
+    assert ds.attrs["nsweeps"] == 16
+    assert ds.attrs["Conventions"] == "CF/Radial instrument_parameters"
+    assert ds["DBZH"].shape == (719, 1832)
+    assert ds["DBZH"].dims == ("azimuth", "range")
+    assert int(ds.sweep_number.values) == 0
+
+
+def test_open_nexrad_level2_backend(nexradlevel2_file):
+    ds = xr.open_dataset(nexradlevel2_file, engine="nexradlevel2")
+    assert ds.attrs["instrument_name"] == "KATX"
+    assert ds.attrs["nsweeps"] == 16
+    assert ds.attrs["Conventions"] == "CF/Radial instrument_parameters"
+    assert ds["DBZH"].shape == (719, 1832)
+    assert ds["DBZH"].dims == ("azimuth", "range")
+    assert int(ds.sweep_number.values) == 0

xradar/io/__init__.py

@@ -13,7 +13,24 @@
 .. automodule:: xradar.io.export
 
 """
-from .backends import *  # noqa
+from .backends import (
+    CfRadial1BackendEntrypoint,  # noqa
+    FurunoBackendEntrypoint,  # noqa
+    GamicBackendEntrypoint,  # noqa
+    IrisBackendEntrypoint,  # noqa
+    NexradLevel2BackendEntrypoint,  # noqa
+    OdimBackendEntrypoint,  # noqa
+    RainbowBackendEntrypoint,  # noqa
+    open_cfradial1_datatree,  # noqa
+    open_furuno_datatree,  # noqa
+    open_gamic_datatree,  # noqa
+    open_iris_datatree,  # noqa
+    open_nexradlevel2_datatree,  # noqa
+    open_odim_datatree,  # noqa
+    open_rainbow_datatree,  # noqa
+)
+
+# noqa
 from .export import *  # noqa
 
 __all__ = [s for s in dir() if not s.startswith("_")]

xradar/io/backends/__init__.py

@@ -15,6 +15,7 @@
 .. automodule:: xradar.io.backends.furuno
 .. automodule:: xradar.io.backends.rainbow
 .. automodule:: xradar.io.backends.iris
+.. automodule:: xradar.io.backends.nexrad_level2
 
 """
 
@@ -24,5 +25,9 @@
 from .iris import *  # noqa
 from .odim import *  # noqa
 from .rainbow import *  # noqa
+from .nexrad_level2 import (
+    NexradLevel2BackendEntrypoint,  # noqa
+    open_nexradlevel2_datatree,  # noqa
+)
 
 __all__ = [s for s in dir() if not s.startswith("_")]

xradar/io/backends/common.py

@@ -12,10 +12,15 @@
 
 """
 
+import bz2
+import gzip
 import io
+import itertools
 import struct
 from collections import OrderedDict
+from collections.abc import MutableMapping
 
+import fsspec
 import h5netcdf
 import numpy as np
 import xarray as xr
@@ -229,3 +234,167 @@ def _unpack_dictionary(buffer, dictionary, rawdata=False):
 UINT1 = {"fmt": "B", "dtype": "unit8"}
 UINT2 = {"fmt": "H", "dtype": "uint16"}
 UINT4 = {"fmt": "I", "dtype": "unint32"}
+
+
+def prepare_for_read(filename, storage_options={"anon": True}):
+    """
+    Return a file like object read for reading.
+
+    Open a file for reading in binary mode with transparent decompression of
+    Gzip and BZip2 files. The resulting file-like object should be closed.
+
+    Parameters
+    ----------
+    filename : str or file-like object
+        Filename or file-like object which will be opened. File-like objects
+        will not be examined for compressed data.
+
+    storage_options : dict, optional
+        Parameters passed to the backend file-system such as Google Cloud Storage,
+        Amazon Web Service S3.
+
+    Returns
+    -------
+    file_like : file-like object
+        File like object from which data can be read.
+
+    """
+    # if a file-like object was provided, return
+    if hasattr(filename, "read"):  # file-like object
+        return filename
+
+    # look for compressed data by examining the first few bytes
+    fh = fsspec.open(filename, mode="rb", compression="infer", **storage_options).open()
+    magic = fh.read(3)
+    fh.close()
+
+    # If the data is still compressed, use gunzip/bz2 to uncompress the data
+    if magic.startswith(b"\x1f\x8b"):
+        return gzip.GzipFile(filename, "rb")
+
+    if magic.startswith(b"BZh"):
+        return bz2.BZ2File(filename, "rb")
+
+    return fsspec.open(
+        filename, mode="rb", compression="infer", **storage_options
+    ).open()
+
+
+def make_time_unit_str(dtobj):
+    """Return a time unit string from a datetime object."""
+    return "seconds since " + dtobj.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+class LazyLoadDict(MutableMapping):
+    """
+    A dictionary-like class supporting lazy loading of specified keys.
+
+    Keys which are lazy loaded are specified using the set_lazy method.
+    The callable object which produces the specified key is provided as the
+    second argument to this method. This object gets called when the value
+    of the key is loaded. After this initial call the results is cached
+    in the traditional dictionary which is used for supplemental access to
+    this key.
+
+    Testing for keys in this dictionary using the "key in d" syntax will
+    result in the loading of a lazy key, use "key in d.keys()" to prevent
+    this evaluation.
+
+    The comparison methods, __cmp__, __ge__, __gt__, __le__, __lt__, __ne__,
+    nor the view methods, viewitems, viewkeys, viewvalues, are implemented.
+    Neither is the fromkeys method.
+
+    This is from Py-ART.
+
+    Parameters
+    ----------
+    dic : dict
+        Dictionary containing key, value pairs which will be stored and
+        evaluated traditionally. This dictionary referenced not copied into
+        the LazyLoadDictionary and hence changed to this dictionary may change
+        the original. If this behavior is not desired copy dic in the
+        initalization.
+
+    Examples
+    --------
+    >>> d = LazyLoadDict({'key1': 'value1', 'key2': 'value2'})
+    >>> d.keys()
+    ['key2', 'key1']
+    >>> lazy_func = lambda : 999
+    >>> d.set_lazy('lazykey1', lazy_func)
+    >>> d.keys()
+    ['key2', 'key1', 'lazykey1']
+    >>> d['lazykey1']
+    999
+
+    """
+
+    def __init__(self, dic):
+        """initalize."""
+        self._dic = dic
+        self._lazyload = {}
+
+    # abstract methods
+    def __setitem__(self, key, value):
+        """Set a key which will not be stored and evaluated traditionally."""
+        self._dic[key] = value
+        if key in self._lazyload:
+            del self._lazyload[key]
+
+    def __getitem__(self, key):
+        """Get the value of a key, evaluating a lazy key if needed."""
+        if key in self._lazyload:
+            value = self._lazyload[key]()
+            self._dic[key] = value
+            del self._lazyload[key]
+        return self._dic[key]
+
+    def __delitem__(self, key):
+        """Remove a lazy or traditional key from the dictionary."""
+        if key in self._lazyload:
+            del self._lazyload[key]
+        else:
+            del self._dic[key]
+
+    def __iter__(self):
+        """Iterate over all lazy and traditional keys."""
+        return itertools.chain(self._dic.copy(), self._lazyload.copy())
+
+    def __len__(self):
+        """Return the number of traditional and lazy keys."""
+        return len(self._dic) + len(self._lazyload)
+
+    # additional class to mimic dict behavior
+    def __str__(self):
+        """Return a string representation of the object."""
+        if len(self._dic) == 0 or len(self._lazyload) == 0:
+            seperator = ""
+        else:
+            seperator = ", "
+        lazy_reprs = [(repr(k), repr(v)) for k, v in self._lazyload.items()]
+        lazy_strs = ["{}: LazyLoad({})".format(*r) for r in lazy_reprs]
+        lazy_str = ", ".join(lazy_strs) + "}"
+        return str(self._dic)[:-1] + seperator + lazy_str
+
+    def has_key(self, key):
+        """True if dictionary has key, else False."""
+        return key in self
+
+    def copy(self):
+        """
+        Return a copy of the dictionary.
+
+        Lazy keys are not evaluated in the original or copied dictionary.
+        """
+        dic = self.__class__(self._dic.copy())
+        # load all lazy keys into the copy
+        for key, value_callable in self._lazyload.items():
+            dic.set_lazy(key, value_callable)
+        return dic
+
+    # lazy dictionary specific methods
+    def set_lazy(self, key, value_callable):
+        """Set a lazy key to load from a callable object."""
+        if key in self._dic:
+            del self._dic[key]
+        self._lazyload[key] = value_callable