diff --git a/docs/usage.md b/docs/usage.md index 3e7dca7f..bdf4bcb5 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -25,6 +25,7 @@ notebooks/Mapping_Sweeps :maxdepth: 2 :caption: Examples +notebooks/Auto_Read notebooks/CfRadial1_Model_Transformation notebooks/CfRadial1 notebooks/CfRadial1_Export diff --git a/examples/notebooks/Auto_Read.ipynb b/examples/notebooks/Auto_Read.ipynb new file mode 100644 index 00000000..a54bbd03 --- /dev/null +++ b/examples/notebooks/Auto_Read.ipynb @@ -0,0 +1,246 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Auto read with Xradar" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "### Description" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "This notebook demonstrates how to automatically read various radar data formats using the xradar library. The examples cover multiple radar file formats, showing how xradar handles and organizes the data into DataTree structures. Each dataset showcases a specific radar format, presenting detailed metadata, dimensions, coordinates, and data variables for each radar type.\n", + "\n", + "Available Features:\n", + "\n", + "- **Georeferencing (default: True)**: Automatically adds cartesian coordinates (x, y, z) to the radar data for easier analysis. This feature can be disabled by setting `georeference=False`.\n", + "\n", + "- **Verbose Mode (default: False)**: Enables detailed logging when `verbose=True`, including debug-level information about the file reading process, format attempts, and georeferencing steps.\n", + "\n", + "- **Timeout**: The `timeout` parameter allows you to set a time limit for reading radar files. If the reading process gets stuck or runs indefinitely, a `TimeoutException` is raised, preventing the process from hanging and ensuring smooth operation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from open_radar_data import DATASETS\n", + "\n", + "import xradar as xd" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "## CF/Radial" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "filename = DATASETS.fetch(\"cfrad.20080604_002217_000_SPOL_v36_SUR.nc\")\n", + "radar = xd.io.read(filename)\n", + "radar" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "## ODIM_H5" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "filename = DATASETS.fetch(\"71_20181220_060628.pvol.h5\")\n", + "radar = xd.io.read(filename)\n", + "radar" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "## GAMIC" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "filename = DATASETS.fetch(\"DWD-Vol-2_99999_20180601054047_00.h5\")\n", + "radar = xd.io.read(filename)\n", + "radar" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "## Furuno" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "filename_scn = DATASETS.fetch(\"0080_20210730_160000_01_02.scn.gz\")\n", + "radar = xd.io.read(filename_scn)\n", + "radar" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "## Rainbow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "filename = DATASETS.fetch(\"2013051000000600dBZ.vol\")\n", + "radar = xd.io.read(filename)\n", + "radar" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "## Iris/Sigmet - Reader" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "filename_single = DATASETS.fetch(\"SUR210819000227.RAWKPJV\")\n", + "radar = xd.io.read(filename_single)\n", + "radar" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "filename_volume = DATASETS.fetch(\"cor-main131125105503.RAW2049\")\n", + "radar = xd.io.read(filename_volume)\n", + "radar" + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": {}, + "source": [ + "## NEXRAD Level 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "filename = DATASETS.fetch(\"KATX20130717_195021_V06\")\n", + "radar = xd.io.read(filename)\n", + "radar" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "## Other available options" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "help(xd.io.read)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "filename = DATASETS.fetch(\"cfrad.20080604_002217_000_SPOL_v36_SUR.nc\")\n", + "radar = xd.io.read(filename, verbose=True)\n", + "print(radar.attrs[\"comment\"])" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/io/test_auto_read.py b/tests/io/test_auto_read.py new file mode 100644 index 00000000..c4f98307 --- /dev/null +++ b/tests/io/test_auto_read.py @@ -0,0 +1,228 @@ +import logging +import signal +from unittest.mock import MagicMock, patch + +import pytest +from open_radar_data import DATASETS + +from xradar.io import auto_read + + +# Mocked functions for testing +def mock_open_success(file): + mock_dtree = MagicMock(name="DataTree") + # Mock the behavior of georeference to return the same object without changing attrs + mock_dtree.xradar.georeference.return_value = mock_dtree + return mock_dtree + + +def mock_open_failure(file): + raise ValueError("Failed to open the file.") + + +def mock_georeference(dtree): + return dtree + + +def mock_timeout_handler(signum, frame): + raise auto_read.TimeoutException("Radar file reading timed out.") + + +@pytest.fixture +def sample_file(): + # Fetch a sample radar file for testing + return DATASETS.fetch("KATX20130717_195021_V06") + + +# Test case for the timeout handler +def test_timeout_handler(): + # Mock the signal handling and ensure it raises the timeout exception as expected. + with ( + patch("signal.signal"), + pytest.raises( + auto_read.TimeoutException, match="Radar file reading timed out." + ), + ): + auto_read.timeout_handler(signal.SIGALRM, None) + + +# Test that the alarm is disabled after reading the radar file with timeout +def test_timeout_alarm_disabled(sample_file): + # Mock the alarm function and check if it gets called with 0 to disable it. + with ( + patch("xradar.io.auto_read.io.__all__", ["open_nexradlevel2_datatree"]), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_success, + ), + patch("signal.signal"), + patch("signal.alarm") as mock_alarm, + ): + auto_read.read(sample_file, timeout=1) + # Ensure that alarm was called with 0 to disable it after the read. + mock_alarm.assert_called_with(0) + + +# Test the nonlocal dtree block +def test_read_nonlocal_dtree(sample_file): + # Test that it attempts multiple open_ functions and finally succeeds. + with ( + patch( + "xradar.io.auto_read.io.__all__", + ["open_nexradlevel2_datatree", "open_odim_datatree"], + ), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_failure, + ), + patch( + "xradar.io.auto_read.io.open_odim_datatree", side_effect=mock_open_success + ), + patch("xarray.core.dataset.Dataset.pipe", side_effect=mock_georeference), + ): + dtree = auto_read.read(sample_file, georeference=True, verbose=True) + assert dtree is not None + + +# Test for georeferencing and verbose output +def test_read_with_georeferencing_and_logging(sample_file, caplog): + # Test successful reading with georeferencing and log output. + with ( + patch("xradar.io.auto_read.io.__all__", ["open_nexradlevel2_datatree"]), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_success, + ), + patch("xarray.core.dataset.Dataset.pipe", side_effect=mock_georeference), + ): + with caplog.at_level(logging.DEBUG): + dtree = auto_read.read(sample_file, georeference=True, verbose=True) + assert dtree is not None + + # Check that the log messages contain the correct information + assert "Georeferencing radar data..." in caplog.text + assert ( + "File opened successfully using open_nexradlevel2_datatree." + in caplog.text + ) + + +# Test for exception handling and verbose output during failure +def test_read_failure_with_verbose_output(sample_file, caplog): + # Test that it handles exceptions and logs the verbose failure message. + with ( + patch("xradar.io.auto_read.io.__all__", ["open_nexradlevel2_datatree"]), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_failure, + ), + ): + with caplog.at_level(logging.DEBUG): + with pytest.raises( + ValueError, + match="File could not be opened by any supported format in xradar.io.", + ): + auto_read.read(sample_file, georeference=True, verbose=True) + + # Check that the failure log messages contain the correct information + assert ( + "Failed to open with open_nexradlevel2_datatree" in caplog.text + ) # Capturing log instead of print + + +# Test for raising ValueError when no format can open the file +def test_read_failure_raises_value_error(sample_file): + # Test that it raises ValueError when no format can open the file. + with ( + patch( + "xradar.io.auto_read.io.__all__", + ["open_nexradlevel2_datatree", "open_odim_datatree"], + ), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_failure, + ), + patch( + "xradar.io.auto_read.io.open_odim_datatree", side_effect=mock_open_failure + ), + ): + with pytest.raises( + ValueError, + match="File could not be opened by any supported format in xradar.io.", + ): + auto_read.read(sample_file) + + +# Test read success with georeferencing +def test_read_success(sample_file): + # Test successful reading without timeout and with georeferencing + with ( + patch("xradar.io.auto_read.io.__all__", ["open_nexradlevel2_datatree"]), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_success, + ), + patch("xarray.core.dataset.Dataset.pipe", side_effect=mock_georeference), + ): + dtree = auto_read.read(sample_file, georeference=True, verbose=True) + assert dtree is not None + + +# Test read success without georeferencing +def test_read_success_without_georeference(sample_file): + # Test successful reading without georeferencing + with ( + patch("xradar.io.auto_read.io.__all__", ["open_nexradlevel2_datatree"]), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_success, + ), + ): + dtree = auto_read.read(sample_file, georeference=False) + assert dtree is not None + + +# Test read with timeout raising a TimeoutException +def test_read_with_timeout(sample_file): + # Mock the signal handling and ensure it raises the timeout as expected. + with ( + patch("xradar.io.auto_read.io.__all__", ["open_nexradlevel2_datatree"]), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_success, + ), + patch("signal.signal"), + patch( + "signal.alarm", + side_effect=lambda _: mock_timeout_handler(signal.SIGALRM, None), + ), + ): + with pytest.raises( + auto_read.TimeoutException, match="Radar file reading timed out." + ): + auto_read.read(sample_file, timeout=1) + + +def test_read_comment_update(sample_file): + with ( + patch("xradar.io.auto_read.io.__all__", ["open_nexradlevel2_datatree"]), + patch( + "xradar.io.auto_read.io.open_nexradlevel2_datatree", + side_effect=mock_open_success, + ), + ): + dtree = auto_read.read(sample_file) + assert dtree is not None + + # Print the actual value of the 'comment' attribute for debugging + print(f"Actual comment: {dtree.attrs['comment']}") + + # The initial comment is "im/exported using xradar" (lowercase 'i') + expected_comment_start = "im/exported using xradar" + + # Ensure the comment starts with the correct initial value + assert dtree.attrs["comment"].startswith(expected_comment_start) + + # Ensure the comment has the correct format with 'nexradlevel2' + expected_comment_end = ",\n'nexradlevel2'" + assert dtree.attrs["comment"].endswith(expected_comment_end) diff --git a/xradar/io/__init__.py b/xradar/io/__init__.py index 560175ef..6f37640d 100644 --- a/xradar/io/__init__.py +++ b/xradar/io/__init__.py @@ -15,5 +15,6 @@ """ from .backends import * # noqa from .export import * # noqa +from .auto_read import * # noqa __all__ = [s for s in dir() if not s.startswith("_")] diff --git a/xradar/io/auto_read.py b/xradar/io/auto_read.py new file mode 100644 index 00000000..cd6fdb9e --- /dev/null +++ b/xradar/io/auto_read.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python +# Copyright (c) 2022-2024, openradar developers. +# Distributed under the MIT License. See LICENSE for more info. + +""" +XRadar Auto Reader +================== + +This module provides the ability to automatically read radar files using all +available formats in the `xradar.io` module. It supports handling various file +types, georeferencing, and logging, as well as a timeout mechanism for file reading. + +.. autosummary:: + :nosignatures: + :toctree: generated/ + + {} +""" + +__all__ = [ + "read", +] + +__doc__ = __doc__.format("\n ".join(__all__)) + +import logging +import signal + +from .. import io # noqa + +# Setup a logger for this module +logger = logging.getLogger(__name__) +logging.basicConfig( + level=logging.WARNING +) # Default log level to suppress debug logs unless verbose is set + + +class TimeoutException(Exception): + """ + Custom exception to handle file read timeouts. + + This exception is raised when the radar file reading process exceeds the + specified timeout duration. + """ + + pass + + +def timeout_handler(signum, frame): + """ + Timeout handler to raise a TimeoutException. + + This function is triggered by the alarm signal when the radar file reading + exceeds the allowed timeout duration. + + Parameters + ---------- + signum : int + The signal number (in this case, SIGALRM). + frame : frame object + The current stack frame at the point where the signal was received. + + Raises + ------ + TimeoutException + If the reading operation takes too long and exceeds the timeout. + """ + raise TimeoutException("Radar file reading timed out.") + + +def with_timeout(timeout): + """ + Decorator to enforce a timeout on the file reading process. + + This decorator wraps the file reading function to ensure that if it takes longer than the + specified `timeout` duration, it raises a `TimeoutException`. + + Parameters + ---------- + timeout : int + The maximum number of seconds allowed for the file reading process. + + Returns + ------- + function + A wrapped function that raises `TimeoutException` if it exceeds the timeout. + """ + + def decorator(func): + def wrapper(*args, **kwargs): + # Set the signal handler and a timeout + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(timeout) + try: + return func(*args, **kwargs) + finally: + # Disable the alarm + signal.alarm(0) + + return wrapper + + return decorator + + +def read(file, georeference=True, verbose=False, timeout=None): + """ + Attempt to read a radar file using all available formats in xradar.io. + + This function iterates over all the available file-opening functions in the + `xradar.io` module and attempts to open the provided radar file. If successful, + it can optionally georeference the data (adding x, y, z coordinates) and log + detailed processing information. + + Parameters + ---------- + file : str or Path + The path to the radar file to be read. + georeference : bool, optional + If True, georeference the radar data by adding x, y, z coordinates (default is True). + verbose : bool, optional + If True, prints out detailed processing information (default is False). When set to True, + debug-level logs are enabled. When False, only warnings and errors are logged. + timeout : int or None, optional + Timeout in seconds for reading the radar file. If None, no timeout is applied (default is None). + + Returns + ------- + dtree : DataTree + A `DataTree` object containing the radar data. + + Raises + ------ + ValueError + If the file could not be opened by any supported format in `xradar.io`. + TimeoutException + If reading the file takes longer than the specified timeout. + + Examples + -------- + >>> file = DATASETS.fetch('KATX20130717_195021_V06') + >>> dtree = xd.io.read(file, verbose=True, timeout=10) + Georeferencing radar data... + File opened successfully using open_nexrad_archive. + + Notes + ----- + This function relies on the `xradar` library to support various radar file formats. + It tries to open the file using all available `open_` functions in the `xradar.io` module. + If a `comment` attribute exists in the radar file's metadata, this function appends the + radar type to the comment. The default comment is set to "im/exported using xradar". + """ + # Configure logger level based on 'verbose' + if verbose: + logger.setLevel(logging.DEBUG) # Enable debug messages when verbose is True + logger.debug("Verbose mode activated.") + else: + logger.setLevel( + logging.WARNING + ) # Suppress debug messages when verbose is False + + dtree = None + + # Wrap the read process with a timeout if specified + if timeout: + + @with_timeout(timeout) + def attempt_read(file): + nonlocal dtree + for key in io.__all__: + if "open_" in key: + open_func = getattr(io, key) + try: + dtree = open_func(file) + + # Ensure the 'comment' key exists; if not, create it + if "comment" not in dtree.attrs: + logger.debug("Creating new 'comment' key.") + dtree.attrs["comment"] = "im/exported using xradar" + else: + logger.debug( + f"Existing 'comment': {dtree.attrs['comment']}" + ) + + # Append the key information to the comment without quotes + dtree.attrs["comment"] += f",\n{key.split('_')[1]}" + + # Log the updated comment + logger.debug(f"After update: {dtree.attrs['comment']}") + + if georeference: + logger.debug("Georeferencing radar data...") + dtree = dtree.xradar.georeference() + + logger.debug(f"File opened successfully using {key}.") + break + except Exception as e: + logger.debug(f"Failed to open with {key}: {e}") + continue + + if dtree is None: + raise ValueError( + "File could not be opened by any supported format in xradar.io." + ) + return dtree + + return attempt_read(file) + + # Normal read process without timeout + for key in io.__all__: + if "open_" in key: + open_func = getattr(io, key) + try: + dtree = open_func(file) + + if "comment" not in dtree.attrs: + logger.debug("Creating new 'comment' key.") + dtree.attrs["comment"] = "im/exported using xradar" + else: + logger.debug( + f"Existing 'comment' before update: {dtree.attrs['comment']}" + ) + + dtree.attrs["comment"] += f",\n{key.split('_')[1]}" + + if georeference: + logger.debug("Georeferencing radar data...") + dtree = dtree.xradar.georeference() + + logger.debug(f"File opened successfully using {key}.") + break + except Exception as e: + logger.debug(f"Failed to open with {key}: {e}") + continue + + if dtree is None: + raise ValueError( + "File could not be opened by any supported format in xradar.io." + ) + + return dtree