Add ionospheric file download to workflow

.pre-commit-config.yaml

@@ -36,11 +36,3 @@ repos:
         - types-python-dateutil
         - types-requests
         - "pydantic>=2.1"
-
-  - repo: https://github.com/PyCQA/pydocstyle
-    rev: "6.3.0"
-    hooks:
-      - id: pydocstyle
-        # https://github.com/PyCQA/pydocstyle/pull/608#issuecomment-1381168417
-        additional_dependencies: ['.[toml]']
-        exclude: tests

conda-env.yml

@@ -24,6 +24,7 @@ dependencies:
   # - shapely>=1.8
   - isce3>=0.14.0
   - compass>=0.4.1
+  - backoff
   - dask
   - dolphin>=0.5.1
   - gdal
@@ -38,5 +39,5 @@ dependencies:
   - rich
   - rioxarray
   - sardem
-  - sentineleof
+  - sentineleof>=0.9.5
   - shapely

pyproject.toml

@@ -45,24 +45,36 @@ write_to = "src/sweets/_version.py"
 # https://github.com/pypa/setuptools_scm#version-number-construction
 version_scheme = "no-guess-dev" # Will not guess the next version
 
+[tool.ruff]
+ignore = [
+  "D100", # Missing docstring in public module
+  "D104", # Missing docstring in public package
+  "D105", # Missing docstring in magic method
+  "D107", # Missing docstring in __init__
+  "D203", # 1 blank line required before class docstring
+  "D212", # Multi-line docstring summary should start at the first line
+  "PLR",  # Pylint Refactor
+]
+
+[tool.ruff.lint]
+# Enable the isort rules.
+extend-select = ["I"]
 
 [tool.black]
-target-version = ["py38", "py39", "py310", "py310"]
+target-version = ["py39", "py310", "py311", "py312"]
 preview = true
 
 [tool.isort]
 profile = "black"
-known_first_party = ["sweets"]
+known_first_party = ["dolphin"]
 
 [tool.mypy]
-python_version = "3.8"
+python_version = "3.10"
 ignore_missing_imports = true
 plugins = ["pydantic.mypy"]
 
-[tool.pydocstyle]
-ignore = "D100,D102,D104,D105,D106,D107,D203,D204,D213,D413"
-
+[tool.ruff.per-file-ignores]
+"**/__init__.py" = ["F401"]
+"test/**" = ["D"]
 [tool.pytest.ini_options]
-filterwarnings = [
-  # "error",
-]
+filterwarnings = ["error"]

requirements.txt

@@ -13,5 +13,5 @@ pyproj>=3.2
 requests>=2.10
 rich>=12.0
 sardem>=0.11.1
-sentineleof>=0.6.5
+sentineleof>=0.9.5
 shapely>=1.8

scripts/prep_mintpy.py

@@ -18,10 +18,10 @@
 import numpy as np
 import pyproj
 from dolphin import io
-from dolphin.utils import full_suffix, get_dates
+from dolphin.utils import full_suffix
 from mintpy.utils import arg_utils, ptime, readfile, writefile
 from mintpy.utils.utils0 import calc_azimuth_from_east_north_obs
-from opera_utils import OPERA_DATASET_ROOT
+from opera_utils import OPERA_DATASET_ROOT, get_dates
 
 ####################################################################################
 EXAMPLE = """example:

src/sweets/_geocode_slcs.py

@@ -101,13 +101,15 @@ def create_config_files(
     burst_db_file: Filename,
     dem_file: Filename,
     orbit_dir: Filename,
+    tec_dir: Optional[Filename] = None,
     bbox: Optional[Tuple[float, ...]] = None,
     x_posting: float = 5,
     y_posting: float = 10,
     pol_type: str = "co-pol",
     out_dir: Filename = Path("gslcs"),
     overwrite: bool = False,
     using_zipped: bool = False,
+    enable_corrections: bool = True,
 ) -> List[Path]:
     """Create the geocoding config files for a stack of SLCs.
 
@@ -121,6 +123,9 @@ def create_config_files(
         File containing the DEM
     orbit_dir : Filename
         Directory containing the orbit files
+    tec_dir : Filename, optional
+        Directory containing the ionosphere TEC files.
+        If None, skips correction.
     bbox : Optional[Tuple[float, ...]], optional
         [lon_min, lat_min, lon_max, lat_max] to limit the processing.
         Note that this does not change each burst's bounding box, but
@@ -139,6 +144,9 @@ def create_config_files(
     using_zipped : bool, optional
         If true, will search for. zip files instead of unzipped .SAFE directories.
         By default False.
+    enable_corrections : bool, default = True
+        If true, computes model-based corrections to geocoding.
+        See `compass` for more details on all corrections.
 
     Returns
     -------
@@ -167,5 +175,7 @@ def create_config_files(
         x_spac=x_posting,
         y_spac=y_posting,
         using_zipped=using_zipped,
+        enable_corrections=enable_corrections,
+        tec_dir=tec_dir,
     )
     return sorted((Path(out_dir) / "runconfigs").glob("*"))

src/sweets/_log.py

@@ -17,10 +17,8 @@
 """
 
 import logging
-import time
-from collections.abc import Callable
-from functools import wraps
 
+from dolphin._log import log_runtime
 from rich.console import Console
 from rich.logging import RichHandler
 
@@ -81,34 +79,3 @@ def format_log(logger: logging.Logger, debug: bool = False) -> logging.Logger:
         logger.setLevel(debug)
 
     return logger
-
-
-def log_runtime(f: Callable) -> Callable:
-    """Decorate a function to time how long it takes to run.
-
-    Usage
-    -----
-    @log_runtime
-    def test_func():
-        return 2 + 4
-    """
-    logger = get_log(__name__)
-
-    @wraps(f)
-    def wrapper(*args, **kwargs):
-        t1 = time.time()
-
-        result = f(*args, **kwargs)
-
-        t2 = time.time()
-        elapsed_seconds = t2 - t1
-        elapsed_minutes = elapsed_seconds / 60.0
-        time_string = (
-            f"Total elapsed time for {f.__module__}.{f.__name__} : "
-            f"{elapsed_minutes:.2f} minutes ({elapsed_seconds:.2f} seconds)"
-        )
-
-        logger.info(time_string)
-        return result
-
-    return wrapper

src/sweets/core.py

@@ -7,18 +7,18 @@
 
 import h5py
 import numpy as np
-from dolphin import io, stitching, unwrap
-from dolphin._dates import group_by_date
+from dolphin import stitching, unwrap
 from dolphin.interferogram import Network
-from dolphin.utils import set_num_threads
+from dolphin.utils import _format_date_pair, set_num_threads
 from dolphin.workflows.config import YamlModel
-from opera_utils import group_by_burst
+from opera_utils import group_by_burst, group_by_date
 from pydantic import ConfigDict, Field, field_validator, model_validator
 from shapely import geometry, wkt
 
 from ._burst_db import get_burst_db
 from ._geocode_slcs import create_config_files, run_geocode, run_static_layers
 from ._geometry import stitch_geometry
+from ._ionosphere import download_ionosphere
 from ._log import get_log, log_runtime
 from ._netrc import setup_nasa_netrc
 from ._orbit import download_orbits
@@ -190,12 +190,13 @@ def load(cls, config_file: Filename = "sweets_config.yaml"):
         logger.info(f"Loading config from {config_file}")
         return cls.from_yaml(config_file)
 
-    # Override the constructor to allow recursively construct without validation
     @classmethod
-    def construct(cls, **kwargs):
+    def model_construct(cls, **kwargs):
+        """Override the constructor to allow recursively construct without validation."""
+
         if "asf_query" not in kwargs:
             kwargs["asf_query"] = ASFQuery._construct_empty()
-        return super().construct(
+        return super().model_construct(
             **kwargs,
         )
 
@@ -205,6 +206,7 @@ def __init__(self, **data: Any) -> None:
         self.log_dir = self.work_dir / "logs"
         self.gslc_dir = self.work_dir / "gslcs"
         self.geom_dir = self.work_dir / "geometry"
+        self.iono_dir = self.work_dir / "ionosphere"
         self.ifg_dir = self.work_dir / "interferograms"
         self.stitched_ifg_dir = self.ifg_dir / "stitched"
         self.unw_dir = self.ifg_dir / "unwrapped"
@@ -294,6 +296,8 @@ def _geocode_slcs(self, slc_files, dem_file, burst_db_file):
             out_dir=self.gslc_dir,
             overwrite=self.overwrite,
             using_zipped=not self.asf_query.unzip,
+            # enable_corrections=False,
+            tec_dir=self.iono_dir,
         )
 
         def cfg_to_filename(cfg_path: Path) -> str:
@@ -351,6 +355,14 @@ def cfg_to_static_filename(cfg_path: Path) -> str:
         with ProcessPoolExecutor(max_workers=self.n_workers) as _client:
             new_files = _client.map(run_func, todo_static)
 
+    def _download_ionosphere(self, slc_files) -> list[Path]:
+        """Download one IONEX file per SLC."""
+        self.iono_dir.mkdir(parents=True, exist_ok=True)
+        outputs = []
+        for slc_file in slc_files:
+            outputs.append(download_ionosphere(slc_file, self.iono_dir))
+        return outputs
+
     @log_runtime
     def _stitch_geometry(self, geom_path_list):
         return stitch_geometry(
@@ -425,7 +437,7 @@ def _stitch_interferograms(self, ifg_path_list):
         stitched_ifg_files = []
         for dates, cur_images in grouped_images.items():
             logger.info(f"{dates}: Stitching {len(cur_images)} images.")
-            outfile = self.stitched_ifg_dir / (io._format_date_pair(*dates) + ".int")
+            outfile = self.stitched_ifg_dir / (_format_date_pair(*dates) + ".int")
             stitched_ifg_files.append(outfile)
 
             stitching.merge_images(
@@ -517,6 +529,7 @@ def run(self, starting_step: int = 1):
             burst_db_file = get_burst_db()
             download_orbits(self.asf_query.out_dir, self.orbit_dir)
             rslc_files = self._get_existing_rslcs()
+            self._download_ionosphere(rslc_files)
             self._geocode_slcs(rslc_files, self._dem_filename, burst_db_file)
 
             geom_path_list = self._get_burst_static_layers()

src/sweets/interferogram.py

@@ -13,9 +13,8 @@
 import rioxarray
 from compass.utils.helpers import bbox_to_utm
 from dask.distributed import Client
-from dolphin import utils
 from dolphin.io import DEFAULT_HDF5_OPTIONS, get_raster_xysize, load_gdal, write_arr
-from opera_utils import OPERA_DATASET_NAME
+from opera_utils import OPERA_DATASET_NAME, get_dates
 from pydantic import BaseModel, Field, model_validator
 from rich.progress import track
 
@@ -229,8 +228,8 @@ def _form_ifg_name(slc1: Filename, slc2: Filename, out_dir: Filename) -> Path:
         Path to the interferogram file.
 
     """
-    date1 = utils.get_dates(slc1)[0]
-    date2 = utils.get_dates(slc2)[0]
+    date1 = get_dates(slc1)[0]
+    date2 = get_dates(slc2)[0]
     fmt = "%Y%m%d"
     ifg_name = f"{date1.strftime(fmt)}_{date2.strftime(fmt)}.h5"
     return Path(out_dir) / ifg_name