remove time zones from dates

anemoi/datasets/utils/dates/__init__.py

@@ -9,6 +9,8 @@
 import datetime
 import warnings
 
+def no_time_zone(date):
+    return date.replace(tzinfo=None)
 
 def frequency_to_hours(frequency):
     if isinstance(frequency, int):
@@ -22,7 +24,7 @@ def frequency_to_hours(frequency):
 
 def normalize_date(x):
     if isinstance(x, str):
-        return datetime.datetime.fromisoformat(x)
+        return no_time_zone(datetime.datetime.fromisoformat(x))
     return x
 
 
@@ -81,7 +83,7 @@ def summary(self):
 
 class ValuesDates(Dates):
     def __init__(self, values, **kwargs):
-        self.values = sorted(values)
+        self.values = sorted([no_time_zone(_) for _ in values])
         super().__init__(**kwargs)
 
     def __repr__(self):
@@ -109,6 +111,9 @@ def _(x):
         if isinstance(end, datetime.date) and not isinstance(end, datetime.datetime):
             end = datetime.datetime(end.year, end.month, end.day)
 
+        start = no_time_zone(start)
+        end = no_time_zone(end)
+
         if end <= start:
             raise ValueError(f"End date {end} must be after start date {start}")
 

anemoi/datasets/utils/dates/groups.py

@@ -8,7 +8,7 @@
 
 import itertools
 
-from anemoi.datasets.utils.dates import Dates
+from anemoi.datasets.utils.dates import Dates, no_time_zone
 
 
 class Groups:
@@ -64,7 +64,7 @@ def __len__(self):
 
 class Filter:
     def __init__(self, missing):
-        self.missing = missing
+        self.missing = [no_time_zone(m) for m in missing]
 
     def __call__(self, dates):
         return [d for d in dates if d not in self.missing]

docs/building/building3.txt

@@ -0,0 +1,29 @@
+┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈┈
+📦 Path          : dataset.zarr
+🔢 Format version: 0.20.0
+
+📅 Start      : 2024-01-01 00:00
+📅 End        : 2024-01-01 18:00
+⏰ Frequency  : 6h
+🚫 Missing    : 0
+🌎 Resolution : 1.0
+🌎 Field shape: [181, 360]
+
+📐 Shape      : 4 × 8 × 1 × 65,160 (8 MiB)
+💽 Size       : 3.1 MiB (3,283,650)
+📁 Files      : 34
+
+   Index │ Variable     │         Min │      Max │      Mean │    Stdev
+   ──────┼──────────────┼─────────────┼──────────┼───────────┼─────────
+       0 │ 10u          │    -24.3116 │    25.79 │ 0.0595319 │   5.5856
+       1 │ 10v          │    -21.2397 │   21.851 │ -0.270924 │  4.23947
+       2 │ 2t           │     214.979 │  319.111 │   277.775 │  19.9318
+       3 │ cos_latitude │ 6.12323e-17 │        1 │  0.633086 │ 0.310546
+       4 │ insolation   │           0 │ 0.999995 │  0.231949 │ 0.299927
+       5 │ lsm          │           0 │        1 │  0.335152 │ 0.464236
+       6 │ msl          │     95708.5 │   104284 │    100867 │  1452.67
+       7 │ sin_latitude │          -1 │        1 │         0 │ 0.709057
+   ──────┴──────────────┴─────────────┴──────────┴───────────┴─────────
+🔋 Dataset ready, last update 17 seconds ago.
+📊 Statistics ready.
+

docs/building/building3.yaml

@@ -0,0 +1,22 @@
+dates:
+  start: 2024-01-01T00:00:00Z
+  end: 2024-01-01T18:00:00Z
+  frequency: 6h
+
+input:
+  join:
+  - mars:
+      param: [2t, msl, 10u, 10v, lsm]
+      levtype: sfc
+      grid: [1, 1]
+  - mars:
+      param: [q, t, z]
+      levtype: pl
+      level: [50, 100]
+      grid: [1, 1]
+  - constants:
+      template: ${input.join.0.mars}
+      param:
+      - cos_latitude
+      - sin_latitude
+      - insolation

docs/building/introduction.rst

@@ -118,4 +118,29 @@ This will build the following dataset:
 .. literalinclude:: building2.txt
    :language: console
 
-.. note:: Please note that the pressure levels parameters are named `param_level`. This is the default behaviour. See :ref:`remapping_option` for more information.
+.. note::
+
+   Please note that the pressure levels parameters are named
+   `param_level`. This is the default behaviour. See
+   :ref:`remapping_option` for more information.
+
+Adding some forcing variables
+==
+
+When training a data-driven models, some forcing variables may be required
+such as the solar radiation, the time of day, the day in the year, etc.
+
+These are provided by the ``constants`` source. In that example, we add a few of them.
+The `template` option is used to point to another source, in that case the first
+instance of ``mars``. This source is used to get information about the grid points,
+as some of the forcing variables are grid dependent.
+
+.. literalinclude:: building3.yaml
+   :language: yaml
+
+This will build the following dataset:
+
+.. literalinclude:: building3.txt
+   :language: console
+
+See :ref:`forcing_variables` for more information about forcing variables.

docs/building/naming_variables.rst

@@ -12,8 +12,8 @@ The rename filter is used to rename variables in a dataset.
 
 .. _remapping_option:
 
-*************************
- remapping  option
-*************************
+******************
+ remapping option
+******************
 
 TODO.

docs/building/sources.rst

@@ -28,14 +28,17 @@
 
 .. include:: sources/opendap.rst
 
-***********
- constants
-***********
-
-.. include:: sources/constants.rst
 
 ***************
  accumulations
 ***************
 
 .. include:: sources/accumulations.rst
+
+.. _forcing_variables:
+
+***********
+ constants
+***********
+
+.. include:: sources/constants.rst