fixes to time series mapping file generation (#275)

* fixes to time series mapping file generation

* bump version and sort out review comments

CHANGELOG.md

@@ -13,6 +13,11 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.80.3] - 2024-01-11
+### Fixed
+* Script that generates new time series mapping config with the new transformations
+* Changes to the script so that the translation functions can be called from the customer repos
+
 ## [0.80.2] - 2024-01-02
 ### Changed
 * Parse timestamp to datetime object in a time zone naive fashion to stay consistent with preprocessor

cognite/powerops/_version.py

@@ -1 +1 @@
-__version__ = "0.80.2"
+__version__ = "0.80.3"

cognite/powerops/resync/config/_main.py

@@ -199,8 +199,10 @@ def validate_shop_related_files(cls, value):
 
     @classmethod
     def instantiate_from_dict(cls, config: dict) -> CogShopConfig:
-        time_series_mappings_v2 = TimeSeriesMappingV2.load_from_dict(config["time_series_mappings_v2"])
-        config["time_series_mappings_v2"] = [time_series_mappings_v2]
+        time_series_mappings_v2 = []
+        for time_series_mapping in config["time_series_mappings_v2"]:
+            time_series_mappings_v2.append(TimeSeriesMappingV2.load_from_dict(time_series_mapping))
+        config["time_series_mappings_v2"] = time_series_mappings_v2
         return cls(**config)
 
 

cognite/powerops/resync/config/_shared.py

@@ -262,7 +262,7 @@ def dumps(self) -> dict[str, Any]:
     @classmethod
     def load_from_dict(cls, config: dict) -> TimeSeriesMappingV2:
         time_series_mappings_v2 = {"rows": []}
-        for mapping_entry in config[0]["rows"]:
+        for mapping_entry in config["rows"]:
             if transformations := mapping_entry.get("transformations"):
                 loaded_transformations = [TransformationV2.load(t) for t in transformations]
                 del mapping_entry["transformations"]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-power-ops"
-version = "0.80.2"
+version = "0.80.3"
 description = "SDK for power markets operations on Cognite Data Fusion"
 readme = "README.md"
 authors = ["Cognite <[email protected]>"]

scripts/transformationsv2_config_translations.py

@@ -1,36 +1,49 @@
+import hashlib
+import json
+
 from pathlib import Path
 
 import yaml
 
 from cognite.powerops.resync.config import ReSyncConfig, Transformation
 from cognite.powerops.resync.models._shared_v1_v2.cogshop_model import transformations_v2_transformer
 
-REPO_ROOT = Path(__file__).resolve().parent.parent
+
+def hash_dictionary(d: dict) -> str:
+    s_hashable = json.dumps(d).encode("utf-8")
+    m = hashlib.sha256(s_hashable).hexdigest()
+    return m
 
 
-def generate_new_time_series_mappings(old_time_series_mappings: dict, write_path: Path):
-    new_time_series_mapping = [{"rows": []}]
+def generate_new_time_series_mappings(old_time_series_mappings: list[dict], write_path: Path):
+    new_time_series_mapping: list[dict] = []
 
-    for mapping in old_time_series_mappings["rows"]:
-        if transformations := mapping.get("transformations"):
+    for mapping in old_time_series_mappings:
+        new_mapping = {"rows": []}
+        for mapping_entry in mapping["rows"]:
             new_transformations = []
-            object_name = mapping.get("object_name")
-            object_type = mapping.get("object_type")
-            for transformation in transformations:
-                old_transformation = Transformation(**transformation)
-                new_transformation = transformations_v2_transformer(
-                    old_transformation, object_name=object_name, object_type=object_type
-                )
-                new_transformations.append({new_transformation.name: {"parameters": new_transformation.model_dump()}})
-            new_mapping = mapping.copy()
-            new_mapping["transformations"] = new_transformations
-            new_time_series_mapping[0]["rows"].append(new_mapping)
-            continue
-        new_time_series_mapping[0]["rows"].append(mapping)
+            object_name = mapping_entry.get("object_name")
+            object_type = mapping_entry.get("object_type")
+            new_mapping_entry = mapping_entry.copy()
+            if transformations := mapping_entry.get("transformations"):
+                for transformation in transformations:
+                    old_transformation = Transformation(**transformation)
+                    new_transformation = transformations_v2_transformer(
+                        old_transformation, object_name=object_name, object_type=object_type
+                    )
+                    new_transformations.append(
+                        {new_transformation.name: {"parameters": new_transformation.model_dump()}}
+                    )
+                new_mapping_entry["transformations"] = new_transformations
+            new_mapping["rows"].append(new_mapping_entry)
+
+        new_time_series_mapping.append(new_mapping)
 
     # write to yaml file
     write_path.write_text(yaml.safe_dump(new_time_series_mapping))
 
+    print("--- New time series mapping file is up to date with new transformations ---")
+
 
 def generate_new_price_scenarios_mappings(old_price_scenarios_mappings: dict, write_path: Path):
     new_price_scenarios = {}
@@ -50,49 +63,70 @@ def generate_new_price_scenarios_mappings(old_price_scenarios_mappings: dict, wr
     # write to yaml file
     write_path.write_text(yaml.safe_dump(new_price_scenarios))
 
+    print("--- New price scenarios file is up to date with new transformations ---")
 
-def create_new_transformations_file(
-    old_time_series_mappings: dict, old_price_scenarios_mappings: dict, write_path: Path
+
+def _create_transformations_file(
+    old_time_series_mappings: list[dict], old_price_scenarios_mappings: dict, write_path: Path
 ):
     transofrmations_v2 = {"transformations": []}
-
-    for mapping in old_time_series_mappings["rows"]:
-        if transformations := mapping.get("transformations"):
-            new_transformations = []
-            object_name = mapping.get("object_name")
-            object_type = mapping.get("object_type")
-            for transformation in transformations:
-                old_transformation = Transformation(**transformation)
-                new_transformation = transformations_v2_transformer(
-                    old_transformation, object_name=object_name, object_type=object_type
-                )
-                new_transformations.append({new_transformation.name: {"parameters": new_transformation.model_dump()}})
-            transofrmations_v2["transformations"].extend(new_transformations)
+    transformations_cache = set()
+
+    for mapping in old_time_series_mappings:
+        for mapping_entry in mapping["rows"]:
+            if transformations := mapping_entry.get("transformations"):
+                new_transformations = []
+                object_name = mapping_entry.get("object_name")
+                object_type = mapping_entry.get("object_type")
+                for transformation in transformations:
+                    old_transformation = Transformation(**transformation)
+                    new_transformation = transformations_v2_transformer(
+                        old_transformation, object_name=object_name, object_type=object_type
+                    )
+                    new_transformation_dict = {new_transformation.name: {"parameters": new_transformation.model_dump()}}
+                    d_hash = hash_dictionary(new_transformation_dict)
+                    if d_hash not in transformations_cache:
+                        transformations_cache.add(d_hash)
+                        new_transformations.append(new_transformation_dict)
+                transofrmations_v2["transformations"].extend(new_transformations)
 
     for price_scenario in old_price_scenarios_mappings.values():
         if price_scenario.transformations:
             new_transformations = []
             for t in price_scenario.transformations:
                 new_transformation = transformations_v2_transformer(t)
-                new_transformations.append({new_transformation.name: {"parameters": new_transformation.model_dump()}})
+                new_transformation_dict = {new_transformation.name: {"parameters": new_transformation.model_dump()}}
+                d_hash = hash_dictionary(new_transformation_dict)
+                if d_hash not in transformations_cache:
+                    transformations_cache.add(d_hash)
+                    new_transformations.append(new_transformation_dict)
             transofrmations_v2["transformations"].extend(new_transformations)
 
+    print(f"{len(transformations_cache)} number of new unique transformations extracted from config files")
     # write to yaml file
     write_path.write_text(yaml.safe_dump(transofrmations_v2))
 
 
-if __name__ == "__main__":
-    read_path = REPO_ROOT / "tests" / "data" / "demo"
-    write_path = REPO_ROOT / "tests" / "data" / "generated"
-    cdf_project = "powerops-staging"
+def create_new_config_files_with_transformations_v2(configs_path: Path, cdf_project: str):
+    config = ReSyncConfig.from_yamls(configs_path, cdf_project)
+
+    old_time_series_mappings = [mapping.dumps() for mapping in config.cogshop.time_series_mappings]
+    old_price_scenario_mappings = config.market.price_scenario_by_id
+
+    generate_new_price_scenarios_mappings(
+        old_price_scenario_mappings, configs_path / "market" / "price_scenario_by_id_v2.yaml"
+    )
+    generate_new_time_series_mappings(
+        old_time_series_mappings, configs_path / "cogshop" / "time_series_mappings_v2.yaml"
+    )
+
 
-    config = ReSyncConfig.from_yamls(read_path, cdf_project)
+def create_transformationsV2_file(configs_path: Path, write_path: Path, cdf_project: str):
+    config = ReSyncConfig.from_yamls(configs_path, cdf_project)
 
-    old_time_series_mappings = config.cogshop.time_series_mappings[0].dumps()
+    old_time_series_mappings = [mapping.dumps() for mapping in config.cogshop.time_series_mappings]
     old_price_scenario_mappings = config.market.price_scenario_by_id
 
-    generate_new_price_scenarios_mappings(old_price_scenario_mappings, write_path / "price_scenario_by_id_v2.yaml")
-    generate_new_time_series_mappings(old_time_series_mappings, write_path / "time_series_mappings_v2.yaml")
-    create_new_transformations_file(
+    _create_transformations_file(
         old_time_series_mappings, old_price_scenario_mappings, write_path / "transformations_v2.yaml"
     )