Powerops 1438 investigate missing resources (#119)

* fix model mapping + diff

* Changelog and verison

---------

Co-authored-by: Katrine Holm <[email protected]>

CHANGELOG.md

@@ -13,6 +13,12 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.34.1] - 2023-08-11
+
+### Fixed
+* Mapping of models names accepted by the CLI and the models names' in `resync`.
+
+
 ## [0.34.0] - 2023-08-03
 
 ### Added

cognite/powerops/_models.py

@@ -40,7 +40,7 @@ class _ExtraTypes:
 MODEL_BY_NAME: dict[str, PowerOpsModel] = {
     "production": PowerOpsModel(
         name="Production",
-        description="The production model descripbes the physical assets such as watercourses, "
+        description="The production model describes the physical assets such as watercourses, "
         "plants, and generators located in a price area.",
         graphql_file=GRAPHQL_SCHEMAS / "production.graphql",
         id_=DataModelId(_SPACE, "production", "1"),

cognite/powerops/_version.py

@@ -1 +1 @@
-__version__ = "0.34.0"
+__version__ = "0.34.1"

cognite/powerops/cli.py

@@ -54,6 +54,9 @@ def plan(
     ),
 ):
     log.info(f"Running plan on configuration files located in {path}")
+    if len(models) == 1 and models[0].lower() == "all":
+        models = list(MODEL_BY_NAME.keys())
+
     resync.plan(path, market, echo=log.info, model_names=models)
 
 

cognite/powerops/resync/_main.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional, Callable, overload, Any
 from uuid import uuid4
@@ -31,17 +31,16 @@ def plan(
     path: Path, market: str, echo: Optional[Callable[[str], None]] = None, model_names: Optional[str | list[str]] = None
 ) -> None:
     echo = echo or print
-    model_names = [model_names] if isinstance(model_names, str) else model_names or AVAILABLE_MODELS
+    model_names = _cli_names_to_resync_names(model_names)
     client = get_powerops_client()
     bootstrap_resources, config, models = _load_transform(market, path, client.cdf.config.project, echo, model_names)
-
     _remove_non_existing_relationship_time_series_targets(client.cdf, models, bootstrap_resources, echo)
 
-    # 2.b - preview diff
-    cdf_bootstrap_resources = bootstrap_resources.from_cdf(
+    # ResourceCollection currently collects all resources, not dependent on the local model
+    cdf_bootstrap_resources = ResourceCollection.from_cdf(
         po_client=client, data_set_external_id=config.settings.data_set_external_id
     )
-
+    # 2.b - preview diff
     echo(ResourceCollection.prettify_differences(bootstrap_resources.difference(cdf_bootstrap_resources)))
 
 
@@ -79,7 +78,7 @@ def apply(
 ) -> Model | list[Model]:
     echo = echo or print
     echo_pretty = echo_pretty or echo
-    model_names = [model_names] if isinstance(model_names, str) else model_names or AVAILABLE_MODELS
+    model_names = _cli_names_to_resync_names(model_names)
     client = get_powerops_client()
     collection, config, models = _load_transform(market, path, client.cdf.config.project, echo, model_names)
 
@@ -93,11 +92,7 @@ def apply(
 
     echo("Models About to be uploaded")
     echo_pretty(summaries)
-    if not auto_yes:
-        ans = input("Continue? (y/n)")
-    else:
-        ans = "y"
-
+    ans = "y" if auto_yes else input("Continue? (y/n)")
     if ans.lower() == "y":
         # Step 3 - write bootstrap resources from diffs to CDF
         collection.write_to_cdf(
@@ -108,9 +103,8 @@ def apply(
         echo("Resync written to CDF")
     else:
         echo("Aborting")
-    if len(model_names) == 1:
-        return models[0]
-    return models
+
+    return models[0] if len(model_names) == 1 else models
 
 
 def _load_transform(
@@ -130,7 +124,7 @@ def _load_transform(
 
 def _create_bootstrap_finished_event(echo: Callable[[str], None]) -> Event:
     """Creating a POWEROPS_BOOTSTRAP_FINISHED Event in CDF to signal that bootstrap scripts have been ran"""
-    current_time = int(datetime.utcnow().timestamp() * 1000)  # in milliseconds
+    current_time = int(datetime.now(timezone.utc).timestamp() * 1000)  # in milliseconds
     event = Event(
         start_time=current_time,
         end_time=current_time,
@@ -148,10 +142,15 @@ def _remove_non_existing_relationship_time_series_targets(
     client: CogniteClient, models: list[Model], collection: ResourceCollection, echo: Callable[[str], None]
 ) -> None:
     """Validates that all relationships in the collection have targets that exist in CDF"""
+    to_delete = set()
     for model in models:
         if not isinstance(model, AssetModel):
             continue
         time_series = model.time_series()
+        # retrieve_multiple fails if no time series are provided
+        if not time_series:
+            continue
+
         existing_time_series = client.time_series.retrieve_multiple(
             external_ids=list({t.external_id for t in time_series if t.external_id}), ignore_unknown_ids=True
         )
@@ -175,6 +174,23 @@ def _remove_non_existing_relationship_time_series_targets(
                 collection.relationships.pop(external_id, None)
 
 
+# Only needed while we support both asset models and data models
+def _cli_names_to_resync_names(model_names: Optional[str | list[str]]) -> list[str]:
+    """Map model names as accepted by cli to available models in resync"""
+    if not model_names:
+        return AVAILABLE_MODELS
+    cli_names = {model_names} if isinstance(model_names, str) else set(model_names)
+
+    res: list[str] = []
+    for m in AVAILABLE_MODELS:
+        res.extend(m for c in cli_names if c.casefold() in m.casefold())
+
+    # If any of the market models are present, add the MarketAsset
+    if {"dayahead", "rkom", "benchmark"}.intersection(cli_names):
+        res.append("MarketAsset")
+    return res
+
+
 if __name__ == "__main__":
     demo_data = Path(__file__).parent.parent.parent.parent / "tests" / "test_unit" / "test_bootstrap" / "data" / "demo"
 

cognite/powerops/resync/config/resource_collection.py

@@ -56,6 +56,9 @@ class Config:
         arbitrary_types_allowed = True
         extra = Extra.forbid
 
+    def __len__(self):
+        return len(self.all_cdf_resources)
+
     @property
     def all_cdf_resources(self) -> list[Asset | Relationship | LabelDefinition | Event | CDFSequence]:
         """Not including DM."""
@@ -245,6 +248,8 @@ def from_cdf(
         file_meta = po_client.cdf.files.list(data_set_ids=[data_set_id], limit=None)  # type: ignore[arg-type]
         shop_files = []
         for f in file_meta:
+            if not f.metadata:
+                f.metadata = {}  # Prevent NoneType error
             if f.metadata.get("md5_hash") is None:
                 file_content = po_client.cdf.files.download_bytes(external_id=f.external_id)
                 md5_hash = hashlib.md5(file_content.replace(b"\r\n", b"\n")).hexdigest()

cognite/powerops/resync/models/base.py

@@ -67,9 +67,7 @@ class AssetType(ResourceType, ABC):
 
     @property
     def external_id(self) -> str:
-        if self._external_id:
-            return self._external_id
-        return f"{self.type_}_{self.name}"
+        return self._external_id or f"{self.type_}_{self.name}"
 
     @external_id.setter
     def external_id(self, value: str) -> None:
@@ -87,9 +85,7 @@ def external_id(self, value: str) -> None:
 
     @property
     def type_(self) -> str:
-        if self._type:
-            return self._type
-        return self.parent_external_id.removesuffix("s")
+        return self._type or self.parent_external_id.removesuffix("s")
 
     @type_.setter
     def type_(self, value: str) -> None:

cognite/powerops/resync/models/cogshop.py

@@ -32,3 +32,8 @@ class CogShopAsset(CogShopCore, AssetModel):
     root_asset: ClassVar[Optional[Asset]] = None
     base_mappings: list[CDFSequence] = Field(default_factory=list)
     output_definitions: list[CDFSequence] = Field(default_factory=list)
+
+    @classmethod
+    def from_cdf(cls, client) -> "CogShopAsset":
+        # TODO: undetermined how to handle
+        raise NotImplementedError()

cognite/powerops/resync/models/market/__init__.py

@@ -1,5 +1,6 @@
 from typing import ClassVar, Optional
 
+from cognite.client import CogniteClient
 from cognite.client.data_classes import Asset
 from pydantic.dataclasses import Field
 
@@ -40,10 +41,14 @@ def set_root_asset(
         )
 
     @classmethod
-    def from_cdf(cls, client) -> "MarketModel":
+    def from_cdf(
+        cls,
+        client: CogniteClient,
+        fetch_metadata: bool = True,
+        fetch_content: bool = False,
+    ) -> "MarketModel":
         # TODO:
         # * Missing a from `from_asset` method on each AssetType
-        # * Handle the rewrite from `type_` to `parent_external_id` on AssetType
         raise NotImplementedError()
 
 

cognite/powerops/resync/to_models/to_production_model.py

@@ -209,7 +209,6 @@ def to_production_model(config: ProductionConfig) -> production.ProductionModel:
                 price_area.watercourses.append(watercourse)
 
         model.plants.extend(plants)
-
     return model
 
 

cognite/powerops/resync/to_models/transform.py

@@ -70,7 +70,6 @@ def transform(
     if has_asset_model:
         labels = AssetLabel.as_label_definitions() + RelationshipLabel.as_label_definitions()
         collection.add(labels)
-
     all_models: list[Model] = cast(list[Model], asset_models) + cast(list[Model], data_models)
     for model in all_models:
         collection.add(model.sequences())
@@ -82,5 +81,4 @@ def transform(
         collection.add(asset_model.relationships())
     for data_model in data_models:
         collection.add(data_model.instances())
-
     return collection, all_models

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cognite-power-ops"
-version = "0.34.0"
+version = "0.34.1"
 description = "SDK for power markets operations on Cognite Data Fusion"
 readme = "README.md"
 authors = ["Cognite <[email protected]>"]