Diff production model (#116)

* initial diff on asset type

* relationships

* deleted more relationships without targets and cont from cdf

* abstraction of from_cdf complete, linking not

* any order linking of assets

* a bit of cleanups

* generalizing

* WIP formatting diff

* format_value_added()

* changelog, version bump

---------

Co-authored-by: Katrine Holm <[email protected]>

CHANGELOG.md

@@ -13,6 +13,12 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.34.0] - 2023-08-03
+
+### Added
+* Ability to retrieve production data model from CDF with relationships to other resources.
+* Ability to compare production data model in CDF with production data models generated by `resync`.
+
 ## [0.33.1] - 2023-08-03
 
 ### Changed
@@ -45,7 +51,7 @@ Changes are grouped as follows
 
 ### Added
 
-* Validation that all timeseries exists in CDF before creating relatioinships to them.
+* Validation that all timeseries exists in CDF before creating relationships to them.
 
 
 ## [0.29.0] - 2023-07-20

cognite/powerops/_version.py

@@ -1 +1 @@
-__version__ = "0.33.1"
+__version__ = "0.34.0"

cognite/powerops/clients/shop/api/shop_results_api.py

@@ -54,7 +54,7 @@ def retrieve_objective_function(self, shop_run: ShopRun) -> ObjectiveFunction:
         relationships = retrieve_relationships_from_source_ext_id(
             self._client,
             shop_run.shop_run_event.external_id,
-            RelationshipLabel.OBJECTIVE_SEQUENCE.value,
+            RelationshipLabel.OBJECTIVE_SEQUENCE,
             target_types=["sequence"],
         )
         sequences = self._client.sequences.retrieve_multiple(external_ids=[r.target_external_id for r in relationships])

cognite/powerops/resync/models/cdf_resources.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
-from abc import ABC, abstractmethod
+from abc import ABC, abstractclassmethod, abstractmethod
 from typing import Any, ClassVar, Optional
 
 import pandas as pd
+from cognite.client import CogniteClient
 from cognite.client.data_classes import FileMetadata, Sequence
 from pydantic import BaseModel, ConfigDict
 
@@ -30,10 +31,25 @@ def dump(self, camel_case: bool = False) -> dict[str, Any]:
             dump.pop(read_only_field, None)
         return dump
 
+    @abstractclassmethod
+    def from_cdf(
+        cls,
+        client: CogniteClient,
+        resource_ext_id: str,
+        fetch_content: bool = False,
+    ) -> _CDFResource:
+        ...
+
 
 class CDFSequence(_CDFResource):
     sequence: Sequence
-    content: pd.DataFrame
+    content: Optional[pd.DataFrame]
+
+    def __repr__(self) -> str:
+        return f"CDFSequence(external_id={self.external_id})"
+
+    def __str__(self) -> str:
+        return self.__repr__()
 
     @property
     def external_id(self):
@@ -47,14 +63,53 @@ def _set_data_set_id(self, data_set_id: int):
     def _dump(self, camel_case: bool = False) -> dict[str, Any]:
         return self.sequence.dump(camel_case=camel_case)
 
+    @classmethod
+    def from_cdf(
+        cls,
+        client: CogniteClient,
+        resource_ext_id: str,
+        fetch_content: bool = False,
+    ) -> CDFSequence:
+        sequence = client.sequences.retrieve(external_id=resource_ext_id)
+        if fetch_content:
+            # limit defaults to 100, might not be an issue
+            content = client.sequences.data.retrieve_dataframe(
+                external_id=resource_ext_id,
+                start=0,
+                end=None,
+            )
+        else:
+            content = None
+        return cls(sequence=sequence, content=content)
+
 
 class CDFFile(_CDFResource):
     meta: FileMetadata
     content: Optional[bytes] = None
 
+    def __repr__(self) -> str:
+        return f"CDFFile(external_id={self.external_id})"
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
     @property
     def external_id(self):
         return self.meta.external_id
 
     def _dump(self, camel_case: bool = False) -> dict[str, Any]:
         return self.meta.dump(camel_case=camel_case)
+
+    @classmethod
+    def from_cdf(
+        cls,
+        client: CogniteClient,
+        resource_ext_id: str,
+        fetch_content: bool = False,
+    ) -> CDFFile:
+        meta = client.files.retrieve(external_id=resource_ext_id)
+        if fetch_content:
+            content = client.files.download_bytes(external_id=resource_ext_id)
+        else:
+            content = None
+        return cls(meta=meta, content=content)

cognite/powerops/resync/models/helpers.py

@@ -0,0 +1,134 @@
+import doctest
+import operator
+
+from functools import reduce
+from pprint import pformat
+from deepdiff.model import PrettyOrderedSet
+from typing import Any, Union, Type, get_args, get_origin
+from types import GenericAlias
+
+
+from cognite.client.data_classes import Relationship
+
+
+def isinstance_list(value: Any, type_: Type):
+    return isinstance(value, list) and value and isinstance(value[0], type_)
+
+
+def match_field_from_relationship(model_fields: list[str], relationship: Relationship) -> str:
+    """Find the field on the model that matches the relationship using the label"""
+    if len(relationship.labels) != 1:
+        raise ValueError(f"Expected one label in {relationship.labels=}")
+    label = relationship.labels[0].external_id.split(".")[-1]
+
+    candidates = list(filter(lambda k: label in k, model_fields))
+
+    if len(candidates) != 1:
+        raise ValueError(f"Could not match {relationship.external_id=} to {model_fields=}")
+
+    return candidates[0]
+
+
+def pydantic_model_class_candidate(class_: type = None) -> bool:
+    """GenericAlias is a potential list, get origin checks an optional field"""
+    return isinstance(class_, GenericAlias) or (get_origin(class_) is Union and type(None) in get_args(class_))
+
+
+def format_change_binary(
+    deep_diff: dict[str, dict],
+) -> list[str]:
+    """
+    Formats a dict of changes with updated values to a list of strings
+
+    >>> deep_diff = {
+    ...     "root[0][plants][name]": {
+    ...         "old_value": "Old name",
+    ...         "new_value": "New name",
+    ...     },
+    ... }
+    >>> format_change_binary(deep_diff)
+    [' * [0][plants][name]:\\n', "\\t- 'Old name'\\t", '  -->   ', "'New name'", '\\n']
+    """
+    str_builder = []
+    for path_, change_dict in deep_diff.items():
+        str_builder.extend(
+            (
+                f" * {path_.replace('root', '') }:\n",
+                f'\t- {pformat(change_dict.get("old_value"))}\t',
+                "  -->   ",
+                f'{pformat(change_dict.get("new_value"))}',
+                "\n",
+            )
+        )
+    return str_builder
+
+
+def format_value_removed(deep_diff: dict[str, dict]) -> list[str]:
+    """
+    Formats a dict of values that were removed to a list of strings
+
+    >>> deep_diff = {
+    ...     "root[0]": {
+    ...         "description": "None",
+    ...         "name": "Name"
+    ...     },
+    ... }
+    >>> format_value_removed(deep_diff)
+    [' * [0]:\\n', "\\t- {'description': 'None', 'name': 'Name'}\\n"]
+
+    """
+    str_builder = []
+    for _path, removed in deep_diff.items():
+        str_builder.extend(
+            (
+                f" * {_path.replace('root', '')}:\n",
+                f"\t- {pformat(removed)}\n",
+            )
+        )
+    return str_builder
+
+
+def _get_from_deep_diff_path(deep_diff_path: str, lookup_model: dict) -> Any:
+    """
+    Similar to `format_deep_diff_path` and `get_dict_dot_keys` in
+    `cognite.powerops.clients.shop.data_classes.helpers` but modified
+    to work with the deepdiff format separated from yaml formats
+
+    >>> _path = "root['key_1'][0]['key_3']"
+    >>> _lookup_model = {
+    ...    "key_1": [{"key_2": "value_2", "key_3": "value_3"}]
+    ... }
+    >>> _get_from_deep_diff_path(_path, _lookup_model)
+    'value_3'
+
+    """
+    keys = [
+        int(k) if k.isdigit() else k
+        for k in deep_diff_path.replace("root[", "").replace("'", "").removesuffix("]").split("][")
+    ]
+    try:
+        item = reduce(operator.getitem, keys, lookup_model)
+    except KeyError:
+        item = f"Could not retrieve at {deep_diff_path}"
+    return item
+
+
+def format_value_added(deep_diff: PrettyOrderedSet, lookup_model: dict) -> list[str]:
+    """
+    Formats a dict of values that were added to a list of strings
+    The deep_diff does not contain the new value, so it is fetched from the lookup_model
+    """
+    str_builder = []
+    _path: str = None  # type: ignore
+    for _path in deep_diff:
+        str_builder.extend(
+            (
+                f" * {_path.replace('root', '')}:\n",
+                f"\t- {pformat(_get_from_deep_diff_path(_path, lookup_model))}\n",
+            )
+        )
+    return str_builder
+
+
+if __name__ == "__main__":
+    doctest.testmod()

cognite/powerops/resync/models/market/dayahead.py

@@ -30,6 +30,17 @@ class DayAheadProcess(Process):
     bid_matrix_generator_config: Optional[CDFSequence] = None
     incremental_mapping: list[CDFSequence] = Field(default_factory=list)
 
+    # @classmethod
+    # # i teorien, løftes til AssetType... kanskje
+    # def from_cdf(
+    #     cls,
+    #     client,
+    #     external_id: str,
+    #     fetch_relationships: bool = False,
+    #     fetch_content: bool = False,
+    # ) -> DayAheadProcess:
+    #     raise NotImplementedError()
+
 
 class NordPoolMarket(Market):
     max_price: float