diff --git a/CHANGELOG.md b/CHANGELOG.md index 572b1e6a9..4d7ee33cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ Changes are grouped as follows - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [0.34.1] - 2023-08-11 + +### Fixed +* Mapping of models names accepted by the CLI and the models names' in `resync`. + + ## [0.34.0] - 2023-08-03 ### Added diff --git a/cognite/powerops/_models.py b/cognite/powerops/_models.py index e73d2c593..8bcea7a20 100644 --- a/cognite/powerops/_models.py +++ b/cognite/powerops/_models.py @@ -40,7 +40,7 @@ class _ExtraTypes: MODEL_BY_NAME: dict[str, PowerOpsModel] = { "production": PowerOpsModel( name="Production", - description="The production model descripbes the physical assets such as watercourses, " + description="The production model describes the physical assets such as watercourses, " "plants, and generators located in a price area.", graphql_file=GRAPHQL_SCHEMAS / "production.graphql", id_=DataModelId(_SPACE, "production", "1"), diff --git a/cognite/powerops/_version.py b/cognite/powerops/_version.py index eab0e9cdf..876561368 100644 --- a/cognite/powerops/_version.py +++ b/cognite/powerops/_version.py @@ -1 +1 @@ -__version__ = "0.34.0" +__version__ = "0.34.1" diff --git a/cognite/powerops/cli.py b/cognite/powerops/cli.py index c9e91c306..96b3de8dd 100644 --- a/cognite/powerops/cli.py +++ b/cognite/powerops/cli.py @@ -54,6 +54,9 @@ def plan( ), ): log.info(f"Running plan on configuration files located in {path}") + if len(models) == 1 and models[0].lower() == "all": + models = list(MODEL_BY_NAME.keys()) + resync.plan(path, market, echo=log.info, model_names=models) diff --git a/cognite/powerops/resync/_main.py b/cognite/powerops/resync/_main.py index 9615d3e9b..07abd231b 100644 --- a/cognite/powerops/resync/_main.py +++ b/cognite/powerops/resync/_main.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Optional, Callable, overload, Any from uuid import uuid4 @@ -31,17 +31,16 @@ def plan( path: Path, market: str, echo: Optional[Callable[[str], None]] = None, model_names: Optional[str | list[str]] = None ) -> None: echo = echo or print - model_names = [model_names] if isinstance(model_names, str) else model_names or AVAILABLE_MODELS + model_names = _cli_names_to_resync_names(model_names) client = get_powerops_client() bootstrap_resources, config, models = _load_transform(market, path, client.cdf.config.project, echo, model_names) - _remove_non_existing_relationship_time_series_targets(client.cdf, models, bootstrap_resources, echo) - # 2.b - preview diff - cdf_bootstrap_resources = bootstrap_resources.from_cdf( + # ResourceCollection currently collects all resources, not dependent on the local model + cdf_bootstrap_resources = ResourceCollection.from_cdf( po_client=client, data_set_external_id=config.settings.data_set_external_id ) - + # 2.b - preview diff echo(ResourceCollection.prettify_differences(bootstrap_resources.difference(cdf_bootstrap_resources))) @@ -79,7 +78,7 @@ def apply( ) -> Model | list[Model]: echo = echo or print echo_pretty = echo_pretty or echo - model_names = [model_names] if isinstance(model_names, str) else model_names or AVAILABLE_MODELS + model_names = _cli_names_to_resync_names(model_names) client = get_powerops_client() collection, config, models = _load_transform(market, path, client.cdf.config.project, echo, model_names) @@ -93,11 +92,7 @@ def apply( echo("Models About to be uploaded") echo_pretty(summaries) - if not auto_yes: - ans = input("Continue? (y/n)") - else: - ans = "y" - + ans = "y" if auto_yes else input("Continue? (y/n)") if ans.lower() == "y": # Step 3 - write bootstrap resources from diffs to CDF collection.write_to_cdf( @@ -108,9 +103,8 @@ def apply( echo("Resync written to CDF") else: echo("Aborting") - if len(model_names) == 1: - return models[0] - return models + + return models[0] if len(model_names) == 1 else models def _load_transform( @@ -130,7 +124,7 @@ def _load_transform( def _create_bootstrap_finished_event(echo: Callable[[str], None]) -> Event: """Creating a POWEROPS_BOOTSTRAP_FINISHED Event in CDF to signal that bootstrap scripts have been ran""" - current_time = int(datetime.utcnow().timestamp() * 1000) # in milliseconds + current_time = int(datetime.now(timezone.utc).timestamp() * 1000) # in milliseconds event = Event( start_time=current_time, end_time=current_time, @@ -148,10 +142,15 @@ def _remove_non_existing_relationship_time_series_targets( client: CogniteClient, models: list[Model], collection: ResourceCollection, echo: Callable[[str], None] ) -> None: """Validates that all relationships in the collection have targets that exist in CDF""" + to_delete = set() for model in models: if not isinstance(model, AssetModel): continue time_series = model.time_series() + # retrieve_multiple fails if no time series are provided + if not time_series: + continue + existing_time_series = client.time_series.retrieve_multiple( external_ids=list({t.external_id for t in time_series if t.external_id}), ignore_unknown_ids=True ) @@ -175,6 +174,23 @@ def _remove_non_existing_relationship_time_series_targets( collection.relationships.pop(external_id, None) +# Only needed while we support both asset models and data models +def _cli_names_to_resync_names(model_names: Optional[str | list[str]]) -> list[str]: + """Map model names as accepted by cli to available models in resync""" + if not model_names: + return AVAILABLE_MODELS + cli_names = {model_names} if isinstance(model_names, str) else set(model_names) + + res: list[str] = [] + for m in AVAILABLE_MODELS: + res.extend(m for c in cli_names if c.casefold() in m.casefold()) + + # If any of the market models are present, add the MarketAsset + if {"dayahead", "rkom", "benchmark"}.intersection(cli_names): + res.append("MarketAsset") + return res + + if __name__ == "__main__": demo_data = Path(__file__).parent.parent.parent.parent / "tests" / "test_unit" / "test_bootstrap" / "data" / "demo" diff --git a/cognite/powerops/resync/config/resource_collection.py b/cognite/powerops/resync/config/resource_collection.py index 0d9b45cbc..4da21c2e9 100644 --- a/cognite/powerops/resync/config/resource_collection.py +++ b/cognite/powerops/resync/config/resource_collection.py @@ -56,6 +56,9 @@ class Config: arbitrary_types_allowed = True extra = Extra.forbid + def __len__(self): + return len(self.all_cdf_resources) + @property def all_cdf_resources(self) -> list[Asset | Relationship | LabelDefinition | Event | CDFSequence]: """Not including DM.""" @@ -245,6 +248,8 @@ def from_cdf( file_meta = po_client.cdf.files.list(data_set_ids=[data_set_id], limit=None) # type: ignore[arg-type] shop_files = [] for f in file_meta: + if not f.metadata: + f.metadata = {} # Prevent NoneType error if f.metadata.get("md5_hash") is None: file_content = po_client.cdf.files.download_bytes(external_id=f.external_id) md5_hash = hashlib.md5(file_content.replace(b"\r\n", b"\n")).hexdigest() diff --git a/cognite/powerops/resync/models/base.py b/cognite/powerops/resync/models/base.py index d291e0f8b..985cf1405 100644 --- a/cognite/powerops/resync/models/base.py +++ b/cognite/powerops/resync/models/base.py @@ -67,9 +67,7 @@ class AssetType(ResourceType, ABC): @property def external_id(self) -> str: - if self._external_id: - return self._external_id - return f"{self.type_}_{self.name}" + return self._external_id or f"{self.type_}_{self.name}" @external_id.setter def external_id(self, value: str) -> None: @@ -87,9 +85,7 @@ def external_id(self, value: str) -> None: @property def type_(self) -> str: - if self._type: - return self._type - return self.parent_external_id.removesuffix("s") + return self._type or self.parent_external_id.removesuffix("s") @type_.setter def type_(self, value: str) -> None: diff --git a/cognite/powerops/resync/models/cogshop.py b/cognite/powerops/resync/models/cogshop.py index 5b9fb1dd4..9a1710a1a 100644 --- a/cognite/powerops/resync/models/cogshop.py +++ b/cognite/powerops/resync/models/cogshop.py @@ -32,3 +32,8 @@ class CogShopAsset(CogShopCore, AssetModel): root_asset: ClassVar[Optional[Asset]] = None base_mappings: list[CDFSequence] = Field(default_factory=list) output_definitions: list[CDFSequence] = Field(default_factory=list) + + @classmethod + def from_cdf(cls, client) -> "CogShopAsset": + # TODO: undetermined how to handle + raise NotImplementedError() diff --git a/cognite/powerops/resync/models/market/__init__.py b/cognite/powerops/resync/models/market/__init__.py index 1123404a0..0652ed420 100644 --- a/cognite/powerops/resync/models/market/__init__.py +++ b/cognite/powerops/resync/models/market/__init__.py @@ -1,5 +1,6 @@ from typing import ClassVar, Optional +from cognite.client import CogniteClient from cognite.client.data_classes import Asset from pydantic.dataclasses import Field @@ -40,10 +41,14 @@ def set_root_asset( ) @classmethod - def from_cdf(cls, client) -> "MarketModel": + def from_cdf( + cls, + client: CogniteClient, + fetch_metadata: bool = True, + fetch_content: bool = False, + ) -> "MarketModel": # TODO: # * Missing a from `from_asset` method on each AssetType - # * Handle the rewrite from `type_` to `parent_external_id` on AssetType raise NotImplementedError() diff --git a/cognite/powerops/resync/to_models/to_production_model.py b/cognite/powerops/resync/to_models/to_production_model.py index 7a1cddf0e..3550204dd 100644 --- a/cognite/powerops/resync/to_models/to_production_model.py +++ b/cognite/powerops/resync/to_models/to_production_model.py @@ -209,7 +209,6 @@ def to_production_model(config: ProductionConfig) -> production.ProductionModel: price_area.watercourses.append(watercourse) model.plants.extend(plants) - return model diff --git a/cognite/powerops/resync/to_models/transform.py b/cognite/powerops/resync/to_models/transform.py index ac5d3a969..e80b960c3 100644 --- a/cognite/powerops/resync/to_models/transform.py +++ b/cognite/powerops/resync/to_models/transform.py @@ -70,7 +70,6 @@ def transform( if has_asset_model: labels = AssetLabel.as_label_definitions() + RelationshipLabel.as_label_definitions() collection.add(labels) - all_models: list[Model] = cast(list[Model], asset_models) + cast(list[Model], data_models) for model in all_models: collection.add(model.sequences()) @@ -82,5 +81,4 @@ def transform( collection.add(asset_model.relationships()) for data_model in data_models: collection.add(data_model.instances()) - return collection, all_models diff --git a/pyproject.toml b/pyproject.toml index 130d68a14..aa4e6b658 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cognite-power-ops" -version = "0.34.0" +version = "0.34.1" description = "SDK for power markets operations on Cognite Data Fusion" readme = "README.md" authors = ["Cognite "]