From 94d0035b2c6c5d8a909115998b863a39bf94d57b Mon Sep 17 00:00:00 2001 From: Amador Pahim Date: Wed, 6 Jul 2022 09:00:31 +0100 Subject: [PATCH] Local Bundles Cleanup Dropping everything we don't need after the local bundles are gone. Signed-off-by: Amador Pahim --- managedtenants/core/addon_manager.py | 6 -- managedtenants/core/addons_loader/addon.py | 85 ------------------- managedtenants/core/addons_loader/bundle.py | 53 ------------ managedtenants/core/addons_loader/manifest.py | 72 ---------------- managedtenants/core/addons_loader/package.py | 31 ------- managedtenants/core/addons_loader/sss.py | 5 +- managedtenants/data/selectorsyncset.yaml.j2 | 32 ------- tests/addons/test_addon.py | 12 --- tests/sss/test_extra_resources.py | 2 - tests/sss/test_federated_metrics.py | 37 -------- tests/testutils/addon_helpers.py | 1 - 11 files changed, 1 insertion(+), 335 deletions(-) delete mode 100644 managedtenants/core/addon_manager.py delete mode 100644 managedtenants/core/addons_loader/bundle.py delete mode 100644 managedtenants/core/addons_loader/manifest.py delete mode 100644 managedtenants/core/addons_loader/package.py diff --git a/managedtenants/core/addon_manager.py b/managedtenants/core/addon_manager.py deleted file mode 100644 index 41eaa104..00000000 --- a/managedtenants/core/addon_manager.py +++ /dev/null @@ -1,6 +0,0 @@ -from enum import Enum - - -class AddonManager(Enum): - UKNOWN = None - ADDON_OPERATOR = "addon-operator" diff --git a/managedtenants/core/addons_loader/addon.py b/managedtenants/core/addons_loader/addon.py index 668189d6..e09c951f 100644 --- a/managedtenants/core/addons_loader/addon.py +++ b/managedtenants/core/addons_loader/addon.py @@ -1,25 +1,14 @@ -from pathlib import Path - import jsonschema import semver import yaml from jinja2 import FileSystemLoader -from sretoolbox.container import Image -from managedtenants.core.addon_manager import AddonManager -from managedtenants.core.addons_loader.bundle import Bundle from managedtenants.core.addons_loader.exceptions import AddonLoadError -from managedtenants.core.addons_loader.package import Package from managedtenants.core.addons_loader.sss import Sss from managedtenants.data.paths import SCHEMAS_DIR from managedtenants.utils.general_utils import parse_version_from_imageset_name -from managedtenants.utils.hash import hash_dir_sha256, hash_sha256 from managedtenants.utils.schema import load_schema -# IDs of addons that are managed by the addon-operator -# These addon IDs _MUST_ be stable and not changed or bad things will happen -# (this applies to addon IDs in general, but it's worth pointing out here, too) -_ADDON_OPERATOR_ADDON_IDS = [] _PERMITTED_SUBSCRIPTION_CONFIGS = ["env"] @@ -44,7 +33,6 @@ def __init__( self, path, environment, - override_manager=None, imageset_latest_only=False, ): self.path = path @@ -57,8 +45,6 @@ def __init__( if self.imageset_version is None: self.imagesets_path = None self.imageset = None - self.bundles = None - self.package = None self.catalog_image = None else: if all( @@ -77,35 +63,12 @@ def __init__( self.path / f"addonimagesets/{environment}" ) self.imageset = self.load_imageset(self.imageset_version) - self.package = None - self.bundles = None self.catalog_image = self.imageset["indexImage"] elif "indexImage" in self.metadata: self.imagesets_path = None self.imageset = None - self.bundles = None - self.package = None self.catalog_image = self.metadata["indexImage"] - else: - # Local bundles doesn't exist anymore, - # we have to clean this up - self.imagesets_path = None - self.imageset = None - self.bundles = self.load_bundles(metadata=self.metadata) - self.package = Package(addon=self) - self.catalog_image = self.get_image_name(environment=environment) - self.image_tag = ( # Used with .format(hash=...) - f'{self.metadata["quayRepo"]}:{environment}-{{hash}}' - ) - - if self.metadata["id"] not in _ADDON_OPERATOR_ADDON_IDS: - self.manager = AddonManager.UKNOWN - else: - self.manager = AddonManager.ADDON_OPERATOR - - if override_manager is not None: - self.manager = override_manager self.sss = Sss(addon=self) @@ -131,23 +94,6 @@ def get_subscription_config(self): return self.imageset.get("subscriptionConfig") return self.metadata.get("subscriptionConfig") - def get_image_name(self, environment): - """ - Creates a deterministic image name that is unique per - bundles + metadata hashed content. - """ - bundles_dir = self.path / "bundles" - bundles_hash = hash_dir_sha256(path=bundles_dir) - - metadata_dir = self.path / "metadata" / environment - metadata_hash = hash_dir_sha256(path=metadata_dir) - - hash_tag = hash_sha256(items=(bundles_hash, metadata_hash)) - - return Image( - f'{self.metadata["quayRepo"]}:{environment}-{hash_tag[:7]}' - ) - def load_metadata(self, environment): metadata_path = self.path / "metadata" / environment / "addon.yaml" metadata_dir = self.path / "metadata" / environment @@ -298,10 +244,6 @@ def _validate_extra_resources(self, environment, metadata): f"referenced resource {resource_path} not found" ) - @staticmethod - def instantiate_bundle(args): - return Bundle(path=args[0], metadata=args[1]) - @staticmethod def load_yaml(path): try: @@ -311,33 +253,6 @@ def load_yaml(path): except yaml.YAMLError: return None - def load_bundles(self, metadata): - bundles_path = self.path / "bundles" - - bundles_to_load = [] - for item in sorted(filter(Path.is_dir, bundles_path.iterdir())): - if item.name == "OWNERS": - continue - - bundles_to_load.append((item, metadata)) - - # force list to not lazily evaluate the returned iterator of map() - bundles = list(map(self.instantiate_bundle, bundles_to_load)) - - self._validate_bundle_names(bundles) - - return bundles - - @staticmethod - def _validate_bundle_names(bundles): - for bundle in bundles: - try: - semver.parse(bundle.path.name) - except ValueError as e: - raise AddonLoadError( - f"{bundle.path} directory must be a semantic version" - ) from e - def __repr__(self): return f"{self.__class__.__name__}({repr(self.name)})" diff --git a/managedtenants/core/addons_loader/bundle.py b/managedtenants/core/addons_loader/bundle.py deleted file mode 100644 index 30a70538..00000000 --- a/managedtenants/core/addons_loader/bundle.py +++ /dev/null @@ -1,53 +0,0 @@ -from managedtenants.core.addons_loader.exceptions import BundleLoadError -from managedtenants.core.addons_loader.manifest import Manifest - - -class Bundle: - def __init__(self, path, metadata): - self.path = path - self.manifests = self._load_manifests(metadata=metadata) - - @property - def name(self): - return self.path.name - - @staticmethod - def instantiate_manifest(args): - return Manifest(path=args[0], metadata=args[1]) - - def _load_manifests(self, metadata): - manifests_to_load = [] - for item in sorted(self.path.iterdir()): - if item.name == "OWNERS": - continue - - if item.suffix not in [".yaml", ".yml", ".j2"]: - raise BundleLoadError( - f'{item}: only ".yaml" or ".j2" files are supported' - ) - - manifests_to_load.append((item, metadata)) - - # force list to not lazily evaluate the returned iterator of map() - manifests = list(map(self.instantiate_manifest, manifests_to_load)) - - self._validate_number_of_csvs(manifests) - - return manifests - - def _validate_number_of_csvs(self, manifests): - csv_count = 0 - for manifest in manifests: - if manifest.data["kind"] != "ClusterServiceVersion": - continue - csv_count += 1 - - if csv_count == 1: - return - - raise BundleLoadError( - f"{self.path}: exactly one ClusterServiceVersion must exist" - ) - - def __repr__(self): - return f"{self.__class__.__name__}({repr(self.name)})" diff --git a/managedtenants/core/addons_loader/manifest.py b/managedtenants/core/addons_loader/manifest.py deleted file mode 100644 index 23e30eb6..00000000 --- a/managedtenants/core/addons_loader/manifest.py +++ /dev/null @@ -1,72 +0,0 @@ -import json - -import yaml -from jinja2 import Environment, FileSystemLoader, StrictUndefined -from jinja2.exceptions import UndefinedError - -from managedtenants.core.addons_loader.exceptions import ManifestLoadError - - -class Manifest: - def __init__(self, path, metadata): - self.path = path - self._metadata = metadata - self.data = self._get_data() - - @property - def name(self): - return self.path.name - - @property - def yaml(self): - return yaml.dump(self.data, Dumper=yaml.CSafeDumper) - - @property - def json(self): - return json.dumps(self.data, indent=4) - - def _get_data(self): - data = self._load() - self._validate_csv_install_mode(data) - return data - - def _load(self): - if self.path.suffix == ".j2": - try: - loader = FileSystemLoader(searchpath=str(self.path.parent)) - env = Environment(loader=loader, undefined=StrictUndefined) - template = env.get_template(str(self.name)) - content = template.render(**self._metadata["bundleParameters"]) - except UndefinedError as details: - raise ManifestLoadError( - f"error templating {self.path}: {details.message}" - ) from details - else: - with open(self.path, encoding="utf-8") as file_obj: - content = file_obj.read() - try: - return yaml.load(content, Loader=yaml.CSafeLoader) - except yaml.error.MarkedYAMLError as details: - raise ManifestLoadError(f"{self.path}: {details}") from details - - def _validate_csv_install_mode(self, data): - if data["kind"] != "ClusterServiceVersion": - return - - if "installModes" in data["spec"]: - expected_csv_install_mode = { - "supported": True, - "type": self._metadata["installMode"], - } - if expected_csv_install_mode in data["spec"]["installModes"]: - return - - raise ManifestLoadError( - f"{self.path}: installMode " - f'"{self._metadata["installMode"]}" ' - "must be supported by the " - "ClusterServiceVersion" - ) - - def __repr__(self): - return f"{self.__class__.__name__}({repr(self.name)})" diff --git a/managedtenants/core/addons_loader/package.py b/managedtenants/core/addons_loader/package.py deleted file mode 100644 index b1abc12d..00000000 --- a/managedtenants/core/addons_loader/package.py +++ /dev/null @@ -1,31 +0,0 @@ -import json - -import yaml - - -class Package: - def __init__(self, addon): - self._addon = addon - self.data = self._get_data() - - @property - def name(self): - return f"{self._addon.name}.package.yml" - - @property - def yaml(self): - return yaml.dump(self.data, Dumper=yaml.CSafeDumper) - - @property - def json(self): - return json.dumps(self.data, indent=4) - - def _get_data(self): - return dict( - packageName=self._addon.metadata["id"], - defaultChannel=self._addon.metadata["defaultChannel"], - channels=self._addon.metadata["channels"], - ) - - def __repr__(self): - return f"{self.__class__.__name__}({repr(self._addon.name)})" diff --git a/managedtenants/core/addons_loader/sss.py b/managedtenants/core/addons_loader/sss.py index 1b670e33..0cc4c858 100644 --- a/managedtenants/core/addons_loader/sss.py +++ b/managedtenants/core/addons_loader/sss.py @@ -8,7 +8,6 @@ from jinja2.exceptions import UndefinedError from sretoolbox.utils.logger import get_text_logger -from managedtenants.core.addon_manager import AddonManager from managedtenants.core.addons_loader.exceptions import SssLoadError from managedtenants.data.paths import DATA_DIR @@ -49,9 +48,7 @@ def _get_data(self): # pylint: disable=unnecessary-lambda env.filters["merge_dicts"] = lambda d1, d2: ChainMap(d1, d2) template = env.get_template(str(self._sss_filename)) - content = template.render( - AddonManager=AddonManager, ADDON=self._addon - ) + content = template.render(ADDON=self._addon) except UndefinedError as details: raise SssLoadError( f"error templating {self._sss_filename}: {details.message}" diff --git a/managedtenants/data/selectorsyncset.yaml.j2 b/managedtenants/data/selectorsyncset.yaml.j2 index 23ecfc93..460435b8 100644 --- a/managedtenants/data/selectorsyncset.yaml.j2 +++ b/managedtenants/data/selectorsyncset.yaml.j2 @@ -6,7 +6,6 @@ metadata: {} apiVersion: v1 items: {# addon v1 create OLM resources manually #} -{% if ADDON.manager == AddonManager.UKNOWN %} - apiVersion: hive.openshift.io/v1 kind: SelectorSyncSet metadata: @@ -214,37 +213,6 @@ items: {{ expand_dict(ADDON.metadata['monitoring']['matchLabels']) | indent(12) }} {% endif %} -{# addon v2 Addon CR is created by OCM, only secret management handled by SSS #} -{% elif ADDON.manager == AddonManager.ADDON_OPERATOR %} -- apiVersion: hive.openshift.io/v1 - kind: SelectorSyncSet - metadata: - name: addon-{{ADDON.metadata['id']}} - spec: - clusterDeploymentSelector: - matchLabels: - {{ADDON.metadata['label']}}: 'true' - resourceApplyMode: Sync - - resources: -{# TODO: move pullsecret management into the addon-operator #} -{% for namespace in ADDON.metadata['namespaces'] %} - {% if ADDON.metadata['pullSecret'] is defined %} - - apiVersion: v1 - kind: Secret - metadata: - name: addon-{{ADDON.metadata['id']}}-pullsecret - namespace: {{namespace}} - labels: - addon-pullsecret: "{{ADDON.metadata['id']}}" - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: >- - {{ADDON.metadata['pullSecret']}} - {% endif %} -{% endfor %} -{% endif %} - - apiVersion: hive.openshift.io/v1 kind: SelectorSyncSet metadata: diff --git a/tests/addons/test_addon.py b/tests/addons/test_addon.py index ad3283cc..e79fc968 100644 --- a/tests/addons/test_addon.py +++ b/tests/addons/test_addon.py @@ -36,18 +36,6 @@ def test_addon_metadata(addon, addon_type, request): assert addon.metadata == load_yaml(addon.path / "metadata/stage/addon.yaml") -@pytest.mark.parametrize( - "addon,addon_type", - [ - ("addon_with_imageset", ADDON_WITH_IMAGESET_TYPE), - ], -) -def test_addon_bundles(addon, addon_type, request): - """Test that addon bundles are loaded or not.""" - addon = request.getfixturevalue(addon) - assert addon.bundles is None - - @pytest.mark.parametrize( "addon,addon_type", [ diff --git a/tests/sss/test_extra_resources.py b/tests/sss/test_extra_resources.py index bd60a8cf..e225627a 100644 --- a/tests/sss/test_extra_resources.py +++ b/tests/sss/test_extra_resources.py @@ -7,7 +7,6 @@ from jinja2 import FileSystemLoader import tests.testutils.strategies as custom_strategies -from managedtenants.core.addon_manager import AddonManager from managedtenants.core.addons_loader.sss import Sss from tests.testutils.extra_resources import DEADMANSSNITCH, PAGERDUTYINTEGRATION @@ -23,7 +22,6 @@ def test_multiple_extra_resources_v1(data, is_v2): addon.metadata["extraResources"] = [] if is_v2: - addon.manager = AddonManager.ADDON_OPERATOR addon.metadata["indexImage"] = custom_strategies.quay_image( addon.metadata["id"] ) diff --git a/tests/sss/test_federated_metrics.py b/tests/sss/test_federated_metrics.py index a5f7abb7..7d78cf1c 100644 --- a/tests/sss/test_federated_metrics.py +++ b/tests/sss/test_federated_metrics.py @@ -5,7 +5,6 @@ from hypothesis import given import tests.testutils.strategies as custom_strategies -from managedtenants.core.addon_manager import AddonManager from managedtenants.core.addons_loader.sss import Sss @@ -60,39 +59,3 @@ def test_namespace_and_servicemonitor_v1(data): for k, v in addon.metadata["monitoring"]["matchLabels"].items(): assert sm["spec"]["selector"]["matchLabels"][k] == v - - -# For v2, the namespace and ServiceMonitor should be created and managed -# by the addon-operator. -@given(data=hypothesis_strategies.data()) -def test_namespace_and_servicemonitor_v2(data): - env = data.draw(custom_strategies.environment()) - addon = data.draw(custom_strategies.addon(env)) - addon.metadata["monitoring"] = { - "matchNames": data.draw( - hypothesis_strategies.lists(custom_strategies.k8s_name()) - ), - "matchLabels": data.draw(custom_strategies.labels()), - } - addon.manager = AddonManager.ADDON_OPERATOR - addon.metadata["indexImage"] = custom_strategies.quay_image( - addon.metadata["id"] - ) - - # Rerender selectorsyncset.yaml.j2 - addon.sss = Sss(addon=addon) - walker = addon.sss.walker() - - expected_ns_name = f"redhat-monitoring-{addon.metadata['id']}" - - # validate namespace does not exist - found = False - for ns, _ in walker["sss_deploy"]["spec"]["resources"]["Namespace"]: - if ns == expected_ns_name: - found = True - break - assert not found - - # validate servicemonitor does not exist - with pytest.raises(IndexError): - _ = walker["sss_deploy"]["spec"]["resources"]["ServiceMonitor"][0] diff --git a/tests/testutils/addon_helpers.py b/tests/testutils/addon_helpers.py index 8eef9650..c9f4fb84 100644 --- a/tests/testutils/addon_helpers.py +++ b/tests/testutils/addon_helpers.py @@ -9,7 +9,6 @@ from sretoolbox.container.image import Image from managedtenants.bundles.utils import run -from managedtenants.core.addon_manager import AddonManager from managedtenants.core.addons_loader.addon import Addon from managedtenants.core.addons_loader.sss import Sss from managedtenants.data.paths import SCHEMAS_DIR