From a5943414b7690077c21ca5a3dc5733e61a834a29 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 21 Dec 2024 13:48:45 +0100 Subject: [PATCH 01/25] Copy internal plugin as is --- src/python/pants/backend/k8s/BUILD | 1 + src/python/pants/backend/k8s/__init__.py | 0 src/python/pants/backend/k8s/goals/BUILD | 1 + .../pants/backend/k8s/goals/__init__.py | 0 src/python/pants/backend/k8s/goals/deploy.py | 109 +++++++++++++++++ src/python/pants/backend/k8s/k8s_subsystem.py | 23 ++++ .../pants/backend/k8s/kubectl_subsystem.py | 110 ++++++++++++++++++ src/python/pants/backend/k8s/register.py | 14 +++ src/python/pants/backend/k8s/targets.py | 89 ++++++++++++++ 9 files changed, 347 insertions(+) create mode 100644 src/python/pants/backend/k8s/BUILD create mode 100644 src/python/pants/backend/k8s/__init__.py create mode 100644 src/python/pants/backend/k8s/goals/BUILD create mode 100644 src/python/pants/backend/k8s/goals/__init__.py create mode 100644 src/python/pants/backend/k8s/goals/deploy.py create mode 100644 src/python/pants/backend/k8s/k8s_subsystem.py create mode 100644 src/python/pants/backend/k8s/kubectl_subsystem.py create mode 100644 src/python/pants/backend/k8s/register.py create mode 100644 src/python/pants/backend/k8s/targets.py diff --git a/src/python/pants/backend/k8s/BUILD b/src/python/pants/backend/k8s/BUILD new file mode 100644 index 00000000000..db46e8d6c97 --- /dev/null +++ b/src/python/pants/backend/k8s/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/src/python/pants/backend/k8s/__init__.py b/src/python/pants/backend/k8s/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/k8s/goals/BUILD b/src/python/pants/backend/k8s/goals/BUILD new file mode 100644 index 00000000000..db46e8d6c97 --- /dev/null +++ b/src/python/pants/backend/k8s/goals/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/src/python/pants/backend/k8s/goals/__init__.py b/src/python/pants/backend/k8s/goals/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py new file mode 100644 index 00000000000..81ebca49501 --- /dev/null +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass + +from pants.backend.docker.goals.package_image import DockerPackageFieldSet +from pants.core.goals.deploy import DeployFieldSet, DeployProcess +from pants.engine.addresses import UnparsedAddressInputs +from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest +from pants.engine.fs import MergeDigests, Snapshot +from pants.engine.process import InteractiveProcess +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import DependenciesRequest, HydratedSources, HydrateSourcesRequest, SourcesField, Targets +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from experimental.k8s.k8s_subsystem import K8sSubsystem +from experimental.k8s.kubectl_subsystem import KubectlBinary, KubectlOptions +from experimental.k8s.targets import ( + K8sBundleContextField, + K8sBundleDependenciesField, + K8sBundleSourcesField, + K8sSourceField, +) + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class DeployK8sBundleFieldSet(DeployFieldSet): + required_fields = ( + K8sBundleSourcesField, + K8sBundleContextField, + K8sBundleDependenciesField, + ) + sources: K8sBundleSourcesField + context: K8sBundleContextField + dependencies: K8sBundleDependenciesField + + +@rule(desc="Run k8s deploy process", level=LogLevel.DEBUG) +async def run_k8s_deploy( + field_set: DeployK8sBundleFieldSet, + kubectl: KubectlBinary, + options: KubectlOptions, + k8s_subsystem: K8sSubsystem, +) -> DeployProcess: + context = field_set.context.value + assert context is not None + context = context if options.pass_context else None + if context is not None and context not in options.available_contexts: + raise ValueError(f"context {context} is not listed in `[kubectl].available_contexts`") + + dependencies = await Get(Targets, UnparsedAddressInputs, field_set.sources.to_unparsed_address_inputs()) + file_sources = await MultiGet( + Get( + HydratedSources, + HydrateSourcesRequest( + t.get(SourcesField), + for_sources_types=(K8sSourceField,), + enable_codegen=True, + ), + ) + for t in dependencies + ) + snapshot, target_dependencies = await MultiGet( + Get( + Snapshot, + MergeDigests((*(sources.snapshot.digest for sources in file_sources),)), + ), + Get(Targets, DependenciesRequest(field_set.dependencies)), + ) + + if k8s_subsystem.publish_dependencies: + publish_targets = [tgt for tgt in target_dependencies if DockerPackageFieldSet.is_applicable(tgt)] + else: + publish_targets = [] + + # TODO use KubectlOptions.EnvironmentAware + env = await Get( + EnvironmentVars, + EnvironmentVarsRequest, + EnvironmentVarsRequest(requested=["HOME", "KUBECONFIG", "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT"]), + ) + + process = InteractiveProcess.from_process( + kubectl.apply_configs( + snapshot.files, + input_digest=snapshot.digest, + env=env, + context=context, + ) + ) + + description = f"context {context}" if context is not None else None + + return DeployProcess( + name=field_set.address.spec, + publish_dependencies=tuple(publish_targets), + process=process, + description=description, + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(DeployFieldSet, DeployK8sBundleFieldSet), + ] diff --git a/src/python/pants/backend/k8s/k8s_subsystem.py b/src/python/pants/backend/k8s/k8s_subsystem.py new file mode 100644 index 00000000000..649b0067b2d --- /dev/null +++ b/src/python/pants/backend/k8s/k8s_subsystem.py @@ -0,0 +1,23 @@ +import logging + +from pants.engine.rules import collect_rules +from pants.option.option_types import BoolOption +from pants.option.subsystem import Subsystem + +logger = logging.getLogger(__name__) + + +class K8sSubsystem(Subsystem): + name = "k8s" + options_scope = "k8s" + help = "Kubernetes options" + + # TODO: use https://github.com/pantsbuild/pants/pull/20358 + publish_dependencies = BoolOption( + default=True, + help="Deploy dependencies in `experimental-deploy` goal.", + ) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/k8s/kubectl_subsystem.py b/src/python/pants/backend/k8s/kubectl_subsystem.py new file mode 100644 index 00000000000..e622c01152f --- /dev/null +++ b/src/python/pants/backend/k8s/kubectl_subsystem.py @@ -0,0 +1,110 @@ +import logging +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Optional + +from pants.core.util_rules.search_paths import ExecutableSearchPathsOptionMixin +from pants.core.util_rules.system_binaries import BinaryPath, BinaryPathRequest, BinaryPaths +from pants.engine.fs import Digest +from pants.engine.process import Process, ProcessCacheScope +from pants.engine.rules import Get, collect_rules, rule +from pants.option.option_types import BoolOption, ShellStrListOption, StrListOption +from pants.option.subsystem import Subsystem +from pants.util.logging import LogLevel +from pants.util.strutil import softwrap + +logger = logging.getLogger(__name__) + + +class KubectlOptions(Subsystem): + name = "kubectl" + options_scope = "kubectl" + help = "Kubernetes command line tool" + + available_contexts = StrListOption( + default=[], + help=softwrap( + """ + List of available contexts for `kubectl` command. + """ + ), + ) + pass_context = BoolOption( + default=True, + help=softwrap( + """ + Pass `--context` argument to `kubectl` command. + """ + ), + ) + + class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware): + _env_vars = ShellStrListOption( + help=softwrap( + """ + Environment variables to set for `kubectl` invocations. + + Entries are either strings in the form `ENV_VAR=value` to set an explicit value; + or just `ENV_VAR` to copy the value from Pants's own environment. + """ + ), + advanced=True, + ) + executable_search_paths_help = softwrap( + """ + The PATH value that will be used to find the kubectl binary. + """ + ) + + @property + def env_vars(self) -> tuple[str, ...]: + return tuple(sorted(set(self._env_vars))) + + +@dataclass(frozen=True) +class KubectlBinary(BinaryPath): + """The `kubectl` binary.""" + + def apply_configs( + self, + paths: Sequence[str], + input_digest: Digest, + env: Optional[Mapping[str, str]] = None, + context: Optional[str] = None, + ) -> Process: + argv = (self.path,) + + if context is not None: + argv += ("--context", context) + + argv += ("apply", "-o", "yaml") + + for path in paths: + argv += ("-f", path) + + return Process( + argv=argv, + input_digest=input_digest, + cache_scope=ProcessCacheScope.PER_SESSION, + description=f"Applying kubernetes config {paths}", + env=env, + ) + + +@rule(desc="Finding the `kubectl` binary", level=LogLevel.DEBUG) +async def get_kubectl(kubectl_options_env_aware: KubectlOptions.EnvironmentAware) -> KubectlBinary: + search_path = kubectl_options_env_aware.executable_search_path + request = BinaryPathRequest( + binary_name="kubectl", + search_path=search_path, + # TODO test=BinaryPathTest(args=["version", "--output=json"]), + ) + paths = await Get(BinaryPaths, BinaryPathRequest, request) + logger.debug("kubectl path %s", paths.first_path) + first_path = paths.first_path_or_raise(request, rationale="interact with the kubernetes cluster") + + return KubectlBinary(first_path.path, first_path.fingerprint) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/k8s/register.py b/src/python/pants/backend/k8s/register.py new file mode 100644 index 00000000000..05679c21257 --- /dev/null +++ b/src/python/pants/backend/k8s/register.py @@ -0,0 +1,14 @@ +from experimental.k8s import k8s_subsystem, kubectl_subsystem, targets +from experimental.k8s.goals import deploy + + +def rules(): + return [ + *kubectl_subsystem.rules(), + *k8s_subsystem.rules(), + *deploy.rules(), + ] + + +def target_types(): + return [*targets.target_types()] diff --git a/src/python/pants/backend/k8s/targets.py b/src/python/pants/backend/k8s/targets.py new file mode 100644 index 00000000000..98ebbf251b0 --- /dev/null +++ b/src/python/pants/backend/k8s/targets.py @@ -0,0 +1,89 @@ +from pants.engine.target import ( + COMMON_TARGET_FIELDS, + Dependencies, + MultipleSourcesField, + SingleSourceField, + SpecialCasedDependencies, + StringField, + Target, + TargetFilesGenerator, + generate_multiple_sources_field_help_message, +) +from pants.util.strutil import help_text + + +class K8sSourceField(SingleSourceField): + expected_file_extensions: tuple[str, ...] = (".yml", ".yaml") + + +class K8sSourcesField(MultipleSourcesField): + default = ("*.yaml", "*.yml") + expected_file_extensions: tuple[str, ...] = (".yml", ".yaml") + help = generate_multiple_sources_field_help_message( + "Example: `sources=['example.yaml', 'new_*.yaml', '!old_ignore.yaml']`" + ) + + +class K8sSourceDependenciesField(Dependencies): + pass + + +class K8sSourceTarget(Target): + alias = "k8s_source" + core_fields = ( + # Provides `tags` + *COMMON_TARGET_FIELDS, + K8sSourceField, + K8sSourceDependenciesField, + ) + help = "A single k8s object spec file." + + +class K8sSourceTargetGenerator(TargetFilesGenerator): + alias = "k8s_sources" + generated_target_cls = K8sSourceTarget + + core_fields = ( + # Provides `tags` + *COMMON_TARGET_FIELDS, + K8sSourcesField, + ) + copied_fields = COMMON_TARGET_FIELDS + moved_fields = (K8sSourceDependenciesField,) + help = help_text( + f""" + Generate a `{K8sSourceTarget.alias}` target for each file in the `{K8sSourcesField.alias}` field. + """ + ) + + +class K8sBundleSourcesField(SpecialCasedDependencies): + alias = "sources" + + +class K8sBundleContextField(StringField): + alias = "context" + required = True + help = "The kubectl context to use for publishing." + + +class K8sBundleDependenciesField(Dependencies): + alias = "dependencies" + + +class K8sBundleTarget(Target): + alias = "k8s_bundle" + core_fields = ( + *COMMON_TARGET_FIELDS, + K8sBundleSourcesField, + K8sBundleContextField, + K8sBundleDependenciesField, + ) + + +def target_types(): + return [ + K8sSourceTarget, + K8sSourceTargetGenerator, + K8sBundleTarget, + ] From c13360d6d8da9e75367cfacc461a54d8fe62c5b7 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 21 Dec 2024 14:28:09 +0100 Subject: [PATCH 02/25] Fix linters --- .../pants/backend/experimental/k8s/BUILD | 3 + .../backend/experimental/k8s/__init__.py | 0 .../backend/experimental/k8s/register.py | 16 ++++++ src/python/pants/backend/k8s/BUILD | 2 + src/python/pants/backend/k8s/goals/BUILD | 2 + src/python/pants/backend/k8s/goals/deploy.py | 55 ++++++++++-------- src/python/pants/backend/k8s/k8s_subsystem.py | 23 ++++++-- .../pants/backend/k8s/kubectl_subsystem.py | 56 ++++++++----------- src/python/pants/backend/k8s/register.py | 14 ----- src/python/pants/backend/k8s/targets.py | 8 ++- 10 files changed, 102 insertions(+), 77 deletions(-) create mode 100644 src/python/pants/backend/experimental/k8s/BUILD create mode 100644 src/python/pants/backend/experimental/k8s/__init__.py create mode 100644 src/python/pants/backend/experimental/k8s/register.py delete mode 100644 src/python/pants/backend/k8s/register.py diff --git a/src/python/pants/backend/experimental/k8s/BUILD b/src/python/pants/backend/experimental/k8s/BUILD new file mode 100644 index 00000000000..aa06283b0f4 --- /dev/null +++ b/src/python/pants/backend/experimental/k8s/BUILD @@ -0,0 +1,3 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +python_sources() diff --git a/src/python/pants/backend/experimental/k8s/__init__.py b/src/python/pants/backend/experimental/k8s/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/experimental/k8s/register.py b/src/python/pants/backend/experimental/k8s/register.py new file mode 100644 index 00000000000..05205ef9d71 --- /dev/null +++ b/src/python/pants/backend/experimental/k8s/register.py @@ -0,0 +1,16 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from pants.backend.k8s import k8s_subsystem, kubectl_subsystem, targets +from pants.backend.k8s.goals import deploy + + +def rules(): + return [ + *kubectl_subsystem.rules(), + *k8s_subsystem.rules(), + *deploy.rules(), + ] + + +def target_types(): + return [*targets.target_types()] diff --git a/src/python/pants/backend/k8s/BUILD b/src/python/pants/backend/k8s/BUILD index db46e8d6c97..aa06283b0f4 100644 --- a/src/python/pants/backend/k8s/BUILD +++ b/src/python/pants/backend/k8s/BUILD @@ -1 +1,3 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). python_sources() diff --git a/src/python/pants/backend/k8s/goals/BUILD b/src/python/pants/backend/k8s/goals/BUILD index db46e8d6c97..aa06283b0f4 100644 --- a/src/python/pants/backend/k8s/goals/BUILD +++ b/src/python/pants/backend/k8s/goals/BUILD @@ -1 +1,3 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). python_sources() diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index 81ebca49501..8320633e884 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -1,28 +1,35 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations import logging from dataclasses import dataclass from pants.backend.docker.goals.package_image import DockerPackageFieldSet +from pants.backend.k8s.k8s_subsystem import K8sSubsystem +from pants.backend.k8s.kubectl_subsystem import KubectlBinary, KubectlOptions +from pants.backend.k8s.targets import ( + K8sBundleContextField, + K8sBundleDependenciesField, + K8sBundleSourcesField, + K8sSourceField, +) from pants.core.goals.deploy import DeployFieldSet, DeployProcess from pants.engine.addresses import UnparsedAddressInputs from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest from pants.engine.fs import MergeDigests, Snapshot from pants.engine.process import InteractiveProcess from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import DependenciesRequest, HydratedSources, HydrateSourcesRequest, SourcesField, Targets +from pants.engine.target import ( + DependenciesRequest, + HydratedSources, + HydrateSourcesRequest, + SourcesField, + Targets, +) from pants.engine.unions import UnionRule from pants.util.logging import LogLevel -from experimental.k8s.k8s_subsystem import K8sSubsystem -from experimental.k8s.kubectl_subsystem import KubectlBinary, KubectlOptions -from experimental.k8s.targets import ( - K8sBundleContextField, - K8sBundleDependenciesField, - K8sBundleSourcesField, - K8sSourceField, -) - logger = logging.getLogger(__name__) @@ -42,16 +49,22 @@ class DeployK8sBundleFieldSet(DeployFieldSet): async def run_k8s_deploy( field_set: DeployK8sBundleFieldSet, kubectl: KubectlBinary, - options: KubectlOptions, + kubectl_subsystem: KubectlOptions, k8s_subsystem: K8sSubsystem, ) -> DeployProcess: context = field_set.context.value - assert context is not None - context = context if options.pass_context else None - if context is not None and context not in options.available_contexts: - raise ValueError(f"context {context} is not listed in `[kubectl].available_contexts`") + if context is None: + raise ValueError(f"Missing `{K8sBundleContextField.alias}` field") - dependencies = await Get(Targets, UnparsedAddressInputs, field_set.sources.to_unparsed_address_inputs()) + context = context if kubectl_subsystem.pass_context else None + if context is not None and context not in k8s_subsystem.available_contexts: + raise ValueError( + f"Context `{context}` is not listed in `[{K8sSubsystem.options_scope}].available_contexts`" + ) + + dependencies = await Get( + Targets, UnparsedAddressInputs, field_set.sources.to_unparsed_address_inputs() + ) file_sources = await MultiGet( Get( HydratedSources, @@ -71,16 +84,14 @@ async def run_k8s_deploy( Get(Targets, DependenciesRequest(field_set.dependencies)), ) - if k8s_subsystem.publish_dependencies: - publish_targets = [tgt for tgt in target_dependencies if DockerPackageFieldSet.is_applicable(tgt)] - else: - publish_targets = [] + publish_targets = [ + tgt for tgt in target_dependencies if DockerPackageFieldSet.is_applicable(tgt) + ] - # TODO use KubectlOptions.EnvironmentAware env = await Get( EnvironmentVars, EnvironmentVarsRequest, - EnvironmentVarsRequest(requested=["HOME", "KUBECONFIG", "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT"]), + EnvironmentVarsRequest(requested=kubectl_subsystem.extra_env_vars), ) process = InteractiveProcess.from_process( diff --git a/src/python/pants/backend/k8s/k8s_subsystem.py b/src/python/pants/backend/k8s/k8s_subsystem.py index 649b0067b2d..1a21c43d616 100644 --- a/src/python/pants/backend/k8s/k8s_subsystem.py +++ b/src/python/pants/backend/k8s/k8s_subsystem.py @@ -1,8 +1,11 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). import logging from pants.engine.rules import collect_rules -from pants.option.option_types import BoolOption +from pants.option.option_types import StrListOption from pants.option.subsystem import Subsystem +from pants.util.strutil import softwrap logger = logging.getLogger(__name__) @@ -12,10 +15,20 @@ class K8sSubsystem(Subsystem): options_scope = "k8s" help = "Kubernetes options" - # TODO: use https://github.com/pantsbuild/pants/pull/20358 - publish_dependencies = BoolOption( - default=True, - help="Deploy dependencies in `experimental-deploy` goal.", + available_contexts = StrListOption( + default=[], + help=softwrap( + """ + List of available contexts for `kubectl` command. `k8s_bundle` + context will be validated against this list. + + You have to explicitly provide the list, because it will be shared + with people using the pants repo. We can't parse the KUBE_CONFIG + env var because different people might have different private + clusters, e.g. minikube or kind, which means pants validation will + give different results. + """ + ), ) diff --git a/src/python/pants/backend/k8s/kubectl_subsystem.py b/src/python/pants/backend/k8s/kubectl_subsystem.py index e622c01152f..2513fc0d69d 100644 --- a/src/python/pants/backend/k8s/kubectl_subsystem.py +++ b/src/python/pants/backend/k8s/kubectl_subsystem.py @@ -1,14 +1,20 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). import logging from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Optional -from pants.core.util_rules.search_paths import ExecutableSearchPathsOptionMixin -from pants.core.util_rules.system_binaries import BinaryPath, BinaryPathRequest, BinaryPaths +from pants.core.util_rules.system_binaries import ( + BinaryPath, + BinaryPathRequest, + BinaryPaths, + BinaryPathTest, +) from pants.engine.fs import Digest from pants.engine.process import Process, ProcessCacheScope from pants.engine.rules import Get, collect_rules, rule -from pants.option.option_types import BoolOption, ShellStrListOption, StrListOption +from pants.option.option_types import BoolOption, StrListOption from pants.option.subsystem import Subsystem from pants.util.logging import LogLevel from pants.util.strutil import softwrap @@ -21,14 +27,6 @@ class KubectlOptions(Subsystem): options_scope = "kubectl" help = "Kubernetes command line tool" - available_contexts = StrListOption( - default=[], - help=softwrap( - """ - List of available contexts for `kubectl` command. - """ - ), - ) pass_context = BoolOption( default=True, help=softwrap( @@ -37,28 +35,16 @@ class KubectlOptions(Subsystem): """ ), ) - - class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware): - _env_vars = ShellStrListOption( - help=softwrap( - """ - Environment variables to set for `kubectl` invocations. - - Entries are either strings in the form `ENV_VAR=value` to set an explicit value; - or just `ENV_VAR` to copy the value from Pants's own environment. - """ - ), - advanced=True, - ) - executable_search_paths_help = softwrap( + extra_env_vars = StrListOption( + help=softwrap( """ - The PATH value that will be used to find the kubectl binary. + Additional environment variables that would be made available to all Helm processes + or during value interpolation. """ - ) - - @property - def env_vars(self) -> tuple[str, ...]: - return tuple(sorted(set(self._env_vars))) + ), + default=["HOME", "KUBECONFIG", "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT"], + advanced=True, + ) @dataclass(frozen=True) @@ -72,7 +58,7 @@ def apply_configs( env: Optional[Mapping[str, str]] = None, context: Optional[str] = None, ) -> Process: - argv = (self.path,) + argv: tuple[str, ...] = (self.path,) if context is not None: argv += ("--context", context) @@ -97,11 +83,13 @@ async def get_kubectl(kubectl_options_env_aware: KubectlOptions.EnvironmentAware request = BinaryPathRequest( binary_name="kubectl", search_path=search_path, - # TODO test=BinaryPathTest(args=["version", "--output=json"]), + test=BinaryPathTest(args=["version", "--output=json"]), ) paths = await Get(BinaryPaths, BinaryPathRequest, request) logger.debug("kubectl path %s", paths.first_path) - first_path = paths.first_path_or_raise(request, rationale="interact with the kubernetes cluster") + first_path = paths.first_path_or_raise( + request, rationale="interact with the kubernetes cluster" + ) return KubectlBinary(first_path.path, first_path.fingerprint) diff --git a/src/python/pants/backend/k8s/register.py b/src/python/pants/backend/k8s/register.py deleted file mode 100644 index 05679c21257..00000000000 --- a/src/python/pants/backend/k8s/register.py +++ /dev/null @@ -1,14 +0,0 @@ -from experimental.k8s import k8s_subsystem, kubectl_subsystem, targets -from experimental.k8s.goals import deploy - - -def rules(): - return [ - *kubectl_subsystem.rules(), - *k8s_subsystem.rules(), - *deploy.rules(), - ] - - -def target_types(): - return [*targets.target_types()] diff --git a/src/python/pants/backend/k8s/targets.py b/src/python/pants/backend/k8s/targets.py index 98ebbf251b0..6d3a602d167 100644 --- a/src/python/pants/backend/k8s/targets.py +++ b/src/python/pants/backend/k8s/targets.py @@ -1,3 +1,7 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from typing import ClassVar + from pants.engine.target import ( COMMON_TARGET_FIELDS, Dependencies, @@ -13,12 +17,12 @@ class K8sSourceField(SingleSourceField): - expected_file_extensions: tuple[str, ...] = (".yml", ".yaml") + expected_file_extensions: ClassVar[tuple[str, ...]] = (".yml", ".yaml") class K8sSourcesField(MultipleSourcesField): default = ("*.yaml", "*.yml") - expected_file_extensions: tuple[str, ...] = (".yml", ".yaml") + expected_file_extensions: ClassVar[tuple[str, ...]] = (".yml", ".yaml") help = generate_multiple_sources_field_help_message( "Example: `sources=['example.yaml', 'new_*.yaml', '!old_ignore.yaml']`" ) From e9cbcbf47d2c782fcb2ee579c4955a912c832cfd Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 21 Dec 2024 15:19:41 +0100 Subject: [PATCH 03/25] Fix env var --- src/python/pants/backend/k8s/k8s_subsystem.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/python/pants/backend/k8s/k8s_subsystem.py b/src/python/pants/backend/k8s/k8s_subsystem.py index 1a21c43d616..5de5f575121 100644 --- a/src/python/pants/backend/k8s/k8s_subsystem.py +++ b/src/python/pants/backend/k8s/k8s_subsystem.py @@ -23,10 +23,10 @@ class K8sSubsystem(Subsystem): context will be validated against this list. You have to explicitly provide the list, because it will be shared - with people using the pants repo. We can't parse the KUBE_CONFIG - env var because different people might have different private - clusters, e.g. minikube or kind, which means pants validation will - give different results. + with people using the pants repo. We can't parse the KUBECONFIG env + var because different people might have different private clusters, + e.g. minikube or kind, which means pants validation will give + different results. """ ), ) From 420d6b90bfc64231307f4ddba7f77b0cf8224147 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 21 Dec 2024 23:04:35 +0100 Subject: [PATCH 04/25] Add a test --- .../backend/experimental/k8s/register.py | 6 +- .../pants/backend/k8s/goals/deploy_test.py | 228 ++++++++++++++++++ .../pants/backend/k8s/kubectl_subsystem.py | 69 +----- src/python/pants/backend/k8s/kubectl_tool.py | 71 ++++++ .../k8s/{targets.py => target_types.py} | 2 +- 5 files changed, 313 insertions(+), 63 deletions(-) create mode 100644 src/python/pants/backend/k8s/goals/deploy_test.py create mode 100644 src/python/pants/backend/k8s/kubectl_tool.py rename src/python/pants/backend/k8s/{targets.py => target_types.py} (97%) diff --git a/src/python/pants/backend/experimental/k8s/register.py b/src/python/pants/backend/experimental/k8s/register.py index 05205ef9d71..96c9e3bd007 100644 --- a/src/python/pants/backend/experimental/k8s/register.py +++ b/src/python/pants/backend/experimental/k8s/register.py @@ -1,6 +1,7 @@ # Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.k8s import k8s_subsystem, kubectl_subsystem, targets +from pants.backend.k8s import k8s_subsystem, k8s_tool, kubectl_subsystem +from pants.backend.k8s import target_types as k8s_target_types from pants.backend.k8s.goals import deploy @@ -8,9 +9,10 @@ def rules(): return [ *kubectl_subsystem.rules(), *k8s_subsystem.rules(), + *k8s_tool.rules(), *deploy.rules(), ] def target_types(): - return [*targets.target_types()] + return k8s_target_types.target_types() diff --git a/src/python/pants/backend/k8s/goals/deploy_test.py b/src/python/pants/backend/k8s/goals/deploy_test.py new file mode 100644 index 00000000000..723a918f699 --- /dev/null +++ b/src/python/pants/backend/k8s/goals/deploy_test.py @@ -0,0 +1,228 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from textwrap import dedent +from typing import Iterable, Mapping + +import pytest + +from pants.backend.docker.target_types import DockerImageTarget +from pants.backend.k8s.goals.deploy import DeployK8sBundle, DeployK8sBundleFieldSet +from pants.backend.k8s.goals.deploy import rules as k8s_deploy_rules +from pants.backend.k8s.target_types import ( + K8sBundleTarget, +) +from pants.core.goals import package +from pants.core.goals.deploy import DeployProcess +from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + target_types=[HelmArtifactTarget, HelmChartTarget, HelmDeploymentTarget, DockerImageTarget], + rules=[ + *k8s_deploy_rules(), + *package.rules(), + QueryRule(KubectlBinary, ()), + QueryRule(DeployProcess, (DeployK8sBundleFieldSet,)), + ], + ) + return rule_runner + + +def _get_process( + rule_runner: RuleRunner, + spec_path: str, + target_name: str, + *, + args: Iterable[str] | None = None, + env: Mapping[str, str] | None = None, +) -> DeployProcess: + rule_runner.set_options(args or (), env=env) + + target = rule_runner.get_target(Address(spec_path, target_name=target_name)) + field_set = DeployK8sBundleFieldSet.create(target) + + return rule_runner.request(DeployProcess, [field_set]) + + +def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/k8s/BUILD": dedent("""\ + k8s_sources() + k8s_bundle(name="pod", dependencies=["pod.yaml"]) + """), + "src/k8s/pod.yaml": dedent("""\ + apiVersion: v1 + kind: Pod + """), + } + ) + + deploy_process = _get_process( + rule_runner, + "src/k8s", + "pod", + ) + + helm = rule_runner.request(HelmBinary, []) + + assert deploy_process.process + assert deploy_process.process.process.argv == ( + helm.path, + "upgrade", + "foo", + "mychart", + "--description", + '"Foo deployment"', + "--namespace", + f"uat-{expected_ns_suffix}", + "--skip-crds", + "--no-hooks", + "--post-renderer", + "./post_renderer_wrapper.sh", + "--values", + ",".join( + [f"__values/src/deployment/{filename}" for filename in expected_value_files_order] + ), + "--set", + "key=foo", + "--set", + "amount=300", + "--set", + 'long_string="This is a long string"', + "--set", + f"build_number={expected_build_number}", + "--install", + "--timeout", + "150s", + "--kubeconfig", + "./kubeconfig", + "--create-namespace", + ) + + +@pytest.mark.parametrize("invalid_passthrough_args", [["--namespace", "foo"], ["--dry-run"]]) +def test_raises_error_when_using_invalid_passthrough_args( + rule_runner: RuleRunner, invalid_passthrough_args: list[str] +) -> None: + rule_runner.write_files( + { + "src/chart/BUILD": """helm_chart(registries=["oci://www.example.com/external"])""", + "src/chart/Chart.yaml": HELM_CHART_FILE, + "src/deployment/BUILD": dedent( + """\ + helm_deployment( + name="bar", + namespace="uat", + chart="//src/chart", + sources=["*.yaml", "subdir/*.yml"] + ) + """ + ), + } + ) + + source_root_patterns = ["/src/*"] + deploy_args = ["--force", "--debug", "--kubeconfig=./kubeconfig", *invalid_passthrough_args] + + invalid_passthrough_args_as_string = " ".join(invalid_passthrough_args) + with pytest.raises( + ExecutionError, + match=f"The following command line arguments are not valid: {invalid_passthrough_args_as_string}.", + ): + _get_process( + rule_runner, + "src/deployment", + "bar", + args=[ + f"--source-root-patterns={repr(source_root_patterns)}", + f"--helm-args={repr(deploy_args)}", + ], + ) + + +def test_can_deploy_3rd_party_chart(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "3rdparty/helm/BUILD": dedent( + """\ + helm_artifact( + name="prometheus-stack", + repository="https://prometheus-community.github.io/helm-charts", + artifact="kube-prometheus-stack", + version="^27.2.0" + ) + """ + ), + "BUILD": dedent( + """\ + helm_deployment( + name="deploy_3rd_party", + chart="//3rdparty/helm:prometheus-stack", + ) + """ + ), + } + ) + deploy_process = _get_process( + rule_runner, + "", + "deploy_3rd_party", + args=[ + "--helm-infer-external-docker-images=['quay.io/kiwigrid/k8s-sidecar','grafana/grafana','bats/bats','k8s.gcr.io/kube-state-metrics/kube-state-metrics','quay.io/prometheus/node-exporter','quay.io/prometheus-operator/prometheus-operator']" + ], + ) + + assert deploy_process.process + assert len(deploy_process.publish_dependencies) == 0 + + +@pytest.mark.parametrize( + "dry_run_args,expected", + [ + ([], False), + (["--experimental-deploy-dry-run=False"], False), + (["--experimental-deploy-dry-run"], True), + ], +) +def test_run_helm_deploy_adheres_to_dry_run_flag( + rule_runner: RuleRunner, dry_run_args: list[str], expected: bool +) -> None: + rule_runner.write_files( + { + "src/chart/BUILD": """helm_chart(registries=["oci://www.example.com/external"])""", + "src/chart/Chart.yaml": HELM_CHART_FILE, + "src/deployment/BUILD": dedent( + """\ + helm_deployment( + name="bar", + namespace="uat", + chart="//src/chart", + sources=["*.yaml", "subdir/*.yml"] + ) + """ + ), + } + ) + + expected_build_number = "34" + expected_ns_suffix = "quxx" + + deploy_args = ["--kubeconfig", "./kubeconfig", "--create-namespace"] + deploy_process = _get_process( + rule_runner, + "src/deployment", + "bar", + args=[f"--helm-args={repr(deploy_args)}", *dry_run_args], + env={"BUILD_NUMBER": expected_build_number, "NS_SUFFIX": expected_ns_suffix}, + ) + + assert deploy_process.process + assert ("--dry-run" in deploy_process.process.process.argv) == expected diff --git a/src/python/pants/backend/k8s/kubectl_subsystem.py b/src/python/pants/backend/k8s/kubectl_subsystem.py index 2513fc0d69d..305bd690586 100644 --- a/src/python/pants/backend/k8s/kubectl_subsystem.py +++ b/src/python/pants/backend/k8s/kubectl_subsystem.py @@ -1,22 +1,11 @@ # Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import Optional -from pants.core.util_rules.system_binaries import ( - BinaryPath, - BinaryPathRequest, - BinaryPaths, - BinaryPathTest, -) -from pants.engine.fs import Digest -from pants.engine.process import Process, ProcessCacheScope -from pants.engine.rules import Get, collect_rules, rule +from pants.core.util_rules.search_paths import ExecutableSearchPathsOptionMixin +from pants.engine.rules import collect_rules from pants.option.option_types import BoolOption, StrListOption from pants.option.subsystem import Subsystem -from pants.util.logging import LogLevel from pants.util.strutil import softwrap logger = logging.getLogger(__name__) @@ -27,6 +16,13 @@ class KubectlOptions(Subsystem): options_scope = "kubectl" help = "Kubernetes command line tool" + class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware): + executable_search_paths_help = softwrap( + """ + The PATH value that will be used to find kubectl binary. + """ + ) + pass_context = BoolOption( default=True, help=softwrap( @@ -47,52 +43,5 @@ class KubectlOptions(Subsystem): ) -@dataclass(frozen=True) -class KubectlBinary(BinaryPath): - """The `kubectl` binary.""" - - def apply_configs( - self, - paths: Sequence[str], - input_digest: Digest, - env: Optional[Mapping[str, str]] = None, - context: Optional[str] = None, - ) -> Process: - argv: tuple[str, ...] = (self.path,) - - if context is not None: - argv += ("--context", context) - - argv += ("apply", "-o", "yaml") - - for path in paths: - argv += ("-f", path) - - return Process( - argv=argv, - input_digest=input_digest, - cache_scope=ProcessCacheScope.PER_SESSION, - description=f"Applying kubernetes config {paths}", - env=env, - ) - - -@rule(desc="Finding the `kubectl` binary", level=LogLevel.DEBUG) -async def get_kubectl(kubectl_options_env_aware: KubectlOptions.EnvironmentAware) -> KubectlBinary: - search_path = kubectl_options_env_aware.executable_search_path - request = BinaryPathRequest( - binary_name="kubectl", - search_path=search_path, - test=BinaryPathTest(args=["version", "--output=json"]), - ) - paths = await Get(BinaryPaths, BinaryPathRequest, request) - logger.debug("kubectl path %s", paths.first_path) - first_path = paths.first_path_or_raise( - request, rationale="interact with the kubernetes cluster" - ) - - return KubectlBinary(first_path.path, first_path.fingerprint) - - def rules(): return collect_rules() diff --git a/src/python/pants/backend/k8s/kubectl_tool.py b/src/python/pants/backend/k8s/kubectl_tool.py new file mode 100644 index 00000000000..bdaa7164df3 --- /dev/null +++ b/src/python/pants/backend/k8s/kubectl_tool.py @@ -0,0 +1,71 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +import logging +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Optional + +from pants.backend.k8s.kubectl_subsystem import KubectlOptions +from pants.core.util_rules.system_binaries import ( + BinaryPath, + BinaryPathRequest, + BinaryPaths, + BinaryPathTest, +) +from pants.engine.fs import Digest +from pants.engine.process import Process, ProcessCacheScope +from pants.engine.rules import Get, collect_rules, rule +from pants.util.logging import LogLevel + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class KubectlBinary(BinaryPath): + """The `kubectl` binary.""" + + def apply_configs( + self, + paths: Sequence[str], + input_digest: Digest, + env: Optional[Mapping[str, str]] = None, + context: Optional[str] = None, + ) -> Process: + argv: tuple[str, ...] = (self.path,) + + if context is not None: + argv += ("--context", context) + + argv += ("apply", "-o", "yaml") + + for path in paths: + argv += ("-f", path) + + return Process( + argv=argv, + input_digest=input_digest, + cache_scope=ProcessCacheScope.PER_SESSION, + description=f"Applying kubernetes config {paths}", + env=env, + ) + + +@rule(desc="Finding the `kubectl` binary", level=LogLevel.DEBUG) +async def get_kubectl(kubectl_options_env_aware: KubectlOptions.EnvironmentAware) -> KubectlBinary: + search_path = kubectl_options_env_aware.executable_search_path + request = BinaryPathRequest( + binary_name="kubectl", + search_path=search_path, + test=BinaryPathTest(args=["version", "--output=json", "--client=true"]), + ) + paths = await Get(BinaryPaths, BinaryPathRequest, request) + logger.debug("kubectl path %s", paths.first_path) + first_path = paths.first_path_or_raise( + request, rationale="interact with the kubernetes cluster" + ) + + return KubectlBinary(first_path.path, first_path.fingerprint) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/k8s/targets.py b/src/python/pants/backend/k8s/target_types.py similarity index 97% rename from src/python/pants/backend/k8s/targets.py rename to src/python/pants/backend/k8s/target_types.py index 6d3a602d167..edcaa5aa3b7 100644 --- a/src/python/pants/backend/k8s/targets.py +++ b/src/python/pants/backend/k8s/target_types.py @@ -68,7 +68,7 @@ class K8sBundleSourcesField(SpecialCasedDependencies): class K8sBundleContextField(StringField): alias = "context" required = True - help = "The kubectl context to use for publishing." + help = "The kubectl context to use for deploy." class K8sBundleDependenciesField(Dependencies): From 940959915cd0301653436cf2339aa36f5fbb0b5b Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 21 Dec 2024 23:35:39 +0100 Subject: [PATCH 05/25] Fixing the test... --- src/python/pants/backend/k8s/goals/BUILD | 1 + src/python/pants/backend/k8s/goals/deploy.py | 5 +- .../pants/backend/k8s/goals/deploy_test.py | 160 +++--------------- 3 files changed, 27 insertions(+), 139 deletions(-) diff --git a/src/python/pants/backend/k8s/goals/BUILD b/src/python/pants/backend/k8s/goals/BUILD index aa06283b0f4..e177aeaaef9 100644 --- a/src/python/pants/backend/k8s/goals/BUILD +++ b/src/python/pants/backend/k8s/goals/BUILD @@ -1,3 +1,4 @@ # Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). python_sources() +python_tests(name="tests") diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index 8320633e884..a31295772cb 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -7,8 +7,9 @@ from pants.backend.docker.goals.package_image import DockerPackageFieldSet from pants.backend.k8s.k8s_subsystem import K8sSubsystem -from pants.backend.k8s.kubectl_subsystem import KubectlBinary, KubectlOptions -from pants.backend.k8s.targets import ( +from pants.backend.k8s.kubectl_subsystem import KubectlOptions +from pants.backend.k8s.kubectl_tool import KubectlBinary +from pants.backend.k8s.target_types import ( K8sBundleContextField, K8sBundleDependenciesField, K8sBundleSourcesField, diff --git a/src/python/pants/backend/k8s/goals/deploy_test.py b/src/python/pants/backend/k8s/goals/deploy_test.py index 723a918f699..6fefde27e60 100644 --- a/src/python/pants/backend/k8s/goals/deploy_test.py +++ b/src/python/pants/backend/k8s/goals/deploy_test.py @@ -8,27 +8,29 @@ import pytest -from pants.backend.docker.target_types import DockerImageTarget -from pants.backend.k8s.goals.deploy import DeployK8sBundle, DeployK8sBundleFieldSet +from pants.backend.k8s import k8s_subsystem, kubectl_tool +from pants.backend.k8s.goals.deploy import DeployK8sBundleFieldSet from pants.backend.k8s.goals.deploy import rules as k8s_deploy_rules -from pants.backend.k8s.target_types import ( - K8sBundleTarget, -) -from pants.core.goals import package +from pants.backend.k8s.kubectl_tool import KubectlBinary +from pants.backend.k8s.target_types import K8sBundleTarget, K8sSourceTargetGenerator from pants.core.goals.deploy import DeployProcess from pants.engine.addresses import Address -from pants.engine.internals.scheduler import ExecutionError -from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner +from pants.engine.environment import EnvironmentName +from pants.testutil.rule_runner import QueryRule, RuleRunner @pytest.fixture def rule_runner() -> RuleRunner: rule_runner = RuleRunner( - target_types=[HelmArtifactTarget, HelmChartTarget, HelmDeploymentTarget, DockerImageTarget], + target_types=[ + K8sSourceTargetGenerator, + K8sBundleTarget, + ], rules=[ *k8s_deploy_rules(), - *package.rules(), - QueryRule(KubectlBinary, ()), + *k8s_subsystem.rules(), + *kubectl_tool.rules(), + QueryRule(KubectlBinary, (EnvironmentName,)), QueryRule(DeployProcess, (DeployK8sBundleFieldSet,)), ], ) @@ -54,14 +56,18 @@ def _get_process( def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: rule_runner.write_files( { - "src/k8s/BUILD": dedent("""\ + "src/k8s/BUILD": dedent( + """\ k8s_sources() - k8s_bundle(name="pod", dependencies=["pod.yaml"]) - """), - "src/k8s/pod.yaml": dedent("""\ + k8s_bundle(name="pod", dependencies=["pod.yaml"], context="local") + """ + ), + "src/k8s/pod.yaml": dedent( + """\ apiVersion: v1 kind: Pod - """), + """ + ), } ) @@ -71,7 +77,7 @@ def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: "pod", ) - helm = rule_runner.request(HelmBinary, []) + kubectl = rule_runner.request(KubectlBinary, []) assert deploy_process.process assert deploy_process.process.process.argv == ( @@ -106,123 +112,3 @@ def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: "./kubeconfig", "--create-namespace", ) - - -@pytest.mark.parametrize("invalid_passthrough_args", [["--namespace", "foo"], ["--dry-run"]]) -def test_raises_error_when_using_invalid_passthrough_args( - rule_runner: RuleRunner, invalid_passthrough_args: list[str] -) -> None: - rule_runner.write_files( - { - "src/chart/BUILD": """helm_chart(registries=["oci://www.example.com/external"])""", - "src/chart/Chart.yaml": HELM_CHART_FILE, - "src/deployment/BUILD": dedent( - """\ - helm_deployment( - name="bar", - namespace="uat", - chart="//src/chart", - sources=["*.yaml", "subdir/*.yml"] - ) - """ - ), - } - ) - - source_root_patterns = ["/src/*"] - deploy_args = ["--force", "--debug", "--kubeconfig=./kubeconfig", *invalid_passthrough_args] - - invalid_passthrough_args_as_string = " ".join(invalid_passthrough_args) - with pytest.raises( - ExecutionError, - match=f"The following command line arguments are not valid: {invalid_passthrough_args_as_string}.", - ): - _get_process( - rule_runner, - "src/deployment", - "bar", - args=[ - f"--source-root-patterns={repr(source_root_patterns)}", - f"--helm-args={repr(deploy_args)}", - ], - ) - - -def test_can_deploy_3rd_party_chart(rule_runner: RuleRunner) -> None: - rule_runner.write_files( - { - "3rdparty/helm/BUILD": dedent( - """\ - helm_artifact( - name="prometheus-stack", - repository="https://prometheus-community.github.io/helm-charts", - artifact="kube-prometheus-stack", - version="^27.2.0" - ) - """ - ), - "BUILD": dedent( - """\ - helm_deployment( - name="deploy_3rd_party", - chart="//3rdparty/helm:prometheus-stack", - ) - """ - ), - } - ) - deploy_process = _get_process( - rule_runner, - "", - "deploy_3rd_party", - args=[ - "--helm-infer-external-docker-images=['quay.io/kiwigrid/k8s-sidecar','grafana/grafana','bats/bats','k8s.gcr.io/kube-state-metrics/kube-state-metrics','quay.io/prometheus/node-exporter','quay.io/prometheus-operator/prometheus-operator']" - ], - ) - - assert deploy_process.process - assert len(deploy_process.publish_dependencies) == 0 - - -@pytest.mark.parametrize( - "dry_run_args,expected", - [ - ([], False), - (["--experimental-deploy-dry-run=False"], False), - (["--experimental-deploy-dry-run"], True), - ], -) -def test_run_helm_deploy_adheres_to_dry_run_flag( - rule_runner: RuleRunner, dry_run_args: list[str], expected: bool -) -> None: - rule_runner.write_files( - { - "src/chart/BUILD": """helm_chart(registries=["oci://www.example.com/external"])""", - "src/chart/Chart.yaml": HELM_CHART_FILE, - "src/deployment/BUILD": dedent( - """\ - helm_deployment( - name="bar", - namespace="uat", - chart="//src/chart", - sources=["*.yaml", "subdir/*.yml"] - ) - """ - ), - } - ) - - expected_build_number = "34" - expected_ns_suffix = "quxx" - - deploy_args = ["--kubeconfig", "./kubeconfig", "--create-namespace"] - deploy_process = _get_process( - rule_runner, - "src/deployment", - "bar", - args=[f"--helm-args={repr(deploy_args)}", *dry_run_args], - env={"BUILD_NUMBER": expected_build_number, "NS_SUFFIX": expected_ns_suffix}, - ) - - assert deploy_process.process - assert ("--dry-run" in deploy_process.process.process.argv) == expected From 7a0d00d4ef1149ca77ba3d4a4c5797600cf01c25 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sun, 22 Dec 2024 00:01:53 +0100 Subject: [PATCH 06/25] Download kubectl via TemplatedExternalTool --- .../backend/experimental/k8s/register.py | 3 +- src/python/pants/backend/k8s/goals/deploy.py | 13 ++-- .../pants/backend/k8s/goals/deploy_test.py | 46 +++--------- .../pants/backend/k8s/kubectl_subsystem.py | 71 ++++++++++++++++--- src/python/pants/backend/k8s/kubectl_tool.py | 35 ++------- 5 files changed, 85 insertions(+), 83 deletions(-) diff --git a/src/python/pants/backend/experimental/k8s/register.py b/src/python/pants/backend/experimental/k8s/register.py index 96c9e3bd007..2f3f8974869 100644 --- a/src/python/pants/backend/experimental/k8s/register.py +++ b/src/python/pants/backend/experimental/k8s/register.py @@ -1,6 +1,6 @@ # Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.k8s import k8s_subsystem, k8s_tool, kubectl_subsystem +from pants.backend.k8s import k8s_subsystem, kubectl_subsystem from pants.backend.k8s import target_types as k8s_target_types from pants.backend.k8s.goals import deploy @@ -9,7 +9,6 @@ def rules(): return [ *kubectl_subsystem.rules(), *k8s_subsystem.rules(), - *k8s_tool.rules(), *deploy.rules(), ] diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index a31295772cb..54d70c54d1a 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -7,8 +7,7 @@ from pants.backend.docker.goals.package_image import DockerPackageFieldSet from pants.backend.k8s.k8s_subsystem import K8sSubsystem -from pants.backend.k8s.kubectl_subsystem import KubectlOptions -from pants.backend.k8s.kubectl_tool import KubectlBinary +from pants.backend.k8s.kubectl_subsystem import Kubectl from pants.backend.k8s.target_types import ( K8sBundleContextField, K8sBundleDependenciesField, @@ -19,6 +18,7 @@ from pants.engine.addresses import UnparsedAddressInputs from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest from pants.engine.fs import MergeDigests, Snapshot +from pants.engine.platform import Platform from pants.engine.process import InteractiveProcess from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( @@ -49,15 +49,15 @@ class DeployK8sBundleFieldSet(DeployFieldSet): @rule(desc="Run k8s deploy process", level=LogLevel.DEBUG) async def run_k8s_deploy( field_set: DeployK8sBundleFieldSet, - kubectl: KubectlBinary, - kubectl_subsystem: KubectlOptions, + kubectl: Kubectl, k8s_subsystem: K8sSubsystem, + platform: Platform, ) -> DeployProcess: context = field_set.context.value if context is None: raise ValueError(f"Missing `{K8sBundleContextField.alias}` field") - context = context if kubectl_subsystem.pass_context else None + context = context if kubectl.pass_context else None if context is not None and context not in k8s_subsystem.available_contexts: raise ValueError( f"Context `{context}` is not listed in `[{K8sSubsystem.options_scope}].available_contexts`" @@ -92,12 +92,13 @@ async def run_k8s_deploy( env = await Get( EnvironmentVars, EnvironmentVarsRequest, - EnvironmentVarsRequest(requested=kubectl_subsystem.extra_env_vars), + EnvironmentVarsRequest(requested=kubectl.extra_env_vars), ) process = InteractiveProcess.from_process( kubectl.apply_configs( snapshot.files, + platform=platform, input_digest=snapshot.digest, env=env, context=context, diff --git a/src/python/pants/backend/k8s/goals/deploy_test.py b/src/python/pants/backend/k8s/goals/deploy_test.py index 6fefde27e60..c7e03ee76b1 100644 --- a/src/python/pants/backend/k8s/goals/deploy_test.py +++ b/src/python/pants/backend/k8s/goals/deploy_test.py @@ -8,14 +8,14 @@ import pytest -from pants.backend.k8s import k8s_subsystem, kubectl_tool +from pants.backend.k8s import k8s_subsystem, kubectl_subsystem from pants.backend.k8s.goals.deploy import DeployK8sBundleFieldSet from pants.backend.k8s.goals.deploy import rules as k8s_deploy_rules -from pants.backend.k8s.kubectl_tool import KubectlBinary +from pants.backend.k8s.kubectl_subsystem import Kubectl from pants.backend.k8s.target_types import K8sBundleTarget, K8sSourceTargetGenerator from pants.core.goals.deploy import DeployProcess from pants.engine.addresses import Address -from pants.engine.environment import EnvironmentName +from pants.engine.platform import Platform from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -29,8 +29,9 @@ def rule_runner() -> RuleRunner: rules=[ *k8s_deploy_rules(), *k8s_subsystem.rules(), - *kubectl_tool.rules(), - QueryRule(KubectlBinary, (EnvironmentName,)), + *kubectl_subsystem.rules(), + QueryRule(Kubectl, ()), + QueryRule(Platform, ()), QueryRule(DeployProcess, (DeployK8sBundleFieldSet,)), ], ) @@ -77,38 +78,11 @@ def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: "pod", ) - kubectl = rule_runner.request(KubectlBinary, []) + kubectl = rule_runner.request(Kubectl, []) + platform = rule_runner.request(Platform, []) assert deploy_process.process assert deploy_process.process.process.argv == ( - helm.path, - "upgrade", - "foo", - "mychart", - "--description", - '"Foo deployment"', - "--namespace", - f"uat-{expected_ns_suffix}", - "--skip-crds", - "--no-hooks", - "--post-renderer", - "./post_renderer_wrapper.sh", - "--values", - ",".join( - [f"__values/src/deployment/{filename}" for filename in expected_value_files_order] - ), - "--set", - "key=foo", - "--set", - "amount=300", - "--set", - 'long_string="This is a long string"', - "--set", - f"build_number={expected_build_number}", - "--install", - "--timeout", - "150s", - "--kubeconfig", - "./kubeconfig", - "--create-namespace", + kubectl.generate_exe(platform), + "apply", ) diff --git a/src/python/pants/backend/k8s/kubectl_subsystem.py b/src/python/pants/backend/k8s/kubectl_subsystem.py index 305bd690586..e5493f219b8 100644 --- a/src/python/pants/backend/k8s/kubectl_subsystem.py +++ b/src/python/pants/backend/k8s/kubectl_subsystem.py @@ -1,8 +1,14 @@ # Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging +from collections.abc import Mapping, Sequence +from typing import Optional +from pants.core.util_rules.external_tool import TemplatedExternalTool from pants.core.util_rules.search_paths import ExecutableSearchPathsOptionMixin +from pants.engine.internals.native_engine import Digest +from pants.engine.platform import Platform +from pants.engine.process import Process, ProcessCacheScope from pants.engine.rules import collect_rules from pants.option.option_types import BoolOption, StrListOption from pants.option.subsystem import Subsystem @@ -11,17 +17,27 @@ logger = logging.getLogger(__name__) -class KubectlOptions(Subsystem): +class Kubectl(TemplatedExternalTool): name = "kubectl" options_scope = "kubectl" help = "Kubernetes command line tool" - class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware): - executable_search_paths_help = softwrap( - """ - The PATH value that will be used to find kubectl binary. - """ - ) + default_version = "1.32.0" + default_known_versions = [ + "1.32.0|linux_arm64|d7389b9743b0b909c364d11bba94d13302171d751430b58c13dcdf248e924276|7605249", + "1.32.0|linux_x86_64|d7389b9743b0b909c364d11bba94d13302171d751430b58c13dcdf248e924276|7605249", + "1.32.0|macos_arm64|d7389b9743b0b909c364d11bba94d13302171d751430b58c13dcdf248e924276|7605249", + "1.32.0|macos_x86_64|d7389b9743b0b909c364d11bba94d13302171d751430b58c13dcdf248e924276|7605249", + ] + version_constraints = ">=1,<2" + + default_url_template = "https://dl.k8s.io/release/{version}/bin/{platform}/kubectl" + default_url_platform_mapping = { + "linux_arm64": "linux/arm64", + "linux_x86_64": "linux/amd64", + "macos_arm64": "darwin/arm64", + "macos_x86_64": "darwin/amd64", + } pass_context = BoolOption( default=True, @@ -31,6 +47,7 @@ class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAw """ ), ) + extra_env_vars = StrListOption( help=softwrap( """ @@ -38,10 +55,48 @@ class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAw or during value interpolation. """ ), - default=["HOME", "KUBECONFIG", "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT"], + default=[ + "HOME", + "KUBECONFIG", + "KUBERNETES_SERVICE_HOST", + "KUBERNETES_SERVICE_PORT", + ], advanced=True, ) + class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware): + executable_search_paths_help = softwrap( + """ + The PATH value that will be used to find kubectl binary. + """ + ) + + def apply_configs( + self, + paths: Sequence[str], + input_digest: Digest, + platform: Platform, + env: Optional[Mapping[str, str]] = None, + context: Optional[str] = None, + ) -> Process: + argv: tuple[str, ...] = (self.generate_exe(platform),) + + if context is not None: + argv += ("--context", context) + + argv += ("apply", "-o", "yaml") + + for path in paths: + argv += ("-f", path) + + return Process( + argv=argv, + input_digest=input_digest, + cache_scope=ProcessCacheScope.PER_SESSION, + description=f"Applying kubernetes config {paths}", + env=env, + ) + def rules(): return collect_rules() diff --git a/src/python/pants/backend/k8s/kubectl_tool.py b/src/python/pants/backend/k8s/kubectl_tool.py index bdaa7164df3..3a33f0906b4 100644 --- a/src/python/pants/backend/k8s/kubectl_tool.py +++ b/src/python/pants/backend/k8s/kubectl_tool.py @@ -1,19 +1,15 @@ # Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging -from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Optional -from pants.backend.k8s.kubectl_subsystem import KubectlOptions +from pants.backend.k8s.kubectl_subsystem import Kubectl from pants.core.util_rules.system_binaries import ( BinaryPath, BinaryPathRequest, BinaryPaths, BinaryPathTest, ) -from pants.engine.fs import Digest -from pants.engine.process import Process, ProcessCacheScope from pants.engine.rules import Get, collect_rules, rule from pants.util.logging import LogLevel @@ -24,34 +20,11 @@ class KubectlBinary(BinaryPath): """The `kubectl` binary.""" - def apply_configs( - self, - paths: Sequence[str], - input_digest: Digest, - env: Optional[Mapping[str, str]] = None, - context: Optional[str] = None, - ) -> Process: - argv: tuple[str, ...] = (self.path,) - - if context is not None: - argv += ("--context", context) - - argv += ("apply", "-o", "yaml") - - for path in paths: - argv += ("-f", path) - - return Process( - argv=argv, - input_digest=input_digest, - cache_scope=ProcessCacheScope.PER_SESSION, - description=f"Applying kubernetes config {paths}", - env=env, - ) - @rule(desc="Finding the `kubectl` binary", level=LogLevel.DEBUG) -async def get_kubectl(kubectl_options_env_aware: KubectlOptions.EnvironmentAware) -> KubectlBinary: +async def get_kubectl( + kubectl_options_env_aware: Kubectl.EnvironmentAware, +) -> KubectlBinary: search_path = kubectl_options_env_aware.executable_search_path request = BinaryPathRequest( binary_name="kubectl", From 8ff08ab57107708163fc95ad87077705f01be07d Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sun, 22 Dec 2024 00:10:20 +0100 Subject: [PATCH 07/25] Fix the tests --- .../pants/backend/k8s/goals/deploy_test.py | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/python/pants/backend/k8s/goals/deploy_test.py b/src/python/pants/backend/k8s/goals/deploy_test.py index c7e03ee76b1..4b7b30bb587 100644 --- a/src/python/pants/backend/k8s/goals/deploy_test.py +++ b/src/python/pants/backend/k8s/goals/deploy_test.py @@ -15,6 +15,7 @@ from pants.backend.k8s.target_types import K8sBundleTarget, K8sSourceTargetGenerator from pants.core.goals.deploy import DeployProcess from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError from pants.engine.platform import Platform from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -60,7 +61,7 @@ def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: "src/k8s/BUILD": dedent( """\ k8s_sources() - k8s_bundle(name="pod", dependencies=["pod.yaml"], context="local") + k8s_bundle(name="pod", sources=("src/k8s/pod.yaml",), context="local") """ ), "src/k8s/pod.yaml": dedent( @@ -73,9 +74,7 @@ def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: ) deploy_process = _get_process( - rule_runner, - "src/k8s", - "pod", + rule_runner, "src/k8s", "pod", args=("--k8s-available-contexts=['local']",) ) kubectl = rule_runner.request(Kubectl, []) @@ -84,5 +83,40 @@ def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: assert deploy_process.process assert deploy_process.process.process.argv == ( kubectl.generate_exe(platform), + "--context", + "local", "apply", + "-o", + "yaml", + "-f", + "src/k8s/pod.yaml", ) + + +def test_context_validation(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/k8s/BUILD": dedent( + """\ + k8s_sources() + k8s_bundle(name="pod", sources=("src/k8s/pod.yaml",), context="local") + """ + ), + "src/k8s/pod.yaml": dedent( + """\ + apiVersion: v1 + kind: Pod + """ + ), + } + ) + + with pytest.raises( + ExecutionError, + match=r"ValueError: Context `local` is not listed in `\[k8s\].available_contexts`", + ): + _get_process( + rule_runner, + "src/k8s", + "pod", + ) From 086029fd48c9ac186884d0c4634368e1091f544a Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sun, 22 Dec 2024 16:20:34 +0100 Subject: [PATCH 08/25] Add kubernetes section in docs > fd _category_ docs/docs/ --max-depth 2 --exec rg position '{}' --with-filename --line-number | sort -k 3,3 -n docs/docs/introduction/_category_.json:3: "position": 1 docs/docs/getting-started/_category_.json:3: "position": 2 docs/docs/using-pants/_category_.json:3: "position": 3 docs/docs/python/_category_.json:3: "position": 4 docs/docs/go/_category_.json:3: "position": 5 docs/docs/jvm/_category_.json:3: "position": 6 docs/docs/shell/_category_.json:3: "position": 7 docs/docs/docker/_category_.json:3: "position": 8 docs/docs/kubernetes/_category_.json:3: "position": 9 docs/docs/helm/_category_.json:3: "position": 10 docs/docs/terraform/_category_.json:3: "position": 11 docs/docs/sql/_category_.json:3: "position": 12 docs/docs/ad-hoc-tools/_category_.json:3: "position": 13 docs/docs/javascript/_category_.json:3: "position": 13 docs/docs/writing-plugins/_category_.json:3: "position": 14 docs/docs/releases/_category_.json:3: "position": 15 docs/docs/contributions/_category_.json:3: "position": 16 docs/docs/tutorials/_category_.json:3: "position": 17 --- docs/docs/docker/_category_.json | 2 +- docs/docs/go/_category_.json | 2 +- docs/docs/jvm/_category_.json | 2 +- docs/docs/kubernetes/_category_.json | 4 ++++ docs/docs/kubernetes/index.mdx | 17 +++++++++++++++++ docs/docs/python/_category_.json | 2 +- docs/docs/shell/_category_.json | 2 +- docs/docs/using-pants/_category_.json | 2 +- 8 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 docs/docs/kubernetes/_category_.json create mode 100644 docs/docs/kubernetes/index.mdx diff --git a/docs/docs/docker/_category_.json b/docs/docs/docker/_category_.json index c586575d73c..74aa0682e1a 100644 --- a/docs/docs/docker/_category_.json +++ b/docs/docs/docker/_category_.json @@ -1,4 +1,4 @@ { "label": "Docker", - "position": 9 + "position": 8 } diff --git a/docs/docs/go/_category_.json b/docs/docs/go/_category_.json index f38a2baa9c0..3fd3a61689f 100644 --- a/docs/docs/go/_category_.json +++ b/docs/docs/go/_category_.json @@ -1,4 +1,4 @@ { "label": "Go", - "position": 6 + "position": 5 } diff --git a/docs/docs/jvm/_category_.json b/docs/docs/jvm/_category_.json index 25b92a6af16..e83e4e044ce 100644 --- a/docs/docs/jvm/_category_.json +++ b/docs/docs/jvm/_category_.json @@ -1,4 +1,4 @@ { "label": "JVM", - "position": 7 + "position": 6 } diff --git a/docs/docs/kubernetes/_category_.json b/docs/docs/kubernetes/_category_.json new file mode 100644 index 00000000000..42681f2b080 --- /dev/null +++ b/docs/docs/kubernetes/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Kubernetes", + "position": 9 +} diff --git a/docs/docs/kubernetes/index.mdx b/docs/docs/kubernetes/index.mdx new file mode 100644 index 00000000000..d346e712e4b --- /dev/null +++ b/docs/docs/kubernetes/index.mdx @@ -0,0 +1,17 @@ +--- + title: Kubernetes Overview + sidebar_position: 999 +--- + +--- + +:::caution Kubernetes support is in alpha stage +Pants is currently building support for Kubernetes. Simple use cases might be +supported, but many options are missing. + +Please share feedback for what you need to use Pants with your Kubernetes queries by +either [opening a GitHub +issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our +Slack](/community/getting-help)! +::: + diff --git a/docs/docs/python/_category_.json b/docs/docs/python/_category_.json index 135b4c60348..5775bfc99b5 100644 --- a/docs/docs/python/_category_.json +++ b/docs/docs/python/_category_.json @@ -1,4 +1,4 @@ { "label": "Python", - "position": 5 + "position": 4 } diff --git a/docs/docs/shell/_category_.json b/docs/docs/shell/_category_.json index e40d8ae62f3..6e17751f296 100644 --- a/docs/docs/shell/_category_.json +++ b/docs/docs/shell/_category_.json @@ -1,4 +1,4 @@ { "label": "Shell", - "position": 8 + "position": 7 } diff --git a/docs/docs/using-pants/_category_.json b/docs/docs/using-pants/_category_.json index 098e12f2b55..e7f6418bf8d 100644 --- a/docs/docs/using-pants/_category_.json +++ b/docs/docs/using-pants/_category_.json @@ -1,4 +1,4 @@ { "label": "Using Pants", - "position": 4 + "position": 3 } From 596e94748bd544b32aeb249720aebb6cf45b5cf3 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sun, 22 Dec 2024 18:27:09 +0100 Subject: [PATCH 09/25] Implement tailor goal --- docs/docs/kubernetes/index.mdx | 44 ++++++++++++++ .../backend/experimental/k8s/register.py | 7 ++- .../pants/backend/k8s/goals/deploy_test.py | 4 +- src/python/pants/backend/k8s/goals/tailor.py | 57 +++++++++++++++++++ src/python/pants/backend/k8s/k8s_subsystem.py | 12 +++- src/python/pants/backend/k8s/target_types.py | 4 +- 6 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 src/python/pants/backend/k8s/goals/tailor.py diff --git a/docs/docs/kubernetes/index.mdx b/docs/docs/kubernetes/index.mdx index d346e712e4b..984aab189d6 100644 --- a/docs/docs/kubernetes/index.mdx +++ b/docs/docs/kubernetes/index.mdx @@ -15,3 +15,47 @@ issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our Slack](/community/getting-help)! ::: +## Initial setup + +First, activate the relevant backend in `pants.toml`: + +```toml title="pants.toml" +[GLOBAL] +backend_packages = [ + ... + "pants.backend.experimental.k8s", + ... +] +``` + +The Kubernetes backend adds [`k8s_source`](../../reference/targets/k8s_source.mdx) and +[`k8s_sources`](../../reference/targets/k8s_sources.mdx) target types for Kubernetes object +files. The `tailor` goal will automatically generate the targets for +your .yaml files. + +For example, create a file `src/k8s/configmap.yaml`: + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: spark-defaults-conf +data: + spark-defaults.conf: | + spark.driver.memory=1g + spark.executor.cores=1 + spark.executor.instances=1 + spark.executor.memory=2g +``` + +Now run: + +```bash +pants tailor src/k8s: +``` +``` +Created src/k8s/BUILD: + - Add k8s_sources target k8s +``` + diff --git a/src/python/pants/backend/experimental/k8s/register.py b/src/python/pants/backend/experimental/k8s/register.py index 2f3f8974869..1003e5ab398 100644 --- a/src/python/pants/backend/experimental/k8s/register.py +++ b/src/python/pants/backend/experimental/k8s/register.py @@ -2,14 +2,15 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.k8s import k8s_subsystem, kubectl_subsystem from pants.backend.k8s import target_types as k8s_target_types -from pants.backend.k8s.goals import deploy +from pants.backend.k8s.goals import deploy, tailor def rules(): return [ - *kubectl_subsystem.rules(), - *k8s_subsystem.rules(), *deploy.rules(), + *k8s_subsystem.rules(), + *kubectl_subsystem.rules(), + *tailor.rules(), ] diff --git a/src/python/pants/backend/k8s/goals/deploy_test.py b/src/python/pants/backend/k8s/goals/deploy_test.py index 4b7b30bb587..f99c76f8273 100644 --- a/src/python/pants/backend/k8s/goals/deploy_test.py +++ b/src/python/pants/backend/k8s/goals/deploy_test.py @@ -12,7 +12,7 @@ from pants.backend.k8s.goals.deploy import DeployK8sBundleFieldSet from pants.backend.k8s.goals.deploy import rules as k8s_deploy_rules from pants.backend.k8s.kubectl_subsystem import Kubectl -from pants.backend.k8s.target_types import K8sBundleTarget, K8sSourceTargetGenerator +from pants.backend.k8s.target_types import K8sBundleTarget, K8sSourcesTargetGenerator from pants.core.goals.deploy import DeployProcess from pants.engine.addresses import Address from pants.engine.internals.scheduler import ExecutionError @@ -24,7 +24,7 @@ def rule_runner() -> RuleRunner: rule_runner = RuleRunner( target_types=[ - K8sSourceTargetGenerator, + K8sSourcesTargetGenerator, K8sBundleTarget, ], rules=[ diff --git a/src/python/pants/backend/k8s/goals/tailor.py b/src/python/pants/backend/k8s/goals/tailor.py new file mode 100644 index 00000000000..248c7b9c879 --- /dev/null +++ b/src/python/pants/backend/k8s/goals/tailor.py @@ -0,0 +1,57 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from dataclasses import dataclass + +from pants.backend.k8s.k8s_subsystem import K8sSubsystem +from pants.backend.k8s.target_types import K8sSourcesTargetGenerator +from pants.core.goals.tailor import ( + AllOwnedSources, + PutativeTarget, + PutativeTargets, + PutativeTargetsRequest, +) +from pants.engine.fs import PathGlobs, Paths +from pants.engine.internals.selectors import Get +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.dirutil import group_by_dir +from pants.util.logging import LogLevel + + +@dataclass(frozen=True) +class PutativeK8sTargetsRequest(PutativeTargetsRequest): + pass + + +@rule(level=LogLevel.DEBUG, desc="Determine candidate k8s targets to create") +async def find_putative_targets( + req: PutativeK8sTargetsRequest, + all_owned_sources: AllOwnedSources, + k8s: K8sSubsystem, +) -> PutativeTargets: + putative_targets = [] + + if k8s.tailor_source_targets: + all_k8s_files_globs = req.path_globs("*.yaml") + all_k8s_files = await Get(Paths, PathGlobs, all_k8s_files_globs) + unowned_k8s_files = set(all_k8s_files.files) - set(all_owned_sources) + + for dirname, filenames in group_by_dir(unowned_k8s_files).items(): + name = None + putative_targets.append( + PutativeTarget.for_target_type( + K8sSourcesTargetGenerator, + path=dirname, + name=name, + triggering_sources=sorted(filenames), + ) + ) + + return PutativeTargets(putative_targets) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PutativeTargetsRequest, PutativeK8sTargetsRequest), + ] diff --git a/src/python/pants/backend/k8s/k8s_subsystem.py b/src/python/pants/backend/k8s/k8s_subsystem.py index 5de5f575121..7d6bc29ce09 100644 --- a/src/python/pants/backend/k8s/k8s_subsystem.py +++ b/src/python/pants/backend/k8s/k8s_subsystem.py @@ -3,7 +3,7 @@ import logging from pants.engine.rules import collect_rules -from pants.option.option_types import StrListOption +from pants.option.option_types import BoolOption, StrListOption from pants.option.subsystem import Subsystem from pants.util.strutil import softwrap @@ -31,6 +31,16 @@ class K8sSubsystem(Subsystem): ), ) + tailor_source_targets = BoolOption( + default=True, + help=softwrap( + """ + If true, add `k8s_sources` targets with the `tailor` goal. + """ + ), + advanced=True, + ) + def rules(): return collect_rules() diff --git a/src/python/pants/backend/k8s/target_types.py b/src/python/pants/backend/k8s/target_types.py index edcaa5aa3b7..fd422b449d4 100644 --- a/src/python/pants/backend/k8s/target_types.py +++ b/src/python/pants/backend/k8s/target_types.py @@ -43,7 +43,7 @@ class K8sSourceTarget(Target): help = "A single k8s object spec file." -class K8sSourceTargetGenerator(TargetFilesGenerator): +class K8sSourcesTargetGenerator(TargetFilesGenerator): alias = "k8s_sources" generated_target_cls = K8sSourceTarget @@ -88,6 +88,6 @@ class K8sBundleTarget(Target): def target_types(): return [ K8sSourceTarget, - K8sSourceTargetGenerator, + K8sSourcesTargetGenerator, K8sBundleTarget, ] From fb2c8e5cc25b9e00043bf5663ed9a1d35e2a7616 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sun, 22 Dec 2024 23:29:38 +0100 Subject: [PATCH 10/25] Add a script to fetch kubectl versions --- build-support/bin/BUILD | 8 ++ build-support/bin/external_tool_versions.py | 141 ++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 build-support/bin/external_tool_versions.py diff --git a/build-support/bin/BUILD b/build-support/bin/BUILD index 39c84a09d10..0596c2f4e1c 100644 --- a/build-support/bin/BUILD +++ b/build-support/bin/BUILD @@ -24,3 +24,11 @@ pex_binary( name="terraform_tool_versions", entry_point="terraform_tool_versions.py", ) + +pex_binary( + name="external-tool-versions", + entry_point="external_tool_versions.py", + dependencies=[ + "src/python/pants/backend/k8s/kubectl_subsystem.py", + ], +) diff --git a/build-support/bin/external_tool_versions.py b/build-support/bin/external_tool_versions.py new file mode 100644 index 00000000000..355e968f1d8 --- /dev/null +++ b/build-support/bin/external_tool_versions.py @@ -0,0 +1,141 @@ +import argparse +import hashlib +from multiprocessing.pool import ThreadPool +import re +from dataclasses import dataclass +import xml.etree.ElementTree as ET +from collections.abc import Callable, Iterator +import importlib +from string import Formatter +from urllib.parse import urlparse + +import logging +import requests + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Version: + version: str + platform: str + + +@dataclass(frozen=True) +class VersionHash: + version: str + platform: str + size: int + sha256: str + + +def format_string_to_regex(format_string: str) -> re.Pattern: + """Converts a format string to a regex. + + >>> format_string_to_regex("/release/v{version}/bin/{platform}/kubectl") + re.compile('^\\/release\\/v(?P.*)\\/bin\\/(?P.*)\\/kubectl$') + """ + result_regex = ["^"] + parts = Formatter().parse(format_string) + for literal_text, field_name, format_spec, conversion in parts: + escaped_text = literal_text.replace("/", r"\/") + result_regex.append(escaped_text) + if field_name is not None: + result_regex.append(rf"(?P<{field_name}>.*)") + result_regex.append("$") + return re.compile("".join(result_regex)) + + +def get_k8s_versions(url_template: str) -> Iterator[Version]: + path_template = urlparse(url_template).path + logger.info("path template: %s", path_template) + + regex = format_string_to_regex(path_template) + logger.info("path regex: %s", regex) + + response = requests.get("https://cdn.dl.k8s.io/", allow_redirects=True) + root = ET.fromstring(response.text) + tag = "{http://doc.s3.amazonaws.com/2006-03-01}" + for item in root.iter(f"{tag}Contents"): + key_element = item.find(f"{tag}Key") + if key_element is None: + raise RuntimeError("Failed to parse xml, did it change?") + + key = key_element.text + if match := regex.match(f"/{key}"): + version = match.group("version") + platform = match.group("platform") + yield Version(version=version, platform=platform) + + +DOMAIN_TO_VERSIONS_MAPPING: dict[str, Callable[[str], Iterator[Version]]] = { + # TODO github.com + "dl.k8s.io": get_k8s_versions, +} + + +def fetch_version(url_template: str, version: Version) -> VersionHash: + url = url_template.format(version=version.version, platform=version.platform) + response = requests.get(url) + size = len(response.content) + sha256 = hashlib.sha256(response.content) + return VersionHash( + version=version.version, + platform=version.platform, + size=size, + sha256=sha256.hexdigest(), + ) + + +def main(): + logging.basicConfig(level="INFO", format="%(message)s") + parser = argparse.ArgumentParser() + parser.add_argument( + "-t", + "--tool", + help="Python tool location, for example: pants.backend.tools.taplo.subsystem:Taplo", + required=True, + ) + parser.add_argument( + "--platforms", + default="macos_arm64,macos_x86_64,linux_arm64,linux_x86_64", + help="Comma separated list of platforms", + ) + parser.add_argument( + "-w", + "--workers", + default=16, + help="Thread pool size", + ) + + args = parser.parse_args() + + module_string, class_name = args.tool.split(":") + module = importlib.import_module(module_string) + cls = getattr(module, class_name) + + platforms = args.platforms.split(",") + mapped_platforms = set(cls.default_url_platform_mapping.get(p) for p in platforms) + + domain = urlparse(cls.default_url_template).netloc + get_versions = DOMAIN_TO_VERSIONS_MAPPING[domain] + pool = ThreadPool(processes=args.workers) + results = [] + for version in get_versions(cls.default_url_template): + if version.platform not in mapped_platforms: + continue + + logger.info("fetching version: %s", version) + results.append( + pool.apply_async( + fetch_version, + args=(cls.default_url_template, version), + ) + ) + + for result in results: + print(result.get(60)) + + +if __name__ == "__main__": + main() From db2d216142ed71364394115d1a7eaf6b119a4824 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Mon, 23 Dec 2024 00:14:08 +0100 Subject: [PATCH 11/25] Add more known versions --- build-support/bin/BUILD | 1 + build-support/bin/external_tool_versions.py | 100 ++++++++------ .../pants/backend/k8s/kubectl_subsystem.py | 122 +++++++++++++++++- 3 files changed, 181 insertions(+), 42 deletions(-) diff --git a/build-support/bin/BUILD b/build-support/bin/BUILD index 0596c2f4e1c..455d7384878 100644 --- a/build-support/bin/BUILD +++ b/build-support/bin/BUILD @@ -31,4 +31,5 @@ pex_binary( dependencies=[ "src/python/pants/backend/k8s/kubectl_subsystem.py", ], + execution_mode="venv", ) diff --git a/build-support/bin/external_tool_versions.py b/build-support/bin/external_tool_versions.py index 355e968f1d8..2a19def4b5c 100644 --- a/build-support/bin/external_tool_versions.py +++ b/build-support/bin/external_tool_versions.py @@ -1,3 +1,9 @@ +"""Script to fetch external tool versions. + +Example: + +pants run build-support/bin:external-tool-versions -- --tool pants.backend.k8s.kubectl_subsystem:Kubectl > list.txt +""" import argparse import hashlib from multiprocessing.pool import ThreadPool @@ -15,12 +21,6 @@ logger = logging.getLogger(__name__) -@dataclass(frozen=True) -class Version: - version: str - platform: str - - @dataclass(frozen=True) class VersionHash: version: str @@ -46,15 +46,14 @@ def format_string_to_regex(format_string: str) -> re.Pattern: return re.compile("".join(result_regex)) -def get_k8s_versions(url_template: str) -> Iterator[Version]: - path_template = urlparse(url_template).path - logger.info("path template: %s", path_template) +def fetch_text(url: str) -> str: + response = requests.get(url) + return response.text - regex = format_string_to_regex(path_template) - logger.info("path regex: %s", regex) - response = requests.get("https://cdn.dl.k8s.io/", allow_redirects=True) - root = ET.fromstring(response.text) +def _parse_k8s_xml(text: str) -> Iterator[str]: + regex = re.compile(r"release\/stable-(?P[0-9\.]+).txt") + root = ET.fromstring(text) tag = "{http://doc.s3.amazonaws.com/2006-03-01}" for item in root.iter(f"{tag}Contents"): key_element = item.find(f"{tag}Key") @@ -62,33 +61,41 @@ def get_k8s_versions(url_template: str) -> Iterator[Version]: raise RuntimeError("Failed to parse xml, did it change?") key = key_element.text - if match := regex.match(f"/{key}"): - version = match.group("version") - platform = match.group("platform") - yield Version(version=version, platform=platform) + if regex.match(key): + yield f"https://cdn.dl.k8s.io/{key}" + + +def get_k8s_versions(url_template: str, pool: ThreadPool) -> Iterator[str]: + response = requests.get("https://cdn.dl.k8s.io/", allow_redirects=True) + urls = _parse_k8s_xml(response.text) + for v in pool.imap_unordered(fetch_text, urls): + yield v.strip().lstrip("v") -DOMAIN_TO_VERSIONS_MAPPING: dict[str, Callable[[str], Iterator[Version]]] = { +DOMAIN_TO_VERSIONS_MAPPING: dict[str, Callable[[str, ThreadPool], Iterator[str]]] = { # TODO github.com "dl.k8s.io": get_k8s_versions, } -def fetch_version(url_template: str, version: Version) -> VersionHash: - url = url_template.format(version=version.version, platform=version.platform) - response = requests.get(url) +def fetch_version(url_template: str, version: str, platform: str) -> VersionHash | None: + url = url_template.format(version=version, platform=platform) + response = requests.get(url, allow_redirects=True) + if response.status_code != 200: + logger.error("failed to fetch version: %s\n%s", version, response.text) + return None + size = len(response.content) sha256 = hashlib.sha256(response.content) return VersionHash( - version=version.version, - platform=version.platform, + version=version, + platform=platform, size=size, sha256=sha256.hexdigest(), ) def main(): - logging.basicConfig(level="INFO", format="%(message)s") parser = argparse.ArgumentParser() parser.add_argument( "-t", @@ -107,34 +114,55 @@ def main(): default=16, help="Thread pool size", ) + parser.add_argument( + "-v", + "--verbose", + action=argparse.BooleanOptionalAction, + default=False, + help="Verbose output", + ) args = parser.parse_args() + level = logging.DEBUG if args.verbose else logging.INFO + logging.basicConfig(level=level, format="%(message)s") + module_string, class_name = args.tool.split(":") module = importlib.import_module(module_string) cls = getattr(module, class_name) platforms = args.platforms.split(",") - mapped_platforms = set(cls.default_url_platform_mapping.get(p) for p in platforms) + platform_mapping = cls.default_url_platform_mapping + mapped_platforms = set(platform_mapping.get(p) for p in platforms) domain = urlparse(cls.default_url_template).netloc get_versions = DOMAIN_TO_VERSIONS_MAPPING[domain] pool = ThreadPool(processes=args.workers) results = [] - for version in get_versions(cls.default_url_template): - if version.platform not in mapped_platforms: - continue - - logger.info("fetching version: %s", version) - results.append( - pool.apply_async( - fetch_version, - args=(cls.default_url_template, version), + for version in get_versions(cls.default_url_template, pool): + for platform in mapped_platforms: + logger.debug("fetching version: %s %s", version, platform) + results.append( + pool.apply_async( + fetch_version, args=(cls.default_url_template, version, platform) + ) ) - ) + backward_platform_mapping = {v: k for k, v in platform_mapping.items()} for result in results: - print(result.get(60)) + v = result.get(60) + if v is None: + continue + print( + "|".join( + [ + v.version, + backward_platform_mapping[v.platform], + v.sha256, + str(v.size), + ] + ) + ) if __name__ == "__main__": diff --git a/src/python/pants/backend/k8s/kubectl_subsystem.py b/src/python/pants/backend/k8s/kubectl_subsystem.py index e5493f219b8..e9c428a24c7 100644 --- a/src/python/pants/backend/k8s/kubectl_subsystem.py +++ b/src/python/pants/backend/k8s/kubectl_subsystem.py @@ -24,14 +24,122 @@ class Kubectl(TemplatedExternalTool): default_version = "1.32.0" default_known_versions = [ - "1.32.0|linux_arm64|d7389b9743b0b909c364d11bba94d13302171d751430b58c13dcdf248e924276|7605249", - "1.32.0|linux_x86_64|d7389b9743b0b909c364d11bba94d13302171d751430b58c13dcdf248e924276|7605249", - "1.32.0|macos_arm64|d7389b9743b0b909c364d11bba94d13302171d751430b58c13dcdf248e924276|7605249", - "1.32.0|macos_x86_64|d7389b9743b0b909c364d11bba94d13302171d751430b58c13dcdf248e924276|7605249", + "1.0.7|linux_x86_64|78fa9a5628ae29b091ea0bbab58635fe899fd0c5c01646fb2bf7b6dff6097b70|19387320", + "1.0.7|macos_x86_64|f40f5606ff24e97b9df926febb6f490ba138ddd9cfb8c1d132036408b7181d87|19136448", + "1.1.8|linux_x86_64|b2222986e9f05da8091a16134022d243b3c46a5899486d1b775dbc950ebf36cd|22589688", + "1.1.8|macos_x86_64|da67619ec248a480db20d0704680e0ea91fe0787c8070f342d570c70d957e060|22097696", + "1.10.13|linux_arm64|f52bd7804bec8dadb64af1610bb5c89fd2f2c37a4533c4723df04edb937d0f87|52620142", + "1.10.13|linux_x86_64|0157b02fe9f42a6a5fc738597259d9d22f2d1cb4235d96347e01d1cf7b984980|55100736", + "1.10.13|macos_x86_64|94ec5409ac6715dfec11309795da9b30254b9d818f8b1d38d1ca2e914945868d|54670032", + "1.11.10|linux_arm64|8bbf7e9b666a8c639907e839fad226b46604fd22f8a33a0fc85acda248d571a9|54746905", + "1.11.10|linux_x86_64|fe1a101b476e54515458f98bb0747f2ddfefb69861fe9786a88d7b5ce61e6f45|55461388", + "1.11.10|macos_x86_64|cf27dd3e8d5a13439de2214b7a07e2371cdbeaf151ee4c7d63e7e7c34ebbf117|54985744", + "1.12.10|linux_arm64|c306e470c31227d4de7c3c0cecb06a694501563d1332301d4ac2080422870021|56618322", + "1.12.10|linux_x86_64|b1250b8cadea0e8ad2896f6379abd52bcbbfa2d8ff3253d86d7e705003d83da4|57361431", + "1.12.10|macos_x86_64|67acf17bf77b0a9729d17ad5c058a358c0cbd12dbf46478c37111cdd97a0828c|56860112", + "1.13.12|linux_arm64|47ffe9064318c6a9613f6ac5a5f96ffb43dec6dc4a37ea4b2992bf378c8e6f02|36593184", + "1.13.12|linux_x86_64|3578dbaec9fd043cf2779fbc54afb4297f3e8b50df7493191313bccbb8046300|39271904", + "1.13.12|macos_x86_64|ddbdc7591569f321b8b0a01dcbd679f6b0a7756f1427a51a39eadfce8d34bea7|44353656", + "1.14.10|linux_arm64|7927cfdbf6c793626d0e437ca2c45dd2c1431b6629193ef17f798e81a76b4234|41337728", + "1.14.10|linux_x86_64|7729c6612bec76badc7926a79b26e0d9b06cc312af46dbb80ea7416d1fce0b36|43119424", + "1.14.10|macos_x86_64|43d2c24eafb2ef09a6ac77c2b99070668e83edaa325a16a362e304ba578fdc48|48716328", + "1.15.12|linux_arm64|ef9a4272d556851c645d6788631a2993823260a7e1176a281620284b4c3406da|41228416", + "1.15.12|linux_x86_64|a32b762279c33cb8d8f4198f3facdae402248c3164e9b9b664c3afbd5a27472e|43059232", + "1.15.12|macos_x86_64|1b06cab9ee7988f8e71c48dd1d9379aa7c14123cbbc63e12ab5342c3e9130972|48668112", + "1.16.15|linux_arm64|74719f137dc6d589a3b8a667bcb0f3c57eebd8f050dd2f7ad5b59ceb892a7b99|40697856", + "1.16.15|linux_x86_64|e8913069293156ddf55f243814a22d2384fc18b165efb6200606fdeaad146605|42967040", + "1.16.15|macos_x86_64|aff54bfaaed905813f61a2d0ca039176d6d309e59f92ebdb297c7da1df105485|48907792", + "1.17.17|linux_arm64|6ffc1749adbda24474e67678fcc4a1e704c4e1b9354508965bbab3578bd801ba|41091072", + "1.17.17|linux_x86_64|8329fac94c66bf7a475b630972a8c0b036bab1f28a5584115e8dd26483de8349|43458560", + "1.17.17|macos_x86_64|e76b57bbed823a8572f7ccbf9caae56855048434d818dc13559e89ead5f91578|49542736", + "1.18.20|linux_arm64|31e6bbc657b13ce1b932bf7589bca41a25b0612b4d897b1f363dc9c5a8080a22|41680896", + "1.18.20|linux_x86_64|66a9bb8e9843050340844ca6e72e67632b75b9ebb651559c49db22f35450ed2f|43958272", + "1.18.20|macos_x86_64|bc709378c27da458b31395787a74cd9ac58dce7dbe2a7ba92f8bc2221eeed0be|50095696", + "1.19.16|linux_arm64|6ad55694db34b9ffbc3cb41761a50160eea0a962eb86899410593931b4e602d0|39845888", + "1.19.16|linux_x86_64|6b9d9315877c624097630ac3c9a13f1f7603be39764001da7a080162f85cbc7e|42950656", + "1.19.16|macos_x86_64|7fdbc38bb9d93514cfc20e7770991a9f726836f800778647211a4802959fcf01|49406176", + "1.2.7|linux_x86_64|d5585f95aba909e80d8364523a198d9c70327c3563c4d1b3fa6c9cb66c8d5efc|41025232", + "1.2.7|macos_x86_64|6a9ce64210aea84349ebd9296d7f53d05c734d65e166a6392b96af2588dfa860|40468352", + "1.20.15|linux_arm64|d479febfb2e967bd86240b5c0b841e40e39e1ef610afd6f224281a23318c13dc|37158912", + "1.20.15|linux_x86_64|d283552d3ef3b0fd47c08953414e1e73897a1b3f88c8a520bb2e7de4e37e96f3|40243200", + "1.20.15|macos_x86_64|6b6cf555a34271379b45013dfa9b580329314254aafc91b543bf2d83ebd1db74|46242192", + "1.21.14|linux_arm64|a23151bca5d918e9238546e7af416422b51cda597a22abaae5ca50369abfbbaa|43319296", + "1.21.14|linux_x86_64|0c1682493c2abd7bc5fe4ddcdb0b6e5d417aa7e067994ffeca964163a988c6ee|46686208", + "1.21.14|macos_arm64|e0e6e413e19abc9deb15f9bd3c72f73ff5539973758e64ebca0f5eb085de6a00|51872962", + "1.21.14|macos_x86_64|30c529fe2891eb93dda99597b5c84cb10d2318bb92ae89e1e6189b3ae5fb6296|52867344", + "1.22.17|linux_arm64|8fc2f8d5c80a6bf60be06f8cf28679a05ce565ce0bc81e70aaac38e0f7da6259|43515904", + "1.22.17|linux_x86_64|7506a0ae7a59b35089853e1da2b0b9ac0258c5309ea3d165c3412904a9051d48|46944256", + "1.22.17|macos_arm64|b2d881bd6d3c688645cbc9e5b4cf4fe8945e1cfc3f2c07c795d2ee605ce4e568|52098562", + "1.22.17|macos_x86_64|c3b8ae5ad48e1e126b5db2e7e22bb1e6ac54901a7f94ce499d12316f705e5e15|53133440", + "1.23.17|linux_arm64|c4a48fdc6038beacbc5de3e4cf6c23639b643e76656aabe2b7798d3898ec7f05|43778048", + "1.23.17|linux_x86_64|f09f7338b5a677f17a9443796c648d2b80feaec9d6a094ab79a77c8a01fde941|45174784", + "1.23.17|macos_arm64|3b4590d67b31e3a94a9633064571c981907555da5376c34960cddfcd552f6114|51181986", + "1.23.17|macos_x86_64|7ece6543e3ca2ae9698ef61bbb2a4e249aa21319df4ea1b27c136a9b005dd7d8|51721104", + "1.24.17|linux_arm64|66885bda3a202546778c77f0b66dcf7f576b5a49ff9456acf61329da784a602d|44630016", + "1.24.17|linux_x86_64|3e9588e3326c7110a163103fc3ea101bb0e85f4d6fd228cf928fa9a2a20594d5|46706688", + "1.24.17|macos_arm64|7addbe3f1e22a366fa05aed4f268e77e83d902b40a5854e192b4205ed92e5f8d|52955666", + "1.24.17|macos_x86_64|1eb904b2c1148ff8431b0bd86677287a48bff000f93fd2d36377fbe956bd1e49|53481056", + "1.25.16|linux_arm64|d6c23c80828092f028476743638a091f2f5e8141273d5228bf06c6671ef46924|43581440", + "1.25.16|linux_x86_64|5a9bc1d3ebfc7f6f812042d5f97b82730f2bdda47634b67bddf36ed23819ab17|45658112", + "1.25.16|macos_arm64|d364f73df218b02642d06f3fa9b7345d64c03567b96ca21d361b487f48a33ccc|50416738", + "1.25.16|macos_x86_64|34e87fdf0613502edbd2a2b00de5ee8c7789ab10e33257d14423dc6879321920|50954608", + "1.26.15|linux_arm64|1396313f0f8e84ab1879757797992f1af043e1050283532e0fd8469902632216|46661632", + "1.26.15|linux_x86_64|b75f359e6fad3cdbf05a0ee9d5872c43383683bb8527a9e078bb5b8a44350a41|48148480", + "1.26.15|macos_arm64|c20b920d7e8e3ce3209c7c109fcfc4c09ad599613bc04b72c3f70d9fee598b68|53860082", + "1.26.15|macos_x86_64|ad4e980f9c304840ec9227a78a998e132ea23f3ca1bc0df7718ed160341bad0b|54047120", + "1.27.16|linux_arm64|2f50cb29d73f696ffb57437d3e2c95b22c54f019de1dba19e2b834e0b4501eb9|47644824", + "1.27.16|linux_x86_64|97ea7cd771d0c6e3332614668a40d2c5996f0053ff11b44b198ea84dba0818cb|49066136", + "1.27.16|macos_arm64|d6bc47098bcb13a0ff5c267b30021b499aff4d960bd92610c2b0bc6f6e7246c9|49017698", + "1.27.16|macos_x86_64|8d7f339660ba9b33ed56d540bed41b37babc945975a9e7027010697249b9ac5a|50145152", + "1.28.15|linux_arm64|7d45d9620e67095be41403ed80765fe47fcfbf4b4ed0bf0d1c8fe80345bda7d3|48169112", + "1.28.15|linux_x86_64|1f7651ad0b50ef4561aa82e77f3ad06599b5e6b0b2a5fb6c4f474d95a77e41c5|49623192", + "1.28.15|macos_arm64|06a276bdb6da95af148d589f6c983ec8ea10c38f277ced6d97123938c8146078|49593410", + "1.28.15|macos_x86_64|3180c84131002037d60fe7322794c20297d0e1b1514eaea20e33f77a00d8f2f4|50716416", + "1.29.12|linux_arm64|1cf2c00bb4f5ee6df69678e95af8ba9a4d4b1050ddefb0ae9d84b5c6f6c0e817|48758936", + "1.29.12|linux_x86_64|35fc028853e6f5299a53f22ab58273ea2d882c0f261ead0a2eed5b844b12dbfb|50225304", + "1.29.12|macos_arm64|5d1c59d8ce4d619bdd78fa849201dbfc9180f6dddcfdb30f29b5bbe20799b897|50169474", + "1.29.12|macos_x86_64|0df5932d0ba7a4665ea8033470f2f1a1db21637c3fabc709faa19db0fc62b5ec|51333056", + "1.3.10|linux_arm64|0c35abb5bf70ffa40b02a1c03c914067bf703e37fb0f53392bcce2476df005f0|55720280", + "1.3.10|linux_x86_64|2e72c96b86074dd969b9c49867874a97e8f594fb3e39d3f0ed2ac7add353666d|56525120", + "1.3.10|macos_x86_64|d2f482cae5aefa2fd6afa5b3d8ecb8de8c5a49b22c42c3dce1b5300a05b0109f|55862352", + "1.30.8|linux_arm64|e51d6a76fade0871a9143b64dc62a5ff44f369aa6cb4b04967d93798bf39d15b|49938584", + "1.30.8|linux_x86_64|7f39bdcf768ce4b8c1428894c70c49c8b4d2eee52f3606eb02f5f7d10f66d692|51454104", + "1.30.8|macos_arm64|52b11bb032f88e4718cd4e3c8374a6b1fad29772aa1ce701276cc4e17d37642f|51395442", + "1.30.8|macos_x86_64|46682e24c3aecfbe92f53b86fb15beb740c43a0fafe0a4e06a1c8bb3ce9e985b|52586352", + "1.31.4|linux_arm64|b97e93c20e3be4b8c8fa1235a41b4d77d4f2022ed3d899230dbbbbd43d26f872|54984856", + "1.31.4|linux_x86_64|298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f|56381592", + "1.31.4|macos_arm64|a756bb911298a85af35c0111c371728a26c532d504fe8b534eb684501fcaf996|56560802", + "1.31.4|macos_x86_64|fd996e9f41fd42c6c1c781a5a85990f4d0d8337ede00a7719afa23be886e0abd|57637984", + "1.32.0|linux_arm64|ba4004f98f3d3a7b7d2954ff0a424caa2c2b06b78c17b1dccf2acc76a311a896|55836824", + "1.32.0|linux_arm64|ba4004f98f3d3a7b7d2954ff0a424caa2c2b06b78c17b1dccf2acc76a311a896|55836824", + "1.32.0|linux_x86_64|646d58f6d98ee670a71d9cdffbf6625aeea2849d567f214bc43a35f8ccb7bf70|57323672", + "1.32.0|linux_x86_64|646d58f6d98ee670a71d9cdffbf6625aeea2849d567f214bc43a35f8ccb7bf70|57323672", + "1.32.0|macos_arm64|5bfd5de53a054b4ef614c60748e28bf47441c7ed4db47ec3c19a3e2fa0eb5555|57472706", + "1.32.0|macos_arm64|5bfd5de53a054b4ef614c60748e28bf47441c7ed4db47ec3c19a3e2fa0eb5555|57472706", + "1.32.0|macos_x86_64|516585916f499077fac8c2fdd2a382818683f831020277472e6bcf8d1a6f9be4|58612096", + "1.32.0|macos_x86_64|516585916f499077fac8c2fdd2a382818683f831020277472e6bcf8d1a6f9be4|58612096", + "1.4.12|linux_arm64|5fc307700d3f2b4682e7a662f251f6dd534f3e12d9a84c20697a73cb6c6a7f22|78167368", + "1.4.12|linux_x86_64|e0376698047be47f37f126fcc4724487dcc8edd2ffb993ae5885779786efb597|79558032", + "1.4.12|macos_x86_64|9c7c5525fe77ebed45dcc949990fcb8998eb6fe0b2441a75c1d58ee7268116d3|63405984", + "1.5.8|linux_arm64|a459fd0e5bd2b002d1423d092c7f1613e095e5485cdda032eaf34303f57adfc3|52137665", + "1.5.8|linux_x86_64|647e233fe0b935300a981b61245b29c7dae6af772dc1f2243cfa1970d2e90219|50372958", + "1.5.8|macos_x86_64|3a4c98ad33892831026a59af6161a2cca0b9928ae098436d88e5224264535e64|50036704", + "1.6.13|linux_arm64|cb891241dbc7e043cafacf6a504a09c2fd4c582798d47235fb5ca64517d2d04b|73688363", + "1.6.13|linux_x86_64|17e29707dcdaac878178d4b137c798cb37993a8a7f0ae214835af4f8e322bafa|70704763", + "1.6.13|macos_x86_64|4623929aaf3037489b2d96561cef4037ad3399f16bdd1469cc5fc9becb4581aa|70232912", + "1.7.16|linux_arm64|123e3f8d0ddfd2b75cb56f76c0b75c56a1960aba73c9a212a153bd425a206b20|70997024", + "1.7.16|linux_x86_64|67e27be929afa1aa103eec0978a2a50ef3df1bd1454b979bb776e472a73c21b2|72497289", + "1.7.16|macos_x86_64|91618ff648ffa9878a64091d7d9440199475855e0fcfae30dab5812c67ea50ac|71985600", + "1.8.15|linux_arm64|ec6d8b93dc74555822dd741eace7e99431d9efc7c3490b4bc46c0bfe24a54b82|52079389", + "1.8.15|linux_x86_64|ac6c59308b91536bc1482c094576bf8685dc5372509a383fb4833ac7299b0e56|53284738", + "1.8.15|macos_x86_64|0a876863dbee07130ead4bd87baf6b5bac7ca63a59d3c0417444a78a145ee3bd|52881856", + "1.9.11|linux_arm64|dd9308db2a76efacff10e9b214f8c04752daa5fa1d1b432c97150bf883e5f091|65530300", + "1.9.11|linux_x86_64|3aa80b62fbd9cfa082aa26ae6a141a6ac209543d31e6f88ad5df47842ed8ddc3|68375438", + "1.9.11|macos_x86_64|b3e46e2a4ba5e29bc43251e9902c8ff6bc21cdbe8c2e20c79efb94bb3d954c02|67840704", ] version_constraints = ">=1,<2" - default_url_template = "https://dl.k8s.io/release/{version}/bin/{platform}/kubectl" + default_url_template = "https://dl.k8s.io/release/v{version}/bin/{platform}/kubectl" default_url_platform_mapping = { "linux_arm64": "linux/arm64", "linux_x86_64": "linux/amd64", @@ -64,7 +172,9 @@ class Kubectl(TemplatedExternalTool): advanced=True, ) - class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware): + class EnvironmentAware( + ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware + ): executable_search_paths_help = softwrap( """ The PATH value that will be used to find kubectl binary. From f5e11d6daaa993ee6ec0c6d177d72eac1ae7c7c0 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Mon, 23 Dec 2024 00:19:01 +0100 Subject: [PATCH 12/25] Add more docs --- docs/docs/kubernetes/index.mdx | 52 +++++++++++++++- src/python/pants/backend/k8s/goals/deploy.py | 62 +++++++++++++++++--- src/python/pants/backend/k8s/kubectl_tool.py | 44 -------------- 3 files changed, 106 insertions(+), 52 deletions(-) delete mode 100644 src/python/pants/backend/k8s/kubectl_tool.py diff --git a/docs/docs/kubernetes/index.mdx b/docs/docs/kubernetes/index.mdx index 984aab189d6..8a3fc2a4cae 100644 --- a/docs/docs/kubernetes/index.mdx +++ b/docs/docs/kubernetes/index.mdx @@ -35,7 +35,7 @@ your .yaml files. For example, create a file `src/k8s/configmap.yaml`: -```yaml +```yaml title="src/k8s/configmap.yaml" --- apiVersion: v1 kind: ConfigMap @@ -59,3 +59,53 @@ Created src/k8s/BUILD: - Add k8s_sources target k8s ``` +## Deploying objects to a cluster + +We'll be using a local [kind](https://kind.sigs.k8s.io/) cluster throughout the +tutorial. First, spin up a cluster: + +```bash +kind create cluster +``` +``` +Creating cluster "kind" ... + ✓ Ensuring node image (kindest/node:v1.25.3) 🖼 + ✓ Preparing nodes 📦 + ✓ Writing configuration 📜 + ✓ Starting control-plane 🕹️ + ✓ Installing CNI 🔌 + ✓ Installing StorageClass 💾 +Set kubectl context to "kind-kind" +``` + +Second, configure the list of available contexts in `pants.toml`: + +```toml title="pants.toml" +... + +[k8s] +available_contexts = [ + "kind-kind", +] +``` + +Third, create a deployable target `k8s_bundle` in `src/k8s/BUILD`: + +```python title="src/k8s/BUILD" +k8s_sources() +k8s_bundle( + name="configmap", + sources=("src/k8s/configmap.yaml",), + context="kind-kind", +) +``` + +Now you can deploy the target: + +```bash +pants experimental-deploy src/k8s:configmap +``` +``` +✓ src/k8s:configmap deployed to context kind-kind +``` + diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index 54d70c54d1a..85c75e2e070 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -15,11 +15,16 @@ K8sSourceField, ) from pants.core.goals.deploy import DeployFieldSet, DeployProcess +from pants.core.util_rules.external_tool import ( + DownloadedExternalTool, + ExternalToolRequest, +) from pants.engine.addresses import UnparsedAddressInputs from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest from pants.engine.fs import MergeDigests, Snapshot +from pants.engine.internals.native_engine import Digest from pants.engine.platform import Platform -from pants.engine.process import InteractiveProcess +from pants.engine.process import InteractiveProcess, Process, ProcessCacheScope from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( DependenciesRequest, @@ -29,6 +34,7 @@ Targets, ) from pants.engine.unions import UnionRule +from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel logger = logging.getLogger(__name__) @@ -46,6 +52,15 @@ class DeployK8sBundleFieldSet(DeployFieldSet): dependencies: K8sBundleDependenciesField +@dataclass(frozen=True) +class KubectlApply: + paths: tuple[str, ...] + input_digest: Digest + platform: Platform + env: FrozenDict[str, str] | None = None + context: str | None = None + + @rule(desc="Run k8s deploy process", level=LogLevel.DEBUG) async def run_k8s_deploy( field_set: DeployK8sBundleFieldSet, @@ -96,12 +111,15 @@ async def run_k8s_deploy( ) process = InteractiveProcess.from_process( - kubectl.apply_configs( - snapshot.files, - platform=platform, - input_digest=snapshot.digest, - env=env, - context=context, + await Get( + Process, + KubectlApply( + snapshot.files, + platform=platform, + input_digest=snapshot.digest, + env=env, + context=context, + ), ) ) @@ -115,6 +133,36 @@ async def run_k8s_deploy( ) +@rule +async def kubectl_apply_process( + request: KubectlApply, platform: Platform, kubectl: Kubectl +) -> Process: + argv: tuple[str, ...] = (kubectl.generate_exe(platform),) + + if request.context is not None: + argv += ("--context", request.context) + + argv += ("apply", "-o", "yaml") + + for path in request.paths: + argv += ("-f", path) + + kubectl_tool = await Get( + DownloadedExternalTool, ExternalToolRequest, kubectl.get_request(platform) + ) + digest = await Get( + Digest, MergeDigests([kubectl_tool.digest, request.input_digest]) + ) + + return Process( + argv=argv, + input_digest=digest, + cache_scope=ProcessCacheScope.PER_SESSION, + description=f"Applying kubernetes config {request.paths}", + env=request.env, + ) + + def rules(): return [ *collect_rules(), diff --git a/src/python/pants/backend/k8s/kubectl_tool.py b/src/python/pants/backend/k8s/kubectl_tool.py deleted file mode 100644 index 3a33f0906b4..00000000000 --- a/src/python/pants/backend/k8s/kubectl_tool.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). -import logging -from dataclasses import dataclass - -from pants.backend.k8s.kubectl_subsystem import Kubectl -from pants.core.util_rules.system_binaries import ( - BinaryPath, - BinaryPathRequest, - BinaryPaths, - BinaryPathTest, -) -from pants.engine.rules import Get, collect_rules, rule -from pants.util.logging import LogLevel - -logger = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class KubectlBinary(BinaryPath): - """The `kubectl` binary.""" - - -@rule(desc="Finding the `kubectl` binary", level=LogLevel.DEBUG) -async def get_kubectl( - kubectl_options_env_aware: Kubectl.EnvironmentAware, -) -> KubectlBinary: - search_path = kubectl_options_env_aware.executable_search_path - request = BinaryPathRequest( - binary_name="kubectl", - search_path=search_path, - test=BinaryPathTest(args=["version", "--output=json", "--client=true"]), - ) - paths = await Get(BinaryPaths, BinaryPathRequest, request) - logger.debug("kubectl path %s", paths.first_path) - first_path = paths.first_path_or_raise( - request, rationale="interact with the kubernetes cluster" - ) - - return KubectlBinary(first_path.path, first_path.fingerprint) - - -def rules(): - return collect_rules() From 4e5ba6b5080c1c18038f6e1f46da0f979b57c43f Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Mon, 23 Dec 2024 00:27:45 +0100 Subject: [PATCH 13/25] Add notes --- docs/notes/2.25.x.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/notes/2.25.x.md b/docs/notes/2.25.x.md index 912d8553ca3..8d99557dd07 100644 --- a/docs/notes/2.25.x.md +++ b/docs/notes/2.25.x.md @@ -12,6 +12,8 @@ Thank you to [Klayvio](https://www.klaviyo.com/) and [Normal Computing](https:// ### Highlights +New kubernetes backend! See [docs](https://www.pantsbuild.org/stable/docs/kubernetes) for details. + ### Deprecations - **macOS versions**: Pants v2.25 is now built and tested on newer macOS versions: 13 (x86-64, previously 10.15) and macOS 14 (arm64, previously 11). The deprecation of the older versions were announced in Pants 2.23 and 2.24, and are driven by Apple's support schedule; they also help reduce cost for the volunteer-driven Pantsbuild organisation. Using Pants on older versions may or may not work. From f5df27047a3b3961c3f34a3812d2eddef9b17993 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Mon, 23 Dec 2024 00:30:09 +0100 Subject: [PATCH 14/25] pants fix --- build-support/bin/external_tool_versions.py | 16 ++++++++-------- src/python/pants/backend/k8s/goals/deploy.py | 9 ++------- .../pants/backend/k8s/kubectl_subsystem.py | 4 +--- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/build-support/bin/external_tool_versions.py b/build-support/bin/external_tool_versions.py index 2a19def4b5c..d7f5d1c5bd7 100644 --- a/build-support/bin/external_tool_versions.py +++ b/build-support/bin/external_tool_versions.py @@ -1,3 +1,5 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). """Script to fetch external tool versions. Example: @@ -6,16 +8,16 @@ """ import argparse import hashlib -from multiprocessing.pool import ThreadPool +import importlib +import logging import re -from dataclasses import dataclass import xml.etree.ElementTree as ET from collections.abc import Callable, Iterator -import importlib +from dataclasses import dataclass +from multiprocessing.pool import ThreadPool from string import Formatter from urllib.parse import urlparse -import logging import requests logger = logging.getLogger(__name__) @@ -133,7 +135,7 @@ def main(): platforms = args.platforms.split(",") platform_mapping = cls.default_url_platform_mapping - mapped_platforms = set(platform_mapping.get(p) for p in platforms) + mapped_platforms = {platform_mapping.get(p) for p in platforms} domain = urlparse(cls.default_url_template).netloc get_versions = DOMAIN_TO_VERSIONS_MAPPING[domain] @@ -143,9 +145,7 @@ def main(): for platform in mapped_platforms: logger.debug("fetching version: %s %s", version, platform) results.append( - pool.apply_async( - fetch_version, args=(cls.default_url_template, version, platform) - ) + pool.apply_async(fetch_version, args=(cls.default_url_template, version, platform)) ) backward_platform_mapping = {v: k for k, v in platform_mapping.items()} diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index 85c75e2e070..2d7ff682289 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -15,10 +15,7 @@ K8sSourceField, ) from pants.core.goals.deploy import DeployFieldSet, DeployProcess -from pants.core.util_rules.external_tool import ( - DownloadedExternalTool, - ExternalToolRequest, -) +from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest from pants.engine.addresses import UnparsedAddressInputs from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest from pants.engine.fs import MergeDigests, Snapshot @@ -150,9 +147,7 @@ async def kubectl_apply_process( kubectl_tool = await Get( DownloadedExternalTool, ExternalToolRequest, kubectl.get_request(platform) ) - digest = await Get( - Digest, MergeDigests([kubectl_tool.digest, request.input_digest]) - ) + digest = await Get(Digest, MergeDigests([kubectl_tool.digest, request.input_digest])) return Process( argv=argv, diff --git a/src/python/pants/backend/k8s/kubectl_subsystem.py b/src/python/pants/backend/k8s/kubectl_subsystem.py index e9c428a24c7..eaae7fb8f82 100644 --- a/src/python/pants/backend/k8s/kubectl_subsystem.py +++ b/src/python/pants/backend/k8s/kubectl_subsystem.py @@ -172,9 +172,7 @@ class Kubectl(TemplatedExternalTool): advanced=True, ) - class EnvironmentAware( - ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware - ): + class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAware): executable_search_paths_help = softwrap( """ The PATH value that will be used to find kubectl binary. From 4b8a4462c7e13f3f4bc0e3fa7eb0ca6e9d7deb03 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Mon, 23 Dec 2024 11:19:14 +0100 Subject: [PATCH 15/25] Fix mypy --- build-support/bin/external_tool_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-support/bin/external_tool_versions.py b/build-support/bin/external_tool_versions.py index d7f5d1c5bd7..7f6d53d26a6 100644 --- a/build-support/bin/external_tool_versions.py +++ b/build-support/bin/external_tool_versions.py @@ -63,7 +63,7 @@ def _parse_k8s_xml(text: str) -> Iterator[str]: raise RuntimeError("Failed to parse xml, did it change?") key = key_element.text - if regex.match(key): + if key and regex.match(key): yield f"https://cdn.dl.k8s.io/{key}" From 6efda679df5fb52c1db32a304c1d94b50b7b0eba Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 11 Jan 2025 15:38:57 +0100 Subject: [PATCH 16/25] Reuse ExternalToolVersion --- build-support/bin/external_tool_versions.py | 44 ++++++++++++--------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/build-support/bin/external_tool_versions.py b/build-support/bin/external_tool_versions.py index 7f6d53d26a6..9bc45f51308 100644 --- a/build-support/bin/external_tool_versions.py +++ b/build-support/bin/external_tool_versions.py @@ -6,6 +6,7 @@ pants run build-support/bin:external-tool-versions -- --tool pants.backend.k8s.kubectl_subsystem:Kubectl > list.txt """ + import argparse import hashlib import importlib @@ -13,22 +14,15 @@ import re import xml.etree.ElementTree as ET from collections.abc import Callable, Iterator -from dataclasses import dataclass from multiprocessing.pool import ThreadPool from string import Formatter from urllib.parse import urlparse import requests -logger = logging.getLogger(__name__) - +from pants.core.util_rules.external_tool import ExternalToolVersion -@dataclass(frozen=True) -class VersionHash: - version: str - platform: str - size: int - sha256: str +logger = logging.getLogger(__name__) def format_string_to_regex(format_string: str) -> re.Pattern: @@ -80,8 +74,14 @@ def get_k8s_versions(url_template: str, pool: ThreadPool) -> Iterator[str]: } -def fetch_version(url_template: str, version: str, platform: str) -> VersionHash | None: - url = url_template.format(version=version, platform=platform) +def fetch_version( + *, + url_template: str, + version: str, + platform: str, + platform_mapping: dict[str, str], +) -> ExternalToolVersion | None: + url = url_template.format(version=version, platform=platform_mapping[platform]) response = requests.get(url, allow_redirects=True) if response.status_code != 200: logger.error("failed to fetch version: %s\n%s", version, response.text) @@ -89,10 +89,10 @@ def fetch_version(url_template: str, version: str, platform: str) -> VersionHash size = len(response.content) sha256 = hashlib.sha256(response.content) - return VersionHash( + return ExternalToolVersion( version=version, platform=platform, - size=size, + filesize=size, sha256=sha256.hexdigest(), ) @@ -135,20 +135,26 @@ def main(): platforms = args.platforms.split(",") platform_mapping = cls.default_url_platform_mapping - mapped_platforms = {platform_mapping.get(p) for p in platforms} domain = urlparse(cls.default_url_template).netloc get_versions = DOMAIN_TO_VERSIONS_MAPPING[domain] pool = ThreadPool(processes=args.workers) results = [] for version in get_versions(cls.default_url_template, pool): - for platform in mapped_platforms: + for platform in platforms: logger.debug("fetching version: %s %s", version, platform) results.append( - pool.apply_async(fetch_version, args=(cls.default_url_template, version, platform)) + pool.apply_async( + fetch_version, + kwds=dict( + version=version, + platform=platform, + url_template=cls.default_url_template, + platform_mapping=platform_mapping, + ), + ) ) - backward_platform_mapping = {v: k for k, v in platform_mapping.items()} for result in results: v = result.get(60) if v is None: @@ -157,9 +163,9 @@ def main(): "|".join( [ v.version, - backward_platform_mapping[v.platform], + v.platform, v.sha256, - str(v.size), + str(v.filesize), ] ) ) From 6bcb4e4d33063bb27df5799b55cc074a8e57acd3 Mon Sep 17 00:00:00 2001 From: Gregory Borodin Date: Sat, 11 Jan 2025 21:49:03 +0100 Subject: [PATCH 17/25] Update build-support/bin/external_tool_versions.py Co-authored-by: Daniel Goldman --- build-support/bin/external_tool_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-support/bin/external_tool_versions.py b/build-support/bin/external_tool_versions.py index 9bc45f51308..820bfa2faa3 100644 --- a/build-support/bin/external_tool_versions.py +++ b/build-support/bin/external_tool_versions.py @@ -156,7 +156,7 @@ def main(): ) for result in results: - v = result.get(60) + v = result.get(timeout=60) if v is None: continue print( From 9d84c9ed8fe353734d7d91e8a7f588267aa12a1f Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 11 Jan 2025 22:03:00 +0100 Subject: [PATCH 18/25] Sort versions, show progress via tqdm --- 3rdparty/python/requirements.txt | 3 + 3rdparty/python/user_reqs.lock | 239 +++++++++++--------- build-support/bin/external_tool_versions.py | 18 +- 3 files changed, 146 insertions(+), 114 deletions(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index fd06427f0fc..e3054c0f49a 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -60,3 +60,6 @@ python-gnupg==0.4.9 # For validating signatures # Only used for release management PyGithub==2.4.0 + +# Only used in build-support/ +tqdm~=4.67.1 diff --git a/3rdparty/python/user_reqs.lock b/3rdparty/python/user_reqs.lock index f99e3c1ab38..863ff4b2496 100644 --- a/3rdparty/python/user_reqs.lock +++ b/3rdparty/python/user_reqs.lock @@ -35,6 +35,7 @@ // "starlette==0.19.1", // "strawberry-graphql[fastapi]==0.114.0", // "toml==0.10.2", +// "tqdm~=4.67.1", // "types-PyYAML==6.0.3", // "types-freezegun==1.1.6", // "types-requests==2.28.1", @@ -145,13 +146,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2", - "url": "https://files.pythonhosted.org/packages/6a/21/5b6702a7f963e95456c0de2d495f67bf5fd62840ac655dc451586d23d39a/attrs-24.2.0-py3-none-any.whl" + "hash": "ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308", + "url": "https://files.pythonhosted.org/packages/89/aa/ab0f7891a01eeb2d2e338ae8fecbe57fcebea1a24dbb64d45801bfab481d/attrs-24.3.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", - "url": "https://files.pythonhosted.org/packages/fc/0f/aafca9af9315aee06a89ffde799a10a582fe8de76c563ee80bbcdc08b3fb/attrs-24.2.0.tar.gz" + "hash": "8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff", + "url": "https://files.pythonhosted.org/packages/48/c8/6260f8ccc11f0917360fc0da435c5c9c7504e3db174d5a12a1494887b045/attrs-24.3.0.tar.gz" } ], "project_name": "attrs", @@ -167,24 +168,23 @@ "hypothesis; extra == \"cov\"", "hypothesis; extra == \"dev\"", "hypothesis; extra == \"tests\"", - "importlib-metadata; python_version < \"3.8\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"benchmark\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"cov\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"dev\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"tests\"", - "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\") and extra == \"tests-mypy\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"benchmark\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"cov\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"dev\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"tests\"", + "mypy>=1.11.1; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"tests-mypy\"", "myst-parser; extra == \"docs\"", - "pre-commit; extra == \"dev\"", + "pre-commit-uv; extra == \"dev\"", "pympler; extra == \"benchmark\"", "pympler; extra == \"cov\"", "pympler; extra == \"dev\"", "pympler; extra == \"tests\"", "pytest-codspeed; extra == \"benchmark\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"benchmark\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"cov\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"dev\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"tests\"", - "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\") and extra == \"tests-mypy\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"benchmark\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"cov\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"dev\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"tests\"", + "pytest-mypy-plugins; (platform_python_implementation == \"CPython\" and python_version >= \"3.10\") and extra == \"tests-mypy\"", "pytest-xdist[psutil]; extra == \"benchmark\"", "pytest-xdist[psutil]; extra == \"cov\"", "pytest-xdist[psutil]; extra == \"dev\"", @@ -198,8 +198,8 @@ "sphinxcontrib-towncrier; extra == \"docs\"", "towncrier<24.7; extra == \"docs\"" ], - "requires_python": ">=3.7", - "version": "24.2.0" + "requires_python": ">=3.8", + "version": "24.3.0" }, { "artifacts": [ @@ -330,84 +330,74 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", - "url": "https://files.pythonhosted.org/packages/bf/9b/08c0432272d77b04803958a4598a51e2a4b51c06640af8b8f0f908c18bf2/charset_normalizer-3.4.0-py3-none-any.whl" - }, - { - "algorithm": "sha256", - "hash": "8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", - "url": "https://files.pythonhosted.org/packages/13/bc/87c2c9f2c144bedfa62f894c3007cd4530ba4b5351acb10dc786428a50f0/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" + "hash": "d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", + "url": "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5", - "url": "https://files.pythonhosted.org/packages/3b/a0/a68980ab8a1f45a36d9745d35049c1af57d27255eff8c907e3add84cf68f/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", + "url": "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz" }, { "algorithm": "sha256", - "hash": "bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", - "url": "https://files.pythonhosted.org/packages/4c/92/97509850f0d00e9f14a46bc751daabd0ad7765cff29cdfb66c68b6dad57f/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146", + "url": "https://files.pythonhosted.org/packages/28/a3/a42e70d03cbdabc18997baf4f0227c73591a08041c149e710045c281f97b/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl" }, { "algorithm": "sha256", - "hash": "f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365", - "url": "https://files.pythonhosted.org/packages/75/d2/0ab54463d3410709c09266dfb416d032a08f97fd7d60e94b8c6ef54ae14b/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl" + "hash": "234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd", + "url": "https://files.pythonhosted.org/packages/37/ed/be39e5258e198655240db5e19e0b11379163ad7070962d6b0c87ed2c4d39/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl" }, { "algorithm": "sha256", - "hash": "c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", - "url": "https://files.pythonhosted.org/packages/77/d5/8c982d58144de49f59571f940e329ad6e8615e1e82ef84584c5eeb5e1d72/charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl" + "hash": "237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8", + "url": "https://files.pythonhosted.org/packages/3d/7b/82865ba54c765560c8433f65e8acb9217cb839a9e32b42af4aa8e945870f/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl" }, { "algorithm": "sha256", - "hash": "63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", - "url": "https://files.pythonhosted.org/packages/8d/c9/27e41d481557be53d51e60750b85aa40eaf52b841946b3cdeff363105737/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl" + "hash": "c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77", + "url": "https://files.pythonhosted.org/packages/68/85/f4288e96039abdd5aeb5c546fa20a37b50da71b5cf01e75e87f16cd43304/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl" }, { "algorithm": "sha256", - "hash": "0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", - "url": "https://files.pythonhosted.org/packages/9c/61/73589dcc7a719582bf56aae309b6103d2762b526bffe189d635a7fcfd998/charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl" + "hash": "f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3", + "url": "https://files.pythonhosted.org/packages/71/64/d24ab1a997efb06402e3fc07317e94da358e2585165930d9d59ad45fcae2/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" }, { "algorithm": "sha256", - "hash": "6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", - "url": "https://files.pythonhosted.org/packages/bf/19/411a64f01ee971bed3231111b69eb56f9331a769072de479eae7de52296d/charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl" + "hash": "8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125", + "url": "https://files.pythonhosted.org/packages/72/80/41ef5d5a7935d2d3a773e3eaebf0a9350542f2cab4eac59a7a4741fbbbbe/charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl" }, { "algorithm": "sha256", - "hash": "47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", - "url": "https://files.pythonhosted.org/packages/d7/a1/493919799446464ed0299c8eef3c3fad0daf1c3cd48bff9263c731b0d9e2/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl" + "hash": "28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1", + "url": "https://files.pythonhosted.org/packages/7a/28/0b9fefa7b8b080ec492110af6d88aa3dea91c464b17d53474b6e9ba5d2c5/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" }, { "algorithm": "sha256", - "hash": "ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6", - "url": "https://files.pythonhosted.org/packages/e2/29/d227805bff72ed6d6cb1ce08eec707f7cfbd9868044893617eb331f16295/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl" + "hash": "09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd", + "url": "https://files.pythonhosted.org/packages/85/e4/65699e8ab3014ecbe6f5c71d1a55d810fb716bbfd74f6283d5c2aa87febf/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl" }, { "algorithm": "sha256", - "hash": "3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", - "url": "https://files.pythonhosted.org/packages/eb/5b/6f10bad0f6461fa272bfbbdf5d0023b5fb9bc6217c92bf068fa5a99820f5/charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00", + "url": "https://files.pythonhosted.org/packages/88/83/489e9504711fa05d8dde1574996408026bdbdbd938f23be67deebb5eca92/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", - "url": "https://files.pythonhosted.org/packages/ee/44/4f62042ca8cdc0cabf87c0fc00ae27cd8b53ab68be3605ba6d071f742ad3/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl" + "hash": "5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6", + "url": "https://files.pythonhosted.org/packages/b1/82/8e9fe624cc5374193de6860aba3ea8070f584c8565ee77c168ec13274bd2/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl" }, { "algorithm": "sha256", - "hash": "223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", - "url": "https://files.pythonhosted.org/packages/f2/4f/e1808dc01273379acc506d18f1504eb2d299bd4131743b9fc54d7be4df1e/charset_normalizer-3.4.0.tar.gz" - }, - { - "algorithm": "sha256", - "hash": "8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c", - "url": "https://files.pythonhosted.org/packages/fb/9d/9c13753a5a6e0db4a0a6edb1cef7aee39859177b64e1a1e748a6e3ba62c2/charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl" + "hash": "eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12", + "url": "https://files.pythonhosted.org/packages/c6/c7/32da20821cf387b759ad24627a9aca289d2822de929b8a41b6241767b461/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" } ], "project_name": "charset-normalizer", "requires_dists": [], - "requires_python": ">=3.7.0", - "version": "3.4.0" + "requires_python": ">=3.7", + "version": "3.4.1" }, { "artifacts": [ @@ -431,13 +421,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl" + "hash": "63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", + "url": "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", - "url": "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz" + "hash": "ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", + "url": "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz" } ], "project_name": "click", @@ -446,7 +436,7 @@ "importlib-metadata; python_version < \"3.8\"" ], "requires_python": ">=3.7", - "version": "8.1.7" + "version": "8.1.8" }, { "artifacts": [ @@ -1137,43 +1127,43 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "2206a1752d9fac011e95ca83926a269fb0ef5536f7e053966d058316e24d929f", - "url": "https://files.pythonhosted.org/packages/a4/68/99ebf43b6b0321175cff0a05f0ce7fa51a8de67d390ccb8ab0d534be86a9/pydantic-1.10.19-py3-none-any.whl" + "hash": "cd9c1a6a60d54281defb35292d3f2b70bce1b62fe404fd991688fa146715936a", + "url": "https://files.pythonhosted.org/packages/e8/08/ce63c53094a258983667f1ab22f0fb841310f16a07dba6a550e8006bb2e8/pydantic-1.10.20-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "e351df83d1c9cffa53d4e779009a093be70f1d5c6bb7068584086f6a19042526", - "url": "https://files.pythonhosted.org/packages/43/09/c7eb4c39faf7f01ebaed3fae8bf0b31388f2f7ffcefb07b2e5b9ea0f0617/pydantic-1.10.19-cp311-cp311-musllinux_1_1_x86_64.whl" + "hash": "4d6efdd3653713c1e24a46ec966e296faf28933da319692e2adbe5bb6d980418", + "url": "https://files.pythonhosted.org/packages/2a/0a/93d1e3825d8a2e687f1025b03251108df86171831bff4b2ff0db58b83eef/pydantic-1.10.20-cp311-cp311-musllinux_1_2_i686.whl" }, { "algorithm": "sha256", - "hash": "07d00ca5ef0de65dd274005433ce2bb623730271d495a7d190a91c19c5679d34", - "url": "https://files.pythonhosted.org/packages/79/4c/fea1176272425a1b972db48b5b2582165095f22d88d4a249f02439dcd3e5/pydantic-1.10.19-cp311-cp311-macosx_11_0_arm64.whl" + "hash": "bed9ec215fa46fb4024777d4098816e2779b5c0192c3646aa1766c7527031c7d", + "url": "https://files.pythonhosted.org/packages/6c/0e/b5c0d7e7950ff6816ba9e2cabcb8655f34137bedf961780fe2254a23b6db/pydantic-1.10.20-cp311-cp311-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "ad57004e5d73aee36f1e25e4e73a4bc853b473a1c30f652dc8d86b0a987ffce3", - "url": "https://files.pythonhosted.org/packages/85/e5/34b62732fa683d1171be07fb40f0bab3fb35bc52e56bfcae1629aee236c4/pydantic-1.10.19-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "e6fbf8fdc5991560709cb24cebab2e3812b7c8e3eac922a8bf51a531046fb494", + "url": "https://files.pythonhosted.org/packages/7a/f6/e8bba6e8dfa7d17a04463361f8d4442417f15e2fad5da5d219ba9b376d85/pydantic-1.10.20-cp311-cp311-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "fea36c2065b7a1d28c6819cc2e93387b43dd5d3cf5a1e82d8132ee23f36d1f10", - "url": "https://files.pythonhosted.org/packages/a1/2d/df30554721cdad26b241b7a92e726dd1c3716d90c92915731eb00e17a9f7/pydantic-1.10.19.tar.gz" + "hash": "1f6d485ef36dcd597f7d6226bf8cb222b3b5a1a9191261b7ec5677b08e9230bc", + "url": "https://files.pythonhosted.org/packages/8c/13/113f132d8638dda224b06a37cad967c84c2e612d0c07480550a55bbda619/pydantic-1.10.20-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "d7a8a1dd68bac29f08f0a3147de1885f4dccec35d4ea926e6e637fac03cdb4b3", - "url": "https://files.pythonhosted.org/packages/d9/e7/c3276090605233eeda49e3f290ef6e8dc59962f883fa7934455996986d67/pydantic-1.10.19-cp311-cp311-macosx_10_9_x86_64.whl" + "hash": "664110d074bfc14432326d59ec51da0c260050935fe7cdb3579d24b02766dd93", + "url": "https://files.pythonhosted.org/packages/95/51/8c1642cb9c80f84fa05c650395caf19c1fceba335a9960914cda26d33f84/pydantic-1.10.20-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" }, { "algorithm": "sha256", - "hash": "0d32227ea9a3bf537a2273fd2fdb6d64ab4d9b83acd9e4e09310a777baaabb98", - "url": "https://files.pythonhosted.org/packages/f1/72/7cf7dfc8e68098751a5cee8969a967dad2acf9ce460963d071296bdeee81/pydantic-1.10.19-cp311-cp311-musllinux_1_1_i686.whl" + "hash": "8dbb8fb6f0ee469c197047e5e69f51677f0bf6297619c98c814a22965a5486d4", + "url": "https://files.pythonhosted.org/packages/c3/eb/081e5c6bad10570613df9b769bc8553bab771554cbd71371115e38d5e303/pydantic-1.10.20.tar.gz" }, { "algorithm": "sha256", - "hash": "dce355fe7ae53e3090f7f5fa242423c3a7b53260747aa398b4b3aaf8b25f41c3", - "url": "https://files.pythonhosted.org/packages/f5/23/be131d6162cd2c4f7f29cf0a881c0e9bdbf7c37010803f8a85010bf016bf/pydantic-1.10.19-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "604904991eb38e68674892bfea9e9fce6cea62b4d9b9593a2575080842512edb", + "url": "https://files.pythonhosted.org/packages/ec/ab/2193ce40469409f94e23a6eb859809f91a0513c57ada65037af2d74639e4/pydantic-1.10.20-cp311-cp311-musllinux_1_2_x86_64.whl" } ], "project_name": "pydantic", @@ -1183,7 +1173,7 @@ "typing-extensions>=4.2.0" ], "requires_python": ">=3.7", - "version": "1.10.19" + "version": "1.10.20" }, { "artifacts": [ @@ -1227,13 +1217,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", - "url": "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl" + "hash": "9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", + "url": "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199", - "url": "https://files.pythonhosted.org/packages/8e/62/8336eff65bcbc8e4cb5d05b55faf041285951b6e80f33e2bff2024788f31/pygments-2.18.0.tar.gz" + "hash": "61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", + "url": "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz" } ], "project_name": "pygments", @@ -1241,7 +1231,7 @@ "colorama>=0.4.6; extra == \"windows-terminal\"" ], "requires_python": ">=3.8", - "version": "2.18.0" + "version": "2.19.1" }, { "artifacts": [ @@ -1333,13 +1323,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "93d9577b88da0bbea8cc8334ee8b918ed014968fd2ec383e868fb8afb1ccef84", - "url": "https://files.pythonhosted.org/packages/be/ec/2eb3cd785efd67806c46c13a17339708ddc346cbb684eade7a6e6f79536a/pyparsing-3.2.0-py3-none-any.whl" + "hash": "506ff4f4386c4cec0590ec19e6302d3aedb992fdc02c761e90416f158dacf8e1", + "url": "https://files.pythonhosted.org/packages/1c/a7/c8a2d361bf89c0d9577c934ebb7421b25dc84bf3a8e3ac0a40aed9acc547/pyparsing-3.2.1-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "cbf74e27246d595d9a74b186b810f6fbb86726dbf3b9532efb343f6d7294fe9c", - "url": "https://files.pythonhosted.org/packages/8c/d5/e5aeee5387091148a19e1145f63606619cb5f20b83fccb63efae6474e7b2/pyparsing-3.2.0.tar.gz" + "hash": "61980854fd66de3a90028d679a954d5f2623e83144b5afe5ee86f43d762e5f0a", + "url": "https://files.pythonhosted.org/packages/8b/1a/3544f4f299a47911c2ab3710f534e52fea62a633c96806995da5d25be4b2/pyparsing-3.2.1.tar.gz" } ], "project_name": "pyparsing", @@ -1348,7 +1338,7 @@ "railroad-diagrams; extra == \"diagrams\"" ], "requires_python": ">=3.9", - "version": "3.2.0" + "version": "3.2.1" }, { "artifacts": [ @@ -1880,6 +1870,35 @@ "requires_python": ">=3.8", "version": "2.2.1" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", + "url": "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", + "url": "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz" + } + ], + "project_name": "tqdm", + "requires_dists": [ + "colorama; platform_system == \"Windows\"", + "ipywidgets>=6; extra == \"notebook\"", + "nbval; extra == \"dev\"", + "pytest-asyncio>=0.24; extra == \"dev\"", + "pytest-cov; extra == \"dev\"", + "pytest-timeout; extra == \"dev\"", + "pytest>=6; extra == \"dev\"", + "requests; extra == \"discord\"", + "requests; extra == \"telegram\"", + "slack-sdk; extra == \"slack\"" + ], + "requires_python": ">=3.7", + "version": "4.67.1" + }, { "artifacts": [ { @@ -2263,54 +2282,59 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "d2c63b93548eda58abf5188e505ffed0229bf675f7c3090f8e36ad55b8cbc371", - "url": "https://files.pythonhosted.org/packages/4b/d9/a8ba5e9507a9af1917285d118388c5eb7a81834873f45df213a6fe923774/wrapt-1.17.0-py3-none-any.whl" + "hash": "f3117feb1fc479eaf84b549d3f229d5d2abdb823f003bc2a1c6dd70072912fa0", + "url": "https://files.pythonhosted.org/packages/94/47/299f204e352655c117b9dec03fc585866df7eea72660515208ec67c185c4/wrapt-1.17.1-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "74bf625b1b4caaa7bad51d9003f8b07a468a704e0644a700e936c357c17dd45a", - "url": "https://files.pythonhosted.org/packages/0e/40/def56538acddc2f764c157d565b9f989072a1d2f2a8e384324e2e104fc7d/wrapt-1.17.0-cp311-cp311-macosx_11_0_arm64.whl" + "hash": "6fd88935b12b59a933ef45facb57575095f205d30d0ae8dd1a3b485bc4fa2fbd", + "url": "https://files.pythonhosted.org/packages/34/a4/2fbce8654c321c9412caf84e49cfab7666d1a3c87d78da840a63679f64f0/wrapt-1.17.1-cp311-cp311-musllinux_1_2_i686.whl" }, { "algorithm": "sha256", - "hash": "16187aa2317c731170a88ef35e8937ae0f533c402872c1ee5e6d079fcf320801", - "url": "https://files.pythonhosted.org/packages/24/a1/fc03dca9b0432725c2e8cdbf91a349d2194cf03d8523c124faebe581de09/wrapt-1.17.0.tar.gz" + "hash": "53e2986a65eba7c399d7ad1ccd204562d4ffe6e937344fe5a49eb5a83858f797", + "url": "https://files.pythonhosted.org/packages/42/ee/a6bd5e48448239b9acd1bbcc157239f68e801e34e543419f015c050a2773/wrapt-1.17.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "81b1289e99cf4bad07c23393ab447e5e96db0ab50974a280f7954b071d41b489", - "url": "https://files.pythonhosted.org/packages/29/ef/fcdb776b12df5ea7180d065b28fa6bb27ac785dddcd7202a0b6962bbdb47/wrapt-1.17.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + "hash": "ec3e763e7ca8dcba0792fc3e8ff7061186f59e9aafe4438e6bb1f635a6ab0901", + "url": "https://files.pythonhosted.org/packages/65/0d/b2038b8616fc24ddc018a1c5c34b2e7c01e43c7fdfa359a0a60d012ae44b/wrapt-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl" }, { "algorithm": "sha256", - "hash": "4e4b4385363de9052dac1a67bfb535c376f3d19c238b5f36bddc95efae15e12d", - "url": "https://files.pythonhosted.org/packages/42/92/c48ba92cda6f74cb914dc3c5bba9650dc80b790e121c4b987f3a46b028f5/wrapt-1.17.0-cp311-cp311-musllinux_1_2_i686.whl" + "hash": "da0d0c1c4bd55f9ace919454776dbf0821f537b9a77f739f0c3e34b14728b3b3", + "url": "https://files.pythonhosted.org/packages/68/51/868dde1acc33b010068600ee9b92bc9f7fbdf3dcca8fc8341e66efe1eecb/wrapt-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "9f2939cd4a2a52ca32bc0b359015718472d7f6de870760342e7ba295be9ebaf9", - "url": "https://files.pythonhosted.org/packages/55/b5/698bd0bf9fbb3ddb3a2feefbb7ad0dea1205f5d7d05b9cbab54f5db731aa/wrapt-1.17.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "hash": "cd7649f0c493d35f9aad9790bbecd7b6fd2e2f7141f6cb1e1e9bb7a681d6d0a4", + "url": "https://files.pythonhosted.org/packages/6d/7f/7586a6b0fc29b9a145d4ff712e05cca7319e03f11bc8160da2c65efcef80/wrapt-1.17.1-cp311-cp311-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "0f2a28eb35cf99d5f5bd12f5dd44a0f41d206db226535b37b0c60e9da162c3ed", - "url": "https://files.pythonhosted.org/packages/89/e2/8c299f384ae4364193724e2adad99f9504599d02a73ec9199bf3f406549d/wrapt-1.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "hash": "67c30d3fe245adb0eb1061a0e526905970a0dabe7c5fba5078e0ee9d19f28167", + "url": "https://files.pythonhosted.org/packages/bf/32/f78e4939b80cae4ab15d88b365c1c0f942ae1517608480001b08c8626a84/wrapt-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl" }, { "algorithm": "sha256", - "hash": "bdf62d25234290db1837875d4dceb2151e4ea7f9fff2ed41c0fde23ed542eb5b", - "url": "https://files.pythonhosted.org/packages/8a/0a/9276d3269334138b88a2947efaaf6335f61d547698e50dff672ade24f2c6/wrapt-1.17.0-cp311-cp311-musllinux_1_2_x86_64.whl" + "hash": "0aad4f54b3155d673a5c4706a71a0a84f3d415b2fc8a2a399a964d70f18846a2", + "url": "https://files.pythonhosted.org/packages/c3/32/ab1f6ec2c691576614d3f290e5d7c2a435ba38c529186beed17a66f71e9f/wrapt-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" }, { "algorithm": "sha256", - "hash": "6a9653131bda68a1f029c52157fd81e11f07d485df55410401f745007bd6d339", - "url": "https://files.pythonhosted.org/packages/ce/07/701a5cee28cb4d5df030d4b2649319e36f3d9fdd8000ef1d84eb06b9860d/wrapt-1.17.0-cp311-cp311-musllinux_1_2_aarch64.whl" + "hash": "16b2fdfa09a74a3930175b6d9d7d008022aa72a4f02de2b3eecafcc1adfd3cfe", + "url": "https://files.pythonhosted.org/packages/c8/dd/35c573cc2b4b8d65ea96bba0247d05710f284857d30e2266d1874f1c727d/wrapt-1.17.1.tar.gz" + }, + { + "algorithm": "sha256", + "hash": "5ebea3ebb6a394f50f150a52e279508e91c8770625ac8fcb5d8cf35995a320f2", + "url": "https://files.pythonhosted.org/packages/fc/4f/34cefb08717f2ba781da57fab09c53ab9737b9e0df5745167af3180d3955/wrapt-1.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" } ], "project_name": "wrapt", "requires_dists": [], "requires_python": ">=3.8", - "version": "1.17.0" + "version": "1.17.1" } ], "platform_tag": null @@ -2350,6 +2374,7 @@ "starlette==0.19.1", "strawberry-graphql[fastapi]==0.114.0", "toml==0.10.2", + "tqdm~=4.67.1", "types-PyYAML==6.0.3", "types-freezegun==1.1.6", "types-requests==2.28.1", diff --git a/build-support/bin/external_tool_versions.py b/build-support/bin/external_tool_versions.py index 820bfa2faa3..aea8e352a21 100644 --- a/build-support/bin/external_tool_versions.py +++ b/build-support/bin/external_tool_versions.py @@ -19,6 +19,8 @@ from urllib.parse import urlparse import requests +from packaging.version import Version +from tqdm import tqdm from pants.core.util_rules.external_tool import ExternalToolVersion @@ -84,7 +86,7 @@ def fetch_version( url = url_template.format(version=version, platform=platform_mapping[platform]) response = requests.get(url, allow_redirects=True) if response.status_code != 200: - logger.error("failed to fetch version: %s\n%s", version, response.text) + logger.debug("failed to fetch version: %s\n%s", version, response.text) return None size = len(response.content) @@ -139,11 +141,11 @@ def main(): domain = urlparse(cls.default_url_template).netloc get_versions = DOMAIN_TO_VERSIONS_MAPPING[domain] pool = ThreadPool(processes=args.workers) - results = [] + futures = [] for version in get_versions(cls.default_url_template, pool): for platform in platforms: logger.debug("fetching version: %s %s", version, platform) - results.append( + futures.append( pool.apply_async( fetch_version, kwds=dict( @@ -155,10 +157,12 @@ def main(): ) ) - for result in results: - v = result.get(timeout=60) - if v is None: - continue + results: list[ExternalToolVersion] = [ + result for future in tqdm(futures) if (result := future.get(timeout=60)) is not None + ] + results.sort(key=lambda e: Version(e.version)) + + for v in results: print( "|".join( [ From ed0be1f91eaf2436ea8a9ae587faafd1b3675b2e Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 11 Jan 2025 22:06:22 +0100 Subject: [PATCH 19/25] Better error message --- src/python/pants/backend/k8s/goals/deploy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index 2d7ff682289..d36535a3acb 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -67,7 +67,9 @@ async def run_k8s_deploy( ) -> DeployProcess: context = field_set.context.value if context is None: - raise ValueError(f"Missing `{K8sBundleContextField.alias}` field") + raise ValueError( + f"Missing `{K8sBundleContextField.alias}` field on target `{field_set.address.spec}`" + ) context = context if kubectl.pass_context else None if context is not None and context not in k8s_subsystem.available_contexts: From 72722dd3815ebbf0869c913eec9170edbafb4b25 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 11 Jan 2025 22:26:48 +0100 Subject: [PATCH 20/25] Use immutable_input_digests for kubectl tool --- src/python/pants/backend/k8s/goals/deploy.py | 9 +++++++-- src/python/pants/backend/k8s/goals/deploy_test.py | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index d36535a3acb..94cc26a96ef 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -149,14 +149,19 @@ async def kubectl_apply_process( kubectl_tool = await Get( DownloadedExternalTool, ExternalToolRequest, kubectl.get_request(platform) ) - digest = await Get(Digest, MergeDigests([kubectl_tool.digest, request.input_digest])) + + tool_relpath = "__kubectl" + immutable_input_digests = { + tool_relpath: kubectl_tool.digest, + } return Process( argv=argv, - input_digest=digest, + input_digest=request.input_digest, cache_scope=ProcessCacheScope.PER_SESSION, description=f"Applying kubernetes config {request.paths}", env=request.env, + immutable_input_digests=immutable_input_digests, ) diff --git a/src/python/pants/backend/k8s/goals/deploy_test.py b/src/python/pants/backend/k8s/goals/deploy_test.py index f99c76f8273..34a35777de4 100644 --- a/src/python/pants/backend/k8s/goals/deploy_test.py +++ b/src/python/pants/backend/k8s/goals/deploy_test.py @@ -14,6 +14,7 @@ from pants.backend.k8s.kubectl_subsystem import Kubectl from pants.backend.k8s.target_types import K8sBundleTarget, K8sSourcesTargetGenerator from pants.core.goals.deploy import DeployProcess +from pants.core.util_rules import external_tool from pants.engine.addresses import Address from pants.engine.internals.scheduler import ExecutionError from pants.engine.platform import Platform @@ -28,6 +29,7 @@ def rule_runner() -> RuleRunner: K8sBundleTarget, ], rules=[ + *external_tool.rules(), *k8s_deploy_rules(), *k8s_subsystem.rules(), *kubectl_subsystem.rules(), From a50b0992ca4abcd3dd85e43b52f878735d72140f Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Sat, 11 Jan 2025 22:30:37 +0100 Subject: [PATCH 21/25] Add types-tqdm --- 3rdparty/python/requirements.txt | 1 + 3rdparty/python/user_reqs.lock | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index e3054c0f49a..c61f6c36880 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -63,3 +63,4 @@ PyGithub==2.4.0 # Only used in build-support/ tqdm~=4.67.1 +types-tqdm diff --git a/3rdparty/python/user_reqs.lock b/3rdparty/python/user_reqs.lock index 863ff4b2496..c12fff01fcd 100644 --- a/3rdparty/python/user_reqs.lock +++ b/3rdparty/python/user_reqs.lock @@ -41,6 +41,7 @@ // "types-requests==2.28.1", // "types-setuptools==62.6.1", // "types-toml==0.10.8", +// "types-tqdm", // "typing-extensions~=4.12", // "urllib3<2", // "uvicorn[standard]==0.17.6" @@ -1991,6 +1992,26 @@ "requires_python": null, "version": "0.10.8" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "a1f1c9cda5c2d8482d2c73957a5398bfdedda10f6bc7b3b4e812d5c910486d29", + "url": "https://files.pythonhosted.org/packages/e5/e9/9832ef6d017d12b6b4740d7c4dd4eab31867c92ab2c95331d5cb4632fb22/types_tqdm-4.67.0.20241221-py3-none-any.whl" + }, + { + "algorithm": "sha256", + "hash": "e56046631056922385abe89aeb18af5611f471eadd7918a0ad7f34d84cd4c8cc", + "url": "https://files.pythonhosted.org/packages/0d/4f/70760ecc463f3383826666837f28d6e328ff28749fea7fd250f04c861708/types_tqdm-4.67.0.20241221.tar.gz" + } + ], + "project_name": "types-tqdm", + "requires_dists": [ + "types-requests" + ], + "requires_python": ">=3.8", + "version": "4.67.0.20241221" + }, { "artifacts": [ { @@ -2380,6 +2401,7 @@ "types-requests==2.28.1", "types-setuptools==62.6.1", "types-toml==0.10.8", + "types-tqdm", "typing-extensions~=4.12", "urllib3<2", "uvicorn[standard]==0.17.6" From da4a3f790be4e5d77e7b704a07d1932398880070 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Mon, 13 Jan 2025 10:23:00 +0100 Subject: [PATCH 22/25] Update k8s docs --- docs/docs/kubernetes/index.mdx | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/docs/kubernetes/index.mdx b/docs/docs/kubernetes/index.mdx index 8a3fc2a4cae..cd65d98ae50 100644 --- a/docs/docs/kubernetes/index.mdx +++ b/docs/docs/kubernetes/index.mdx @@ -33,20 +33,20 @@ The Kubernetes backend adds [`k8s_source`](../../reference/targets/k8s_source.md files. The `tailor` goal will automatically generate the targets for your .yaml files. -For example, create a file `src/k8s/configmap.yaml`: +For example, create a file `src/k8s/webpages.yaml`: -```yaml title="src/k8s/configmap.yaml" +```yaml title="src/k8s/webpages.yaml" --- apiVersion: v1 kind: ConfigMap metadata: - name: spark-defaults-conf + name: webpages data: - spark-defaults.conf: | - spark.driver.memory=1g - spark.executor.cores=1 - spark.executor.instances=1 - spark.executor.memory=2g + index.html: | + + Hello pants! + Hello pants! + ``` Now run: @@ -94,8 +94,8 @@ Third, create a deployable target `k8s_bundle` in `src/k8s/BUILD`: ```python title="src/k8s/BUILD" k8s_sources() k8s_bundle( - name="configmap", - sources=("src/k8s/configmap.yaml",), + name="webpages", + sources=("src/k8s/webpages.yaml",), context="kind-kind", ) ``` @@ -103,9 +103,9 @@ k8s_bundle( Now you can deploy the target: ```bash -pants experimental-deploy src/k8s:configmap +pants experimental-deploy src/k8s:webpages ``` ``` -✓ src/k8s:configmap deployed to context kind-kind +✓ src/k8s:webpages deployed to context kind-kind ``` From 9081ba1ee0123796f11e1d259e1a4e8fc67181f9 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Mon, 13 Jan 2025 11:49:48 +0100 Subject: [PATCH 23/25] Sort versions, include only >=1.21 --- .../pants/backend/k8s/kubectl_subsystem.py | 112 ++++-------------- 1 file changed, 26 insertions(+), 86 deletions(-) diff --git a/src/python/pants/backend/k8s/kubectl_subsystem.py b/src/python/pants/backend/k8s/kubectl_subsystem.py index eaae7fb8f82..bb2ff76645f 100644 --- a/src/python/pants/backend/k8s/kubectl_subsystem.py +++ b/src/python/pants/backend/k8s/kubectl_subsystem.py @@ -24,118 +24,58 @@ class Kubectl(TemplatedExternalTool): default_version = "1.32.0" default_known_versions = [ - "1.0.7|linux_x86_64|78fa9a5628ae29b091ea0bbab58635fe899fd0c5c01646fb2bf7b6dff6097b70|19387320", - "1.0.7|macos_x86_64|f40f5606ff24e97b9df926febb6f490ba138ddd9cfb8c1d132036408b7181d87|19136448", - "1.1.8|linux_x86_64|b2222986e9f05da8091a16134022d243b3c46a5899486d1b775dbc950ebf36cd|22589688", - "1.1.8|macos_x86_64|da67619ec248a480db20d0704680e0ea91fe0787c8070f342d570c70d957e060|22097696", - "1.10.13|linux_arm64|f52bd7804bec8dadb64af1610bb5c89fd2f2c37a4533c4723df04edb937d0f87|52620142", - "1.10.13|linux_x86_64|0157b02fe9f42a6a5fc738597259d9d22f2d1cb4235d96347e01d1cf7b984980|55100736", - "1.10.13|macos_x86_64|94ec5409ac6715dfec11309795da9b30254b9d818f8b1d38d1ca2e914945868d|54670032", - "1.11.10|linux_arm64|8bbf7e9b666a8c639907e839fad226b46604fd22f8a33a0fc85acda248d571a9|54746905", - "1.11.10|linux_x86_64|fe1a101b476e54515458f98bb0747f2ddfefb69861fe9786a88d7b5ce61e6f45|55461388", - "1.11.10|macos_x86_64|cf27dd3e8d5a13439de2214b7a07e2371cdbeaf151ee4c7d63e7e7c34ebbf117|54985744", - "1.12.10|linux_arm64|c306e470c31227d4de7c3c0cecb06a694501563d1332301d4ac2080422870021|56618322", - "1.12.10|linux_x86_64|b1250b8cadea0e8ad2896f6379abd52bcbbfa2d8ff3253d86d7e705003d83da4|57361431", - "1.12.10|macos_x86_64|67acf17bf77b0a9729d17ad5c058a358c0cbd12dbf46478c37111cdd97a0828c|56860112", - "1.13.12|linux_arm64|47ffe9064318c6a9613f6ac5a5f96ffb43dec6dc4a37ea4b2992bf378c8e6f02|36593184", - "1.13.12|linux_x86_64|3578dbaec9fd043cf2779fbc54afb4297f3e8b50df7493191313bccbb8046300|39271904", - "1.13.12|macos_x86_64|ddbdc7591569f321b8b0a01dcbd679f6b0a7756f1427a51a39eadfce8d34bea7|44353656", - "1.14.10|linux_arm64|7927cfdbf6c793626d0e437ca2c45dd2c1431b6629193ef17f798e81a76b4234|41337728", - "1.14.10|linux_x86_64|7729c6612bec76badc7926a79b26e0d9b06cc312af46dbb80ea7416d1fce0b36|43119424", - "1.14.10|macos_x86_64|43d2c24eafb2ef09a6ac77c2b99070668e83edaa325a16a362e304ba578fdc48|48716328", - "1.15.12|linux_arm64|ef9a4272d556851c645d6788631a2993823260a7e1176a281620284b4c3406da|41228416", - "1.15.12|linux_x86_64|a32b762279c33cb8d8f4198f3facdae402248c3164e9b9b664c3afbd5a27472e|43059232", - "1.15.12|macos_x86_64|1b06cab9ee7988f8e71c48dd1d9379aa7c14123cbbc63e12ab5342c3e9130972|48668112", - "1.16.15|linux_arm64|74719f137dc6d589a3b8a667bcb0f3c57eebd8f050dd2f7ad5b59ceb892a7b99|40697856", - "1.16.15|linux_x86_64|e8913069293156ddf55f243814a22d2384fc18b165efb6200606fdeaad146605|42967040", - "1.16.15|macos_x86_64|aff54bfaaed905813f61a2d0ca039176d6d309e59f92ebdb297c7da1df105485|48907792", - "1.17.17|linux_arm64|6ffc1749adbda24474e67678fcc4a1e704c4e1b9354508965bbab3578bd801ba|41091072", - "1.17.17|linux_x86_64|8329fac94c66bf7a475b630972a8c0b036bab1f28a5584115e8dd26483de8349|43458560", - "1.17.17|macos_x86_64|e76b57bbed823a8572f7ccbf9caae56855048434d818dc13559e89ead5f91578|49542736", - "1.18.20|linux_arm64|31e6bbc657b13ce1b932bf7589bca41a25b0612b4d897b1f363dc9c5a8080a22|41680896", - "1.18.20|linux_x86_64|66a9bb8e9843050340844ca6e72e67632b75b9ebb651559c49db22f35450ed2f|43958272", - "1.18.20|macos_x86_64|bc709378c27da458b31395787a74cd9ac58dce7dbe2a7ba92f8bc2221eeed0be|50095696", - "1.19.16|linux_arm64|6ad55694db34b9ffbc3cb41761a50160eea0a962eb86899410593931b4e602d0|39845888", - "1.19.16|linux_x86_64|6b9d9315877c624097630ac3c9a13f1f7603be39764001da7a080162f85cbc7e|42950656", - "1.19.16|macos_x86_64|7fdbc38bb9d93514cfc20e7770991a9f726836f800778647211a4802959fcf01|49406176", - "1.2.7|linux_x86_64|d5585f95aba909e80d8364523a198d9c70327c3563c4d1b3fa6c9cb66c8d5efc|41025232", - "1.2.7|macos_x86_64|6a9ce64210aea84349ebd9296d7f53d05c734d65e166a6392b96af2588dfa860|40468352", - "1.20.15|linux_arm64|d479febfb2e967bd86240b5c0b841e40e39e1ef610afd6f224281a23318c13dc|37158912", - "1.20.15|linux_x86_64|d283552d3ef3b0fd47c08953414e1e73897a1b3f88c8a520bb2e7de4e37e96f3|40243200", - "1.20.15|macos_x86_64|6b6cf555a34271379b45013dfa9b580329314254aafc91b543bf2d83ebd1db74|46242192", - "1.21.14|linux_arm64|a23151bca5d918e9238546e7af416422b51cda597a22abaae5ca50369abfbbaa|43319296", - "1.21.14|linux_x86_64|0c1682493c2abd7bc5fe4ddcdb0b6e5d417aa7e067994ffeca964163a988c6ee|46686208", "1.21.14|macos_arm64|e0e6e413e19abc9deb15f9bd3c72f73ff5539973758e64ebca0f5eb085de6a00|51872962", "1.21.14|macos_x86_64|30c529fe2891eb93dda99597b5c84cb10d2318bb92ae89e1e6189b3ae5fb6296|52867344", - "1.22.17|linux_arm64|8fc2f8d5c80a6bf60be06f8cf28679a05ce565ce0bc81e70aaac38e0f7da6259|43515904", - "1.22.17|linux_x86_64|7506a0ae7a59b35089853e1da2b0b9ac0258c5309ea3d165c3412904a9051d48|46944256", + "1.21.14|linux_arm64|a23151bca5d918e9238546e7af416422b51cda597a22abaae5ca50369abfbbaa|43319296", + "1.21.14|linux_x86_64|0c1682493c2abd7bc5fe4ddcdb0b6e5d417aa7e067994ffeca964163a988c6ee|46686208", "1.22.17|macos_arm64|b2d881bd6d3c688645cbc9e5b4cf4fe8945e1cfc3f2c07c795d2ee605ce4e568|52098562", "1.22.17|macos_x86_64|c3b8ae5ad48e1e126b5db2e7e22bb1e6ac54901a7f94ce499d12316f705e5e15|53133440", - "1.23.17|linux_arm64|c4a48fdc6038beacbc5de3e4cf6c23639b643e76656aabe2b7798d3898ec7f05|43778048", - "1.23.17|linux_x86_64|f09f7338b5a677f17a9443796c648d2b80feaec9d6a094ab79a77c8a01fde941|45174784", + "1.22.17|linux_arm64|8fc2f8d5c80a6bf60be06f8cf28679a05ce565ce0bc81e70aaac38e0f7da6259|43515904", + "1.22.17|linux_x86_64|7506a0ae7a59b35089853e1da2b0b9ac0258c5309ea3d165c3412904a9051d48|46944256", "1.23.17|macos_arm64|3b4590d67b31e3a94a9633064571c981907555da5376c34960cddfcd552f6114|51181986", "1.23.17|macos_x86_64|7ece6543e3ca2ae9698ef61bbb2a4e249aa21319df4ea1b27c136a9b005dd7d8|51721104", - "1.24.17|linux_arm64|66885bda3a202546778c77f0b66dcf7f576b5a49ff9456acf61329da784a602d|44630016", - "1.24.17|linux_x86_64|3e9588e3326c7110a163103fc3ea101bb0e85f4d6fd228cf928fa9a2a20594d5|46706688", + "1.23.17|linux_arm64|c4a48fdc6038beacbc5de3e4cf6c23639b643e76656aabe2b7798d3898ec7f05|43778048", + "1.23.17|linux_x86_64|f09f7338b5a677f17a9443796c648d2b80feaec9d6a094ab79a77c8a01fde941|45174784", "1.24.17|macos_arm64|7addbe3f1e22a366fa05aed4f268e77e83d902b40a5854e192b4205ed92e5f8d|52955666", "1.24.17|macos_x86_64|1eb904b2c1148ff8431b0bd86677287a48bff000f93fd2d36377fbe956bd1e49|53481056", - "1.25.16|linux_arm64|d6c23c80828092f028476743638a091f2f5e8141273d5228bf06c6671ef46924|43581440", - "1.25.16|linux_x86_64|5a9bc1d3ebfc7f6f812042d5f97b82730f2bdda47634b67bddf36ed23819ab17|45658112", + "1.24.17|linux_arm64|66885bda3a202546778c77f0b66dcf7f576b5a49ff9456acf61329da784a602d|44630016", + "1.24.17|linux_x86_64|3e9588e3326c7110a163103fc3ea101bb0e85f4d6fd228cf928fa9a2a20594d5|46706688", "1.25.16|macos_arm64|d364f73df218b02642d06f3fa9b7345d64c03567b96ca21d361b487f48a33ccc|50416738", "1.25.16|macos_x86_64|34e87fdf0613502edbd2a2b00de5ee8c7789ab10e33257d14423dc6879321920|50954608", - "1.26.15|linux_arm64|1396313f0f8e84ab1879757797992f1af043e1050283532e0fd8469902632216|46661632", - "1.26.15|linux_x86_64|b75f359e6fad3cdbf05a0ee9d5872c43383683bb8527a9e078bb5b8a44350a41|48148480", + "1.25.16|linux_arm64|d6c23c80828092f028476743638a091f2f5e8141273d5228bf06c6671ef46924|43581440", + "1.25.16|linux_x86_64|5a9bc1d3ebfc7f6f812042d5f97b82730f2bdda47634b67bddf36ed23819ab17|45658112", "1.26.15|macos_arm64|c20b920d7e8e3ce3209c7c109fcfc4c09ad599613bc04b72c3f70d9fee598b68|53860082", "1.26.15|macos_x86_64|ad4e980f9c304840ec9227a78a998e132ea23f3ca1bc0df7718ed160341bad0b|54047120", - "1.27.16|linux_arm64|2f50cb29d73f696ffb57437d3e2c95b22c54f019de1dba19e2b834e0b4501eb9|47644824", - "1.27.16|linux_x86_64|97ea7cd771d0c6e3332614668a40d2c5996f0053ff11b44b198ea84dba0818cb|49066136", + "1.26.15|linux_arm64|1396313f0f8e84ab1879757797992f1af043e1050283532e0fd8469902632216|46661632", + "1.26.15|linux_x86_64|b75f359e6fad3cdbf05a0ee9d5872c43383683bb8527a9e078bb5b8a44350a41|48148480", "1.27.16|macos_arm64|d6bc47098bcb13a0ff5c267b30021b499aff4d960bd92610c2b0bc6f6e7246c9|49017698", "1.27.16|macos_x86_64|8d7f339660ba9b33ed56d540bed41b37babc945975a9e7027010697249b9ac5a|50145152", - "1.28.15|linux_arm64|7d45d9620e67095be41403ed80765fe47fcfbf4b4ed0bf0d1c8fe80345bda7d3|48169112", - "1.28.15|linux_x86_64|1f7651ad0b50ef4561aa82e77f3ad06599b5e6b0b2a5fb6c4f474d95a77e41c5|49623192", + "1.27.16|linux_arm64|2f50cb29d73f696ffb57437d3e2c95b22c54f019de1dba19e2b834e0b4501eb9|47644824", + "1.27.16|linux_x86_64|97ea7cd771d0c6e3332614668a40d2c5996f0053ff11b44b198ea84dba0818cb|49066136", "1.28.15|macos_arm64|06a276bdb6da95af148d589f6c983ec8ea10c38f277ced6d97123938c8146078|49593410", "1.28.15|macos_x86_64|3180c84131002037d60fe7322794c20297d0e1b1514eaea20e33f77a00d8f2f4|50716416", - "1.29.12|linux_arm64|1cf2c00bb4f5ee6df69678e95af8ba9a4d4b1050ddefb0ae9d84b5c6f6c0e817|48758936", - "1.29.12|linux_x86_64|35fc028853e6f5299a53f22ab58273ea2d882c0f261ead0a2eed5b844b12dbfb|50225304", + "1.28.15|linux_arm64|7d45d9620e67095be41403ed80765fe47fcfbf4b4ed0bf0d1c8fe80345bda7d3|48169112", + "1.28.15|linux_x86_64|1f7651ad0b50ef4561aa82e77f3ad06599b5e6b0b2a5fb6c4f474d95a77e41c5|49623192", "1.29.12|macos_arm64|5d1c59d8ce4d619bdd78fa849201dbfc9180f6dddcfdb30f29b5bbe20799b897|50169474", "1.29.12|macos_x86_64|0df5932d0ba7a4665ea8033470f2f1a1db21637c3fabc709faa19db0fc62b5ec|51333056", - "1.3.10|linux_arm64|0c35abb5bf70ffa40b02a1c03c914067bf703e37fb0f53392bcce2476df005f0|55720280", - "1.3.10|linux_x86_64|2e72c96b86074dd969b9c49867874a97e8f594fb3e39d3f0ed2ac7add353666d|56525120", - "1.3.10|macos_x86_64|d2f482cae5aefa2fd6afa5b3d8ecb8de8c5a49b22c42c3dce1b5300a05b0109f|55862352", - "1.30.8|linux_arm64|e51d6a76fade0871a9143b64dc62a5ff44f369aa6cb4b04967d93798bf39d15b|49938584", - "1.30.8|linux_x86_64|7f39bdcf768ce4b8c1428894c70c49c8b4d2eee52f3606eb02f5f7d10f66d692|51454104", + "1.29.12|linux_arm64|1cf2c00bb4f5ee6df69678e95af8ba9a4d4b1050ddefb0ae9d84b5c6f6c0e817|48758936", + "1.29.12|linux_x86_64|35fc028853e6f5299a53f22ab58273ea2d882c0f261ead0a2eed5b844b12dbfb|50225304", "1.30.8|macos_arm64|52b11bb032f88e4718cd4e3c8374a6b1fad29772aa1ce701276cc4e17d37642f|51395442", "1.30.8|macos_x86_64|46682e24c3aecfbe92f53b86fb15beb740c43a0fafe0a4e06a1c8bb3ce9e985b|52586352", - "1.31.4|linux_arm64|b97e93c20e3be4b8c8fa1235a41b4d77d4f2022ed3d899230dbbbbd43d26f872|54984856", - "1.31.4|linux_x86_64|298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f|56381592", + "1.30.8|linux_arm64|e51d6a76fade0871a9143b64dc62a5ff44f369aa6cb4b04967d93798bf39d15b|49938584", + "1.30.8|linux_x86_64|7f39bdcf768ce4b8c1428894c70c49c8b4d2eee52f3606eb02f5f7d10f66d692|51454104", "1.31.4|macos_arm64|a756bb911298a85af35c0111c371728a26c532d504fe8b534eb684501fcaf996|56560802", "1.31.4|macos_x86_64|fd996e9f41fd42c6c1c781a5a85990f4d0d8337ede00a7719afa23be886e0abd|57637984", + "1.31.4|linux_arm64|b97e93c20e3be4b8c8fa1235a41b4d77d4f2022ed3d899230dbbbbd43d26f872|54984856", + "1.31.4|linux_x86_64|298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f|56381592", + "1.32.0|macos_arm64|5bfd5de53a054b4ef614c60748e28bf47441c7ed4db47ec3c19a3e2fa0eb5555|57472706", + "1.32.0|macos_x86_64|516585916f499077fac8c2fdd2a382818683f831020277472e6bcf8d1a6f9be4|58612096", "1.32.0|linux_arm64|ba4004f98f3d3a7b7d2954ff0a424caa2c2b06b78c17b1dccf2acc76a311a896|55836824", - "1.32.0|linux_arm64|ba4004f98f3d3a7b7d2954ff0a424caa2c2b06b78c17b1dccf2acc76a311a896|55836824", - "1.32.0|linux_x86_64|646d58f6d98ee670a71d9cdffbf6625aeea2849d567f214bc43a35f8ccb7bf70|57323672", "1.32.0|linux_x86_64|646d58f6d98ee670a71d9cdffbf6625aeea2849d567f214bc43a35f8ccb7bf70|57323672", "1.32.0|macos_arm64|5bfd5de53a054b4ef614c60748e28bf47441c7ed4db47ec3c19a3e2fa0eb5555|57472706", - "1.32.0|macos_arm64|5bfd5de53a054b4ef614c60748e28bf47441c7ed4db47ec3c19a3e2fa0eb5555|57472706", "1.32.0|macos_x86_64|516585916f499077fac8c2fdd2a382818683f831020277472e6bcf8d1a6f9be4|58612096", - "1.32.0|macos_x86_64|516585916f499077fac8c2fdd2a382818683f831020277472e6bcf8d1a6f9be4|58612096", - "1.4.12|linux_arm64|5fc307700d3f2b4682e7a662f251f6dd534f3e12d9a84c20697a73cb6c6a7f22|78167368", - "1.4.12|linux_x86_64|e0376698047be47f37f126fcc4724487dcc8edd2ffb993ae5885779786efb597|79558032", - "1.4.12|macos_x86_64|9c7c5525fe77ebed45dcc949990fcb8998eb6fe0b2441a75c1d58ee7268116d3|63405984", - "1.5.8|linux_arm64|a459fd0e5bd2b002d1423d092c7f1613e095e5485cdda032eaf34303f57adfc3|52137665", - "1.5.8|linux_x86_64|647e233fe0b935300a981b61245b29c7dae6af772dc1f2243cfa1970d2e90219|50372958", - "1.5.8|macos_x86_64|3a4c98ad33892831026a59af6161a2cca0b9928ae098436d88e5224264535e64|50036704", - "1.6.13|linux_arm64|cb891241dbc7e043cafacf6a504a09c2fd4c582798d47235fb5ca64517d2d04b|73688363", - "1.6.13|linux_x86_64|17e29707dcdaac878178d4b137c798cb37993a8a7f0ae214835af4f8e322bafa|70704763", - "1.6.13|macos_x86_64|4623929aaf3037489b2d96561cef4037ad3399f16bdd1469cc5fc9becb4581aa|70232912", - "1.7.16|linux_arm64|123e3f8d0ddfd2b75cb56f76c0b75c56a1960aba73c9a212a153bd425a206b20|70997024", - "1.7.16|linux_x86_64|67e27be929afa1aa103eec0978a2a50ef3df1bd1454b979bb776e472a73c21b2|72497289", - "1.7.16|macos_x86_64|91618ff648ffa9878a64091d7d9440199475855e0fcfae30dab5812c67ea50ac|71985600", - "1.8.15|linux_arm64|ec6d8b93dc74555822dd741eace7e99431d9efc7c3490b4bc46c0bfe24a54b82|52079389", - "1.8.15|linux_x86_64|ac6c59308b91536bc1482c094576bf8685dc5372509a383fb4833ac7299b0e56|53284738", - "1.8.15|macos_x86_64|0a876863dbee07130ead4bd87baf6b5bac7ca63a59d3c0417444a78a145ee3bd|52881856", - "1.9.11|linux_arm64|dd9308db2a76efacff10e9b214f8c04752daa5fa1d1b432c97150bf883e5f091|65530300", - "1.9.11|linux_x86_64|3aa80b62fbd9cfa082aa26ae6a141a6ac209543d31e6f88ad5df47842ed8ddc3|68375438", - "1.9.11|macos_x86_64|b3e46e2a4ba5e29bc43251e9902c8ff6bc21cdbe8c2e20c79efb94bb3d954c02|67840704", + "1.32.0|linux_arm64|ba4004f98f3d3a7b7d2954ff0a424caa2c2b06b78c17b1dccf2acc76a311a896|55836824", + "1.32.0|linux_x86_64|646d58f6d98ee670a71d9cdffbf6625aeea2849d567f214bc43a35f8ccb7bf70|57323672", ] version_constraints = ">=1,<2" From d31a9810668edf03a28e07dbc48c146c3099389c Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Mon, 13 Jan 2025 11:50:41 +0100 Subject: [PATCH 24/25] Fix exe path --- src/python/pants/backend/k8s/goals/deploy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index 94cc26a96ef..783156e75e1 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -136,7 +136,8 @@ async def run_k8s_deploy( async def kubectl_apply_process( request: KubectlApply, platform: Platform, kubectl: Kubectl ) -> Process: - argv: tuple[str, ...] = (kubectl.generate_exe(platform),) + tool_relpath = "__kubectl" + argv: tuple[str, ...] = (f"{tool_relpath}/{kubectl.generate_exe(platform)}",) if request.context is not None: argv += ("--context", request.context) @@ -150,7 +151,6 @@ async def kubectl_apply_process( DownloadedExternalTool, ExternalToolRequest, kubectl.get_request(platform) ) - tool_relpath = "__kubectl" immutable_input_digests = { tool_relpath: kubectl_tool.digest, } From 1b32f7692df971ec49c81791ca910753f08aecd8 Mon Sep 17 00:00:00 2001 From: Borodin Gregory Date: Fri, 24 Jan 2025 22:25:38 +0100 Subject: [PATCH 25/25] Update exe path --- src/python/pants/backend/k8s/goals/deploy.py | 2 +- .../pants/backend/k8s/goals/deploy_test.py | 2 +- .../pants/backend/k8s/kubectl_subsystem.py | 26 ------------------- 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/src/python/pants/backend/k8s/goals/deploy.py b/src/python/pants/backend/k8s/goals/deploy.py index 783156e75e1..72e424e56b3 100644 --- a/src/python/pants/backend/k8s/goals/deploy.py +++ b/src/python/pants/backend/k8s/goals/deploy.py @@ -137,7 +137,7 @@ async def kubectl_apply_process( request: KubectlApply, platform: Platform, kubectl: Kubectl ) -> Process: tool_relpath = "__kubectl" - argv: tuple[str, ...] = (f"{tool_relpath}/{kubectl.generate_exe(platform)}",) + argv: tuple[str, ...] = (f"{tool_relpath}/kubectl",) if request.context is not None: argv += ("--context", request.context) diff --git a/src/python/pants/backend/k8s/goals/deploy_test.py b/src/python/pants/backend/k8s/goals/deploy_test.py index 34a35777de4..c3c5a0919ef 100644 --- a/src/python/pants/backend/k8s/goals/deploy_test.py +++ b/src/python/pants/backend/k8s/goals/deploy_test.py @@ -84,7 +84,7 @@ def test_run_k8s_deploy(rule_runner: RuleRunner) -> None: assert deploy_process.process assert deploy_process.process.process.argv == ( - kubectl.generate_exe(platform), + "__kubectl/kubectl", "--context", "local", "apply", diff --git a/src/python/pants/backend/k8s/kubectl_subsystem.py b/src/python/pants/backend/k8s/kubectl_subsystem.py index bb2ff76645f..a5694b069c1 100644 --- a/src/python/pants/backend/k8s/kubectl_subsystem.py +++ b/src/python/pants/backend/k8s/kubectl_subsystem.py @@ -119,32 +119,6 @@ class EnvironmentAware(ExecutableSearchPathsOptionMixin, Subsystem.EnvironmentAw """ ) - def apply_configs( - self, - paths: Sequence[str], - input_digest: Digest, - platform: Platform, - env: Optional[Mapping[str, str]] = None, - context: Optional[str] = None, - ) -> Process: - argv: tuple[str, ...] = (self.generate_exe(platform),) - - if context is not None: - argv += ("--context", context) - - argv += ("apply", "-o", "yaml") - - for path in paths: - argv += ("-f", path) - - return Process( - argv=argv, - input_digest=input_digest, - cache_scope=ProcessCacheScope.PER_SESSION, - description=f"Applying kubernetes config {paths}", - env=env, - ) - def rules(): return collect_rules()