New kubernetes backend

3rdparty/python/requirements.txt

@@ -60,3 +60,7 @@ python-gnupg==0.4.9 # For validating signatures
 
 # Only used for release management
 PyGithub==2.4.0
+
+# Only used in build-support/
+tqdm~=4.67.1
+types-tqdm

build-support/bin/BUILD

@@ -24,3 +24,12 @@ pex_binary(
     name="terraform_tool_versions",
     entry_point="terraform_tool_versions.py",
 )
+
+pex_binary(
+    name="external-tool-versions",
+    entry_point="external_tool_versions.py",
+    dependencies=[
+        "src/python/pants/backend/k8s/kubectl_subsystem.py",
+    ],
+    execution_mode="venv",
+)

build-support/bin/external_tool_versions.py

@@ -0,0 +1,179 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+"""Script to fetch external tool versions.
+
+Example:
+
+pants run build-support/bin:external-tool-versions -- --tool pants.backend.k8s.kubectl_subsystem:Kubectl > list.txt
+"""
+
+import argparse
+import hashlib
+import importlib
+import logging
+import re
+import xml.etree.ElementTree as ET
+from collections.abc import Callable, Iterator
+from multiprocessing.pool import ThreadPool
+from string import Formatter
+from urllib.parse import urlparse
+
+import requests
+from packaging.version import Version
+from tqdm import tqdm
+
+from pants.core.util_rules.external_tool import ExternalToolVersion
+
+logger = logging.getLogger(__name__)
+
+
+def format_string_to_regex(format_string: str) -> re.Pattern:
+    """Converts a format string to a regex.
+
+    >>> format_string_to_regex("/release/v{version}/bin/{platform}/kubectl")
+    re.compile('^\\/release\\/v(?P<version>.*)\\/bin\\/(?P<platform>.*)\\/kubectl$')
+    """
+    result_regex = ["^"]
+    parts = Formatter().parse(format_string)
+    for literal_text, field_name, format_spec, conversion in parts:
+        escaped_text = literal_text.replace("/", r"\/")
+        result_regex.append(escaped_text)
+        if field_name is not None:
+            result_regex.append(rf"(?P<{field_name}>.*)")
+    result_regex.append("$")
+    return re.compile("".join(result_regex))
+
+
+def fetch_text(url: str) -> str:
+    response = requests.get(url)
+    return response.text
+
+
+def _parse_k8s_xml(text: str) -> Iterator[str]:
+    regex = re.compile(r"release\/stable-(?P<version>[0-9\.]+).txt")
+    root = ET.fromstring(text)
+    tag = "{http://doc.s3.amazonaws.com/2006-03-01}"
+    for item in root.iter(f"{tag}Contents"):
+        key_element = item.find(f"{tag}Key")
+        if key_element is None:
+            raise RuntimeError("Failed to parse xml, did it change?")
+
+        key = key_element.text
+        if key and regex.match(key):
+            yield f"https://cdn.dl.k8s.io/{key}"
+
+
+def get_k8s_versions(url_template: str, pool: ThreadPool) -> Iterator[str]:
+    response = requests.get("https://cdn.dl.k8s.io/", allow_redirects=True)
+    urls = _parse_k8s_xml(response.text)
+    for v in pool.imap_unordered(fetch_text, urls):
+        yield v.strip().lstrip("v")
+
+
+DOMAIN_TO_VERSIONS_MAPPING: dict[str, Callable[[str, ThreadPool], Iterator[str]]] = {
+    # TODO github.com
+    "dl.k8s.io": get_k8s_versions,
+}
+
+
+def fetch_version(
+    *,
+    url_template: str,
+    version: str,
+    platform: str,
+    platform_mapping: dict[str, str],
+) -> ExternalToolVersion | None:
+    url = url_template.format(version=version, platform=platform_mapping[platform])
+    response = requests.get(url, allow_redirects=True)
+    if response.status_code != 200:
+        logger.debug("failed to fetch version: %s\n%s", version, response.text)
+        return None
+
+    size = len(response.content)
+    sha256 = hashlib.sha256(response.content)
+    return ExternalToolVersion(
+        version=version,
+        platform=platform,
+        filesize=size,
+        sha256=sha256.hexdigest(),
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-t",
+        "--tool",
+        help="Python tool location, for example: pants.backend.tools.taplo.subsystem:Taplo",
+        required=True,
+    )
+    parser.add_argument(
+        "--platforms",
+        default="macos_arm64,macos_x86_64,linux_arm64,linux_x86_64",
+        help="Comma separated list of platforms",
+    )
+    parser.add_argument(
+        "-w",
+        "--workers",
+        default=16,
+        help="Thread pool size",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        help="Verbose output",
+    )
+
+    args = parser.parse_args()
+
+    level = logging.DEBUG if args.verbose else logging.INFO
+    logging.basicConfig(level=level, format="%(message)s")
+
+    module_string, class_name = args.tool.split(":")
+    module = importlib.import_module(module_string)
+    cls = getattr(module, class_name)
+
+    platforms = args.platforms.split(",")
+    platform_mapping = cls.default_url_platform_mapping
+
+    domain = urlparse(cls.default_url_template).netloc
+    get_versions = DOMAIN_TO_VERSIONS_MAPPING[domain]
+    pool = ThreadPool(processes=args.workers)
+    futures = []
+    for version in get_versions(cls.default_url_template, pool):
+        for platform in platforms:
+            logger.debug("fetching version: %s %s", version, platform)
+            futures.append(
+                pool.apply_async(
+                    fetch_version,
+                    kwds=dict(
+                        version=version,
+                        platform=platform,
+                        url_template=cls.default_url_template,
+                        platform_mapping=platform_mapping,
+                    ),
+                )
+            )
+
+    results: list[ExternalToolVersion] = [
+        result for future in tqdm(futures) if (result := future.get(timeout=60)) is not None
+    ]
+    results.sort(key=lambda e: Version(e.version))
+
+    for v in results:
+        print(
+            "|".join(
+                [
+                    v.version,
+                    v.platform,
+                    v.sha256,
+                    str(v.filesize),
+                ]
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

docs/docs/docker/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Docker",
-  "position": 9
+  "position": 8
 }

docs/docs/go/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Go",
-  "position": 6
+  "position": 5
 }

docs/docs/jvm/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "JVM",
-  "position": 7
+  "position": 6
 }

docs/docs/kubernetes/_category_.json

@@ -0,0 +1,4 @@
+{
+  "label": "Kubernetes",
+  "position": 9
+}

docs/docs/kubernetes/index.mdx

@@ -0,0 +1,111 @@
+---
+    title: Kubernetes Overview
+    sidebar_position: 999
+---
+
+---
+
+:::caution Kubernetes support is in alpha stage
+Pants is currently building support for Kubernetes. Simple use cases might be
+supported, but many options are missing.
+
+Please share feedback for what you need to use Pants with your Kubernetes queries by
+either [opening a GitHub
+issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our
+Slack](/community/getting-help)!
+:::
+
+## Initial setup
+
+First, activate the relevant backend in `pants.toml`:
+
+```toml title="pants.toml"
+[GLOBAL]
+backend_packages = [
+  ...
+  "pants.backend.experimental.k8s",
+  ...
+]
+```
+
+The Kubernetes backend adds [`k8s_source`](../../reference/targets/k8s_source.mdx) and
+[`k8s_sources`](../../reference/targets/k8s_sources.mdx) target types for Kubernetes object
+files. The `tailor` goal will automatically generate the targets for
+your .yaml files.
+
+For example, create a file `src/k8s/webpages.yaml`:
+
+```yaml title="src/k8s/webpages.yaml"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: webpages
+data:
+  index.html: |
+    <html>
+      <head>Hello pants!</head>
+      <body>Hello pants!</body>
+    </html>
+```
+
+Now run:
+
+```bash
+pants tailor src/k8s:
+```
+```
+Created src/k8s/BUILD:
+  - Add k8s_sources target k8s
+```
+
+## Deploying objects to a cluster
+
+We'll be using a local [kind](https://kind.sigs.k8s.io/) cluster throughout the
+tutorial. First, spin up a cluster:
+
+```bash
+kind create cluster
+```
+```
+Creating cluster "kind" ...
+ ✓ Ensuring node image (kindest/node:v1.25.3) 🖼 
+ ✓ Preparing nodes 📦  
+ ✓ Writing configuration 📜 
+ ✓ Starting control-plane 🕹️ 
+ ✓ Installing CNI 🔌 
+ ✓ Installing StorageClass 💾 
+Set kubectl context to "kind-kind"
+```
+
+Second, configure the list of available contexts in `pants.toml`:
+
+```toml title="pants.toml"
+...
+
+[k8s]
+available_contexts = [
+  "kind-kind",
+]
+```
+
+Third, create a deployable target `k8s_bundle` in `src/k8s/BUILD`:
+
+```python title="src/k8s/BUILD"
+k8s_sources()
+k8s_bundle(
+    name="webpages",
+    sources=("src/k8s/webpages.yaml",),
+    context="kind-kind",
+)
+```
+
+Now you can deploy the target:
+
+```bash
+pants experimental-deploy src/k8s:webpages
+```
+```
+✓ src/k8s:webpages deployed to context kind-kind
+```
+

docs/docs/python/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Python",
-  "position": 5
+  "position": 4
 }

docs/docs/shell/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Shell",
-  "position": 8
+  "position": 7
 }

docs/docs/using-pants/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Using Pants",
-  "position": 4
+  "position": 3
 }

docs/notes/2.25.x.md

@@ -12,6 +12,8 @@ Thank you to [Klayvio](https://www.klaviyo.com/) and [Normal Computing](https://
 
 ### Highlights
 
+New kubernetes backend! See [docs](https://www.pantsbuild.org/stable/docs/kubernetes) for details.
+
 ### Deprecations
 
 - **macOS versions**: Pants v2.25 is now built and tested on newer macOS versions: 13 (x86-64, previously 10.15) and macOS 14 (arm64, previously 11). The deprecation of the older versions were announced in Pants 2.23 and 2.24, and are driven by Apple's support schedule; they also help reduce cost for the volunteer-driven Pantsbuild organisation. Using Pants on older versions may or may not work.

src/python/pants/backend/experimental/k8s/BUILD

@@ -0,0 +1,3 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+python_sources()

src/python/pants/backend/experimental/k8s/register.py

@@ -0,0 +1,18 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from pants.backend.k8s import k8s_subsystem, kubectl_subsystem
+from pants.backend.k8s import target_types as k8s_target_types
+from pants.backend.k8s.goals import deploy, tailor
+
+
+def rules():
+    return [
+        *deploy.rules(),
+        *k8s_subsystem.rules(),
+        *kubectl_subsystem.rules(),
+        *tailor.rules(),
+    ]
+
+
+def target_types():
+    return k8s_target_types.target_types()

src/python/pants/backend/k8s/BUILD

@@ -0,0 +1,3 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+python_sources()

src/python/pants/backend/k8s/goals/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+python_sources()
+python_tests(name="tests")