From 9dff165c6043a3fd2e107d624f04e0869d719fe5 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 10:14:45 +0700 Subject: [PATCH 01/51] Remove cargo.lock from the release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6a92032..468a269 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -82,7 +82,7 @@ jobs: echo 'Updating based on explicit version' cargo set-version ${{ inputs.version }} fi - git add Cargo.toml Cargo.lock + git add Cargo.toml - name: Save the version id: get_version run: echo version="$(cargo metadata --format-version 1 --no-deps | jq --raw-output '.packages[0].version')" >> "${GITHUB_OUTPUT}" From 54174743ff9fbcd39959a4955e5bd7bb9a206309 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 13:59:05 +0700 Subject: [PATCH 02/51] Updated the changelog to make compliant [ci skip] --- CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 222c13b..96958df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.0.1] - 2024-10-22 -### Added +## Unreleased +### Changed - Initial release - -[0.0.1]: https://github.com/isambard-sc/openportal/releases/tag/0.0.1 + This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. From 2fe469d24c72e95aece029ac049d7dd56e763968 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 23 Oct 2024 07:02:39 +0000 Subject: [PATCH 03/51] Release 0.0.1 --- CHANGELOG.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96958df..a374b5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,3 @@ - - # Changelog All notable changes to this project will be documented in this file. @@ -11,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.1] - 2024-10-23 ### Changed - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. + +[0.0.1]: https://github.com/isambard-sc/openportal/releases/tag/0.0.1 From 7379051c93b9f6de277c874d2be83e43f13fb695 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 14:18:55 +0700 Subject: [PATCH 04/51] Fixed incorrect reference to 'master' when I am using 'main' [ci skip] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c77e8c..481f177 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,7 @@ jobs: - name: Get version for Helm id: get_helm_version run: | - if [[ "${{ github.ref_name }}" == "master" ]]; then + if [[ "${{ github.ref_name }}" == "main" ]]; then echo version="${{ steps.get_version.outputs.version }}" >> "${GITHUB_OUTPUT}" else echo version="${{ steps.get_version.outputs.version }}.${{ github.ref_name }}" >> "${GITHUB_OUTPUT}" From 51bb1af66e67ed1f7bf6ffcf7fc28952c7b278e2 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 14:20:52 +0700 Subject: [PATCH 05/51] Updated changelog [ci skip] --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a374b5f..78fbc4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +### Added +- Fixing the helm charts so that they version numbers are correctly set. + ## [0.0.1] - 2024-10-23 ### Changed - Initial release From cc0e1b89aea09c09c92b99f9f56269fc717a5693 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 14:30:22 +0700 Subject: [PATCH 06/51] Added REUSE file so that the changelog gets the right license info [ci skip] --- REUSE.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 REUSE.toml diff --git a/REUSE.toml b/REUSE.toml new file mode 100644 index 0000000..4770d14 --- /dev/null +++ b/REUSE.toml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: CC0-1.0 + +version = 1 + +[[annotations]] +path = "CHANGELOG.md" +SPDX-FileCopyrightText = "© 2024 Christopher Woods " +SPDX-FileCopyrightText = "© 2024 Matt Williams " +SPDX-License-Identifier = "CC-BY-SA-4.0" From a0e9eba8bdc610698dc9b29003947f5992e3611d Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 14:40:26 +0700 Subject: [PATCH 07/51] Fixed reuse bug? [ci skip] --- REUSE.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/REUSE.toml b/REUSE.toml index 4770d14..42c6706 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -6,6 +6,5 @@ version = 1 [[annotations]] path = "CHANGELOG.md" -SPDX-FileCopyrightText = "© 2024 Christopher Woods " -SPDX-FileCopyrightText = "© 2024 Matt Williams " +SPDX-FileCopyrightText = "© 2024 Christopher Woods , Matt Williams " SPDX-License-Identifier = "CC-BY-SA-4.0" From 33d2a60bd6dbe77981e700098ca9b029502de10e Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 23 Oct 2024 07:43:53 +0000 Subject: [PATCH 08/51] Release 0.0.2 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78fbc4f..9ce18aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +## [0.0.2] - 2024-10-23 ### Added - Fixing the helm charts so that they version numbers are correctly set. @@ -14,4 +15,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.2]: https://github.com/isambard-sc/openportal/releases/tag/0.0.2 [0.0.1]: https://github.com/isambard-sc/openportal/releases/tag/0.0.1 From 62c72d28d933fc19090b1f681d0074750769b6ad Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 14:56:04 +0700 Subject: [PATCH 09/51] Fixing SBOM generation [ci skip] --- .github/workflows/release.yml | 50 +++++++++++++++++++++++++++++------ CHANGELOG.md | 3 +++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 468a269..c543aa5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -116,8 +116,8 @@ jobs: attestations: write id-token: write - attest: - name: Attest + attest-portal: + name: Attest Portal needs: build-release runs-on: ubuntu-latest permissions: @@ -133,22 +133,56 @@ jobs: with: tool: cargo-sbom - name: Generate SBOM - run: cargo sbom --output-format=spdx_json_2_3 > sbom.spdx.json + run: cargo sbom --package op-portal --output-format=spdx_json_2_3 > sbom-portal.spdx.json - name: Fetch release artefacts uses: actions/download-artifact@v4 with: - pattern: op-* + pattern: op-portal + merge-multiple: true + - name: Attest SBOM + uses: actions/attest-sbom@v1 + with: + subject-path: op-portal + sbom-path: sbom-portal.spdx.json + - name: Store SBOM + uses: actions/upload-artifact@v4 + with: + name: sbom-portal.spdx.json + path: sbom-portal.spdx.json + + attest-bridge: + name: Attest Bridge + needs: build-release + runs-on: ubuntu-latest + permissions: + contents: read + attestations: write + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install cargo-sbom + uses: taiki-e/install-action@v2 + with: + tool: cargo-sbom + - name: Generate SBOM + run: cargo sbom --package op-bridge --output-format=spdx_json_2_3 > sbom-bridge.spdx.json + - name: Fetch release artefacts + uses: actions/download-artifact@v4 + with: + pattern: op-bridge merge-multiple: true - name: Attest SBOM uses: actions/attest-sbom@v1 with: - subject-path: openportal - sbom-path: sbom.spdx.json + subject-path: op-bridge + sbom-path: sbom-bridge.spdx.json - name: Store SBOM uses: actions/upload-artifact@v4 with: - name: sbom.spdx.json - path: sbom.spdx.json + name: sbom-bridge.spdx.json + path: sbom-bridge.spdx.json make-release: name: Make release ${{ needs.tag-release.outputs.ref }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ce18aa..639f9ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +### Added +- Fixing the attestations so that SBOMs are correctly generated for container images. + ## [0.0.2] - 2024-10-23 ### Added - Fixing the helm charts so that they version numbers are correctly set. From b72bbb984144a0186256086fe63169e34b44f5ac Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 14:57:24 +0700 Subject: [PATCH 10/51] Fixed typo [ci skip] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c543aa5..fde62c5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -186,7 +186,7 @@ jobs: make-release: name: Make release ${{ needs.tag-release.outputs.ref }} - needs: [build-release, tag-release, attest] + needs: [build-release, tag-release, attest-bridge, attest-portal] runs-on: ubuntu-latest permissions: contents: write From cf84d7a3a0f05d5ea36e1efd92d74952eedeec15 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 23 Oct 2024 08:06:36 +0000 Subject: [PATCH 11/51] Release 0.0.3 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 639f9ad..4d541d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +## [0.0.3] - 2024-10-23 ### Added - Fixing the attestations so that SBOMs are correctly generated for container images. @@ -18,5 +19,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.3]: https://github.com/isambard-sc/openportal/releases/tag/0.0.3 [0.0.2]: https://github.com/isambard-sc/openportal/releases/tag/0.0.2 [0.0.1]: https://github.com/isambard-sc/openportal/releases/tag/0.0.1 From 75a64c5150642eb1389da9988abfc599ffa607bd Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 15:21:31 +0700 Subject: [PATCH 12/51] Adding python workflow, plus fixing issues [ci skip] --- .github/workflows/build.yml | 4 +- .github/workflows/python.yml | 95 ++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 3 ++ python/Cargo.toml | 5 ++ 4 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/python.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 481f177..436be8e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,14 +87,14 @@ jobs: uses: actions/attest-build-provenance@v1 id: attest-portal with: - subject-name: ghcr.io/${{ github.repository }} + subject-name: ghcr.io/${{ github.repository_owner }}/op-portal subject-digest: ${{ steps.push-portal-to-ghcr.outputs.digest }} push-to-registry: true - name: Attest op-bridge image uses: actions/attest-build-provenance@v1 id: attest-bridge with: - subject-name: ghcr.io/${{ github.repository }} + subject-name: ghcr.io/${{ github.repository_owner }}/op-bridge subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} push-to-registry: true - name: Push op-portal Helm chart diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..b441b81 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,95 @@ +# This file is autogenerated by maturin v1.7.4 +# To update, run +# +# maturin generate-ci github +# +name: Python Module + +on: + workflow_dispatch: + workflow_call: + inputs: + ref: + type: string + required: true + +permissions: + contents: read + +jobs: + linux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-linux-${{ matrix.platform.target }} + path: dist + + musllinux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + manylinux: musllinux_1_2 + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-musllinux-${{ matrix.platform.target }} + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: [linux, musllinux] + permissions: + # Use to sign the release artifacts + id-token: write + # Used to upload release artifacts + contents: write + # Used to generate artifact attestation + attestations: write + steps: + - uses: actions/download-artifact@v4 + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: 'wheels-*/*' + - name: Publish to PyPI + if: "startsWith(github.ref, 'refs/tags/')" + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + with: + command: upload + args: --non-interactive --skip-existing wheels-*/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d541d9..7e3315e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +### Added +- Fixing release issues, and beginning work on the workflow for the Python module + ## [0.0.3] - 2024-10-23 ### Added - Fixing the attestations so that SBOMs are correctly generated for container images. diff --git a/python/Cargo.toml b/python/Cargo.toml index c7c9ddd..57f02aa 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -32,3 +32,8 @@ tracing = "0.1.40" tracing-subscriber = "0.3.18" url = {version="2.5.2", features=["serde"]} uuid = { version="1.10.0", features=["serde", "v4", "fast-rng", "macro-diagnostics"] } + +[features] +extension-module = ["pyo3/extension-module"] +default = ["extension-module"] + From 28e92f4bf857530bdadc5215aec2032c94db0bd1 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 15:23:50 +0700 Subject: [PATCH 13/51] Adding license info [ci skip] --- .github/workflows/python.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b441b81..e39bf99 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -1,8 +1,7 @@ -# This file is autogenerated by maturin v1.7.4 -# To update, run -# -# maturin generate-ci github -# +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT + name: Python Module on: From a135d85551dd708e411f63f6fa4a76f5d0bd0302 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 23 Oct 2024 08:28:30 +0000 Subject: [PATCH 14/51] Release 0.0.4 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e3315e..39e4dcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +## [0.0.4] - 2024-10-23 ### Added - Fixing release issues, and beginning work on the workflow for the Python module @@ -22,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.4]: https://github.com/isambard-sc/openportal/releases/tag/0.0.4 [0.0.3]: https://github.com/isambard-sc/openportal/releases/tag/0.0.3 [0.0.2]: https://github.com/isambard-sc/openportal/releases/tag/0.0.2 [0.0.1]: https://github.com/isambard-sc/openportal/releases/tag/0.0.1 From 5c6c1f56f94dcca3cddbe80b0dfe68cd4f4077c6 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 15:47:30 +0700 Subject: [PATCH 15/51] Trying again to fix sboms --- .github/workflows/release.yml | 4 ++-- CHANGELOG.md | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fde62c5..0276112 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -133,7 +133,7 @@ jobs: with: tool: cargo-sbom - name: Generate SBOM - run: cargo sbom --package op-portal --output-format=spdx_json_2_3 > sbom-portal.spdx.json + run: cargo sbom --cargo-package op-portal --output-format=spdx_json_2_3 > sbom-portal.spdx.json - name: Fetch release artefacts uses: actions/download-artifact@v4 with: @@ -167,7 +167,7 @@ jobs: with: tool: cargo-sbom - name: Generate SBOM - run: cargo sbom --package op-bridge --output-format=spdx_json_2_3 > sbom-bridge.spdx.json + run: cargo sbom --cargo-package op-bridge --output-format=spdx_json_2_3 > sbom-bridge.spdx.json - name: Fetch release artefacts uses: actions/download-artifact@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 39e4dcb..f6d44ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Fixing generation and attestation of SBOMs for container images (finally!) ## [0.0.4] - 2024-10-23 ### Added From bcdd864940f6b74d7bec700c3f9f8caa3819bb72 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 23 Oct 2024 08:50:37 +0000 Subject: [PATCH 16/51] Release 0.0.5 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6d44ac..e00feb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.5] - 2024-10-23 ### Added - Fixing generation and attestation of SBOMs for container images (finally!) @@ -25,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.5]: https://github.com/isambard-sc/openportal/releases/tag/0.0.5 [0.0.4]: https://github.com/isambard-sc/openportal/releases/tag/0.0.4 [0.0.3]: https://github.com/isambard-sc/openportal/releases/tag/0.0.3 [0.0.2]: https://github.com/isambard-sc/openportal/releases/tag/0.0.2 From 02edd9e7830d80f5709596ec12dfcb187917b8ee Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 19:35:39 +0700 Subject: [PATCH 17/51] Separated out the build artefacts [ci skip] --- .github/workflows/build.yml | 15 ++++++++++++--- CHANGELOG.md | 2 ++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 436be8e..c58ad33 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,16 +101,25 @@ jobs: run: helm push "./op-portal-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - name: Push op-bridge Helm chart run: helm push "./op-bridge-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - - name: Store build artefacts + - name: Store portal artefact uses: actions/upload-artifact@v4 with: - name: openportal-agents + name: op-portal path: | target/release/op-portal + - name: Store bridge artefact + uses: actions/upload-artifact@v4 + with: + name: op-bridge + path: | target/release/op-bridge + - name: Store other agent artefacts + uses: actions/upload-artifact@v4 + with: + name: openportal-agents + path: | target/release/op-provider target/release/op-cluster target/release/op-slurm target/release/op-filesystem target/release/op-freeipa - diff --git a/CHANGELOG.md b/CHANGELOG.md index e00feb2..26fea8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Separated out build artefacts so that they can be picked up by the rest of the build ## [0.0.5] - 2024-10-23 ### Added From 0def0c25f9942f75a0a7dfc58839eb72b31927ab Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 23 Oct 2024 12:39:08 +0000 Subject: [PATCH 18/51] Release 0.0.6 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26fea8c..7f1cca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.6] - 2024-10-23 ### Added - Separated out build artefacts so that they can be picked up by the rest of the build @@ -29,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.6]: https://github.com/isambard-sc/openportal/releases/tag/0.0.6 [0.0.5]: https://github.com/isambard-sc/openportal/releases/tag/0.0.5 [0.0.4]: https://github.com/isambard-sc/openportal/releases/tag/0.0.4 [0.0.3]: https://github.com/isambard-sc/openportal/releases/tag/0.0.3 From ffa628ce5bd9365175ff8ade50b71d9ef27daaed Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Wed, 23 Oct 2024 19:49:03 +0700 Subject: [PATCH 19/51] Fixing python build path [ci skip] --- .github/workflows/python.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index e39bf99..d90a1c2 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -35,11 +35,12 @@ jobs: args: --release --out dist --find-interpreter sccache: 'true' manylinux: auto + working-directory: python - name: Upload wheels uses: actions/upload-artifact@v4 with: name: wheels-linux-${{ matrix.platform.target }} - path: dist + path: python/dist musllinux: runs-on: ${{ matrix.platform.runner }} @@ -60,11 +61,12 @@ jobs: args: --release --out dist --find-interpreter sccache: 'true' manylinux: musllinux_1_2 + working-directory: python - name: Upload wheels uses: actions/upload-artifact@v4 with: name: wheels-musllinux-${{ matrix.platform.target }} - path: dist + path: python/dist release: name: Release From e9ce09bb315390adde2b2c8811565c57f05f1177 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Thu, 24 Oct 2024 11:52:07 +0700 Subject: [PATCH 20/51] Got a working set of helm charts - services install and correctly read config from the associated secret. Now need to fix issues relating to how to process k8s health checks, and eventually how to update service if secret changes [ci skip] --- .gitignore | 1 + helm/bridge/Chart.yaml | 2 +- helm/bridge/templates/configmap.yaml | 14 -------------- helm/bridge/templates/deployment.yaml | 10 ++++------ helm/bridge/templates/service.yaml | 2 +- helm/bridge/values.yaml | 4 +--- helm/portal/Chart.yaml | 2 +- helm/portal/templates/configmap.yaml | 14 -------------- helm/portal/templates/deployment.yaml | 8 +++----- helm/portal/templates/service.yaml | 2 +- helm/portal/values.yaml | 4 +--- 11 files changed, 14 insertions(+), 49 deletions(-) delete mode 100644 helm/bridge/templates/configmap.yaml delete mode 100644 helm/portal/templates/configmap.yaml diff --git a/.gitignore b/.gitignore index 0d6ec47..8e198ed 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ Cargo.lock invitation.toml invite*.toml *_config.toml +*-config.toml example*.toml diff --git a/helm/bridge/Chart.yaml b/helm/bridge/Chart.yaml index f475d01..5e5ffd6 100644 --- a/helm/bridge/Chart.yaml +++ b/helm/bridge/Chart.yaml @@ -5,6 +5,6 @@ apiVersion: v2 name: op-bridge version: "0.0.0" # Set by release script -appVersion: "" # Set by release script +appVersion: "0.0.6" # Set by release script sources: - https://github.com/isambard-sc/openportal/ diff --git a/helm/bridge/templates/configmap.yaml b/helm/bridge/templates/configmap.yaml deleted file mode 100644 index 1f235f2..0000000 --- a/helm/bridge/templates/configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: op-bridge-extra-config - labels: - {{- include "op-bridge.labels" . | indent 4 }} -data: - config-extra.toml: | - secret_key_path = "{{ .Values.secret_key_dir }}/key" - {{ .Values.config | toToml | nindent 4 }} diff --git a/helm/bridge/templates/deployment.yaml b/helm/bridge/templates/deployment.yaml index 4ef10ee..6ae4d24 100644 --- a/helm/bridge/templates/deployment.yaml +++ b/helm/bridge/templates/deployment.yaml @@ -17,11 +17,9 @@ spec: metadata: labels: {{- include "op-bridge.labels" . | indent 8 }} - annotations: - checksum/config: {{ pick (include (print $.Template.BasePath "/configmap.yaml") . | fromYaml) "data" | toString | sha1sum }} # restart if config changed spec: containers: - - name: op-portal + - name: op-bridge image: "{{ print .Values.image.registry "/" }}{{ required "image_name must be set" .Values.image.name }}:{{ default .Chart.AppVersion .Values.image.tag }}" args: ["--config-file=/config/config.toml", "run"] env: @@ -34,8 +32,8 @@ spec: path: /health port: {{ .Values.port }} volumeMounts: - - mountPath: {{ required "config_dir must be set" .Values.config_dir | quote }} - name: "config-volume" + - mountPath: "/config" + name: "config-file-volume" readOnly: true resources: requests: @@ -56,6 +54,6 @@ spec: add: - "NET_BIND_SERVICE" volumes: - - name: "config-volume" + - name: "config-file-volume" secret: secretName: {{ .Values.secret_name | quote }} diff --git a/helm/bridge/templates/service.yaml b/helm/bridge/templates/service.yaml index 937683e..b453f7f 100644 --- a/helm/bridge/templates/service.yaml +++ b/helm/bridge/templates/service.yaml @@ -9,7 +9,7 @@ metadata: spec: ports: - protocol: TCP - port: 80 + port: {{ .Values.port }} targetPort: {{ .Values.port }} selector: app.kubernetes.io/name: op-bridge diff --git a/helm/bridge/values.yaml b/helm/bridge/values.yaml index 9efea24..c9cb7f2 100644 --- a/helm/bridge/values.yaml +++ b/helm/bridge/values.yaml @@ -8,7 +8,5 @@ image: name: op-bridge tag: # defaults to appVersion if not set log_level: info -secret_key_dir: "/secret_key" -config_dir: "/config" secret_name: bridge-config -port: 3000 +port: 8080 diff --git a/helm/portal/Chart.yaml b/helm/portal/Chart.yaml index 91176fa..55f54ac 100644 --- a/helm/portal/Chart.yaml +++ b/helm/portal/Chart.yaml @@ -5,6 +5,6 @@ apiVersion: v2 name: op-portal version: "0.0.0" # Set by release script -appVersion: "" # Set by release script +appVersion: "0.0.6" # Set by release script sources: - https://github.com/isambard-sc/openportal/ diff --git a/helm/portal/templates/configmap.yaml b/helm/portal/templates/configmap.yaml deleted file mode 100644 index 7f2d0b9..0000000 --- a/helm/portal/templates/configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: op-portal-extra-config - labels: - {{- include "op-portal.labels" . | indent 4 }} -data: - config-extra.toml: | - secret_key_path = "{{ .Values.secret_key_dir }}/key" - {{ .Values.config | toToml | nindent 4 }} diff --git a/helm/portal/templates/deployment.yaml b/helm/portal/templates/deployment.yaml index f89bb39..b61da76 100644 --- a/helm/portal/templates/deployment.yaml +++ b/helm/portal/templates/deployment.yaml @@ -17,8 +17,6 @@ spec: metadata: labels: {{- include "op-portal.labels" . | indent 8 }} - annotations: - checksum/config: {{ pick (include (print $.Template.BasePath "/configmap.yaml") . | fromYaml) "data" | toString | sha1sum }} # restart if config changed spec: containers: - name: op-portal @@ -34,8 +32,8 @@ spec: path: /health port: {{ .Values.port }} volumeMounts: - - mountPath: {{ required "config_dir must be set" .Values.config_dir | quote }} - name: "config-volume" + - mountPath: "/config" + name: "config-file-volume" readOnly: true resources: requests: @@ -56,6 +54,6 @@ spec: add: - "NET_BIND_SERVICE" volumes: - - name: "config-volume" + - name: "config-file-volume" secret: secretName: {{ .Values.secret_name | quote }} diff --git a/helm/portal/templates/service.yaml b/helm/portal/templates/service.yaml index f4f5a07..17026fc 100644 --- a/helm/portal/templates/service.yaml +++ b/helm/portal/templates/service.yaml @@ -9,7 +9,7 @@ metadata: spec: ports: - protocol: TCP - port: 80 + port: {{ .Values.port }} targetPort: {{ .Values.port }} selector: app.kubernetes.io/name: op-portal diff --git a/helm/portal/values.yaml b/helm/portal/values.yaml index 3f6f831..3a552fa 100644 --- a/helm/portal/values.yaml +++ b/helm/portal/values.yaml @@ -8,7 +8,5 @@ image: name: op-portal tag: # defaults to appVersion if not set log_level: info -secret_key_dir: "/secret_key" -config_dir: "/config" secret_name: portal-config -port: 3000 +port: 8080 From 99e7fae4953cc2ad96a8418d688d7970b8081d92 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Thu, 24 Oct 2024 14:31:26 +0700 Subject: [PATCH 21/51] Added a healthcheck server, which is optional. Will simplify k8s healthchecks --- bridge/src/main.rs | 1 + cluster/src/main.rs | 1 + docs/cmdline/cluster/src/main.rs | 1 + docs/cmdline/portal/src/main.rs | 1 + docs/echo/src/main.rs | 11 ++- docs/job/src/main.rs | 11 ++- filesystem/src/main.rs | 1 + freeipa/src/main.rs | 1 + paddington/Cargo.toml | 1 + paddington/src/client.rs | 6 ++ paddington/src/config.rs | 44 +++++++++--- paddington/src/eventloop.rs | 3 +- paddington/src/healthcheck.rs | 118 +++++++++++++++++++++++++++++++ paddington/src/lib.rs | 1 + paddington/src/server.rs | 6 ++ portal/src/main.rs | 1 + provider/src/main.rs | 1 + python/Cargo.toml | 4 +- python/build.rs | 6 ++ slurm/src/main.rs | 1 + templemeads/src/agent_bridge.rs | 20 +++++- templemeads/src/agent_core.rs | 20 +++++- templemeads/src/bridge_server.rs | 1 + 23 files changed, 240 insertions(+), 21 deletions(-) create mode 100644 paddington/src/healthcheck.rs create mode 100644 python/build.rs diff --git a/bridge/src/main.rs b/bridge/src/main.rs index bd812b8..f11d6df 100644 --- a/bridge/src/main.rs +++ b/bridge/src/main.rs @@ -39,6 +39,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8044".to_owned()), Some("127.0.0.1".to_owned()), Some(8044), + None, Some("http://localhost:3000".to_owned()), Some("127.0.0.1".to_owned()), Some(3000), diff --git a/cluster/src/main.rs b/cluster/src/main.rs index 5c8fa22..b43a44d 100644 --- a/cluster/src/main.rs +++ b/cluster/src/main.rs @@ -34,6 +34,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8045".to_owned()), Some("127.0.0.1".to_owned()), Some(8045), + None, Some(AgentType::Platform), ); diff --git a/docs/cmdline/cluster/src/main.rs b/docs/cmdline/cluster/src/main.rs index 67d8de8..fd942f0 100644 --- a/docs/cmdline/cluster/src/main.rs +++ b/docs/cmdline/cluster/src/main.rs @@ -24,6 +24,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8091".to_owned()), Some("127.0.0.1".to_owned()), Some(8091), + None, Some(AgentType::Instance), ); diff --git a/docs/cmdline/portal/src/main.rs b/docs/cmdline/portal/src/main.rs index b132d40..91d7b7c 100644 --- a/docs/cmdline/portal/src/main.rs +++ b/docs/cmdline/portal/src/main.rs @@ -20,6 +20,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8090".to_owned()), Some("127.0.0.1".to_owned()), Some(8090), + None, Some(AgentType::Portal), ); diff --git a/docs/echo/src/main.rs b/docs/echo/src/main.rs index 44fba09..84bd08c 100644 --- a/docs/echo/src/main.rs +++ b/docs/echo/src/main.rs @@ -181,8 +181,13 @@ async fn run_client(invitation: &Path) -> Result<(), Error> { // create the echo-client service - note that the url, ip and // port aren't used, as this service won't be listening for any // connecting clients - let mut service: ServiceConfig = - ServiceConfig::new("echo-client", "http://localhost:6502", "127.0.0.1", &6502)?; + let mut service: ServiceConfig = ServiceConfig::new( + "echo-client", + "http://localhost:6502", + "127.0.0.1", + &6502, + &None, + )?; // now give the invitation to connect to the server to the client service.add_server(invite)?; @@ -257,7 +262,7 @@ async fn run_server( invitation: &Path, ) -> Result<(), Error> { // create the echo-server service - let mut service = ServiceConfig::new("echo-server", url, ip, port)?; + let mut service = ServiceConfig::new("echo-server", url, ip, port, &None)?; let invite = service.add_client("echo-client", range)?; diff --git a/docs/job/src/main.rs b/docs/job/src/main.rs index 7e26b8e..f303a6d 100644 --- a/docs/job/src/main.rs +++ b/docs/job/src/main.rs @@ -199,8 +199,13 @@ async fn run_cluster(invitation: &Path) -> Result<(), Error> { // create the paddington service for the cluster agent // - note that the url, ip and port aren't used, as this // agent won't be listening for any connecting clients - let mut service: ServiceConfig = - ServiceConfig::new("cluster", "http://localhost:6502", "127.0.0.1", &6502)?; + let mut service: ServiceConfig = ServiceConfig::new( + "cluster", + "http://localhost:6502", + "127.0.0.1", + &6502, + &None, + )?; // now give the invitation to connect to the server to the client service.add_server(invite)?; @@ -255,7 +260,7 @@ async fn run_portal( invitation: &Path, ) -> Result<(), Error> { // create a paddington service configuration for the portal agent - let mut service = ServiceConfig::new("portal", url, ip, port)?; + let mut service = ServiceConfig::new("portal", url, ip, port, &None)?; // add the cluster to the portal, returning an invitation let invite = service.add_client("cluster", range)?; diff --git a/filesystem/src/main.rs b/filesystem/src/main.rs index 0d35da0..36cd475 100644 --- a/filesystem/src/main.rs +++ b/filesystem/src/main.rs @@ -39,6 +39,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8047".to_owned()), Some("127.0.0.1".to_owned()), Some(8047), + None, Some(AgentType::Filesystem), ); diff --git a/freeipa/src/main.rs b/freeipa/src/main.rs index 213f765..f4f7ccc 100644 --- a/freeipa/src/main.rs +++ b/freeipa/src/main.rs @@ -44,6 +44,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8046".to_owned()), Some("127.0.0.1".to_owned()), Some(8046), + None, Some(AgentType::Account), ); diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index d50232e..c6b969c 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -17,6 +17,7 @@ built = { version = "0.7", default-features = false, features = ["git2"] } [dependencies] anyhow = { version="1.0.86", features = ["backtrace"] } +axum = { version = "0.7", features = ["tracing", "query"] } dirs = "5.0.1" futures = "0.3.30" futures-channel = "0.3.30" diff --git a/paddington/src/client.rs b/paddington/src/client.rs index 0b5ef28..3aa2886 100644 --- a/paddington/src/client.rs +++ b/paddington/src/client.rs @@ -5,6 +5,7 @@ use crate::config::{PeerConfig, ServiceConfig}; use crate::connection::Connection; use crate::error::Error; use crate::exchange; +use crate::healthcheck; pub async fn run_once(config: ServiceConfig, peer: PeerConfig) -> Result<(), Error> { let service_name = config.name(); @@ -44,6 +45,11 @@ pub async fn run(config: ServiceConfig, peer: PeerConfig) -> Result<(), Error> { // set the name of the service in the exchange exchange::set_name(&config.name()).await?; + if let Some(healthcheck_port) = config.healthcheck_port() { + // spawn the health check server + healthcheck::spawn(config.ip(), healthcheck_port).await?; + } + loop { match run_once(config.clone(), peer.clone()).await { Ok(_) => { diff --git a/paddington/src/config.rs b/paddington/src/config.rs index c29fc1d..d6f812b 100644 --- a/paddington/src/config.rs +++ b/paddington/src/config.rs @@ -73,6 +73,7 @@ pub struct Defaults { url: String, ip: String, port: u16, + healthcheck_port: Option, } impl Defaults { @@ -82,6 +83,7 @@ impl Defaults { url: Option, ip: Option, port: Option, + healthcheck_port: Option, ) -> Self { let config_file = config_file.unwrap_or( dirs::config_local_dir() @@ -99,6 +101,7 @@ impl Defaults { url: url.unwrap_or("http://localhost:8000".to_owned()), ip: ip.unwrap_or("127.0.0.1".to_owned()), port: port.unwrap_or(8042), + healthcheck_port, } } @@ -121,6 +124,10 @@ impl Defaults { pub fn port(&self) -> u16 { self.port } + + pub fn healthcheck_port(&self) -> Option { + self.healthcheck_port + } } #[derive(Serialize, Deserialize, Clone, Debug)] @@ -412,6 +419,7 @@ pub struct ServiceConfig { url: String, ip: IpAddr, port: u16, + heathcheck_port: Option, servers: Vec, clients: Vec, @@ -419,7 +427,13 @@ pub struct ServiceConfig { } impl ServiceConfig { - pub fn new(name: &str, url: &str, ip: &str, port: &u16) -> Result { + pub fn new( + name: &str, + url: &str, + ip: &str, + port: &u16, + healthcheck_port: &Option, + ) -> Result { Ok(ServiceConfig { name: name.to_string(), url: create_websocket_url(url)?, @@ -427,6 +441,7 @@ impl ServiceConfig { .parse() .with_context(|| format!("Could not parse IP address: {}", ip))?, port: *port, + heathcheck_port: *healthcheck_port, servers: Vec::new(), clients: Vec::new(), encryption: None, @@ -493,6 +508,10 @@ impl ServiceConfig { self.port } + pub fn healthcheck_port(&self) -> Option { + self.heathcheck_port + } + pub fn add_client(&mut self, name: &str, ip: &str) -> Result { let ip = IpOrRange::new(ip) .with_context(|| format!("Could not parse into an IP address or IP range: {}", ip))?; @@ -573,6 +592,7 @@ impl ServiceConfig { url: String, ip: IpAddr, port: u16, + healthcheck_port: &Option, ) -> Result { // see if this config_dir exists - return an error if it does let config_file = path::absolute(config_file).with_context(|| { @@ -586,7 +606,7 @@ impl ServiceConfig { return Err(Error::NotExists(config_file.to_string_lossy().to_string())); } - let config = ServiceConfig::new(&name, &url, &ip.to_string(), &port)?; + let config = ServiceConfig::new(&name, &url, &ip.to_string(), &port, healthcheck_port)?; save::(config.clone(), &config_file)?; // check we can read the config and return it @@ -649,15 +669,17 @@ mod tests { #[test] fn test_invitations() { - let mut primary = ServiceConfig::new("primary", "http://localhost", "127.0.0.1", &5544) - .unwrap_or_else(|e| { - unreachable!("Cannot create service config: {}", e); - }); - - let mut secondary = ServiceConfig::new("secondary", "http://localhost", "127.0.0.1", &5545) - .unwrap_or_else(|e| { - unreachable!("Cannot create service config: {}", e); - }); + let mut primary = + ServiceConfig::new("primary", "http://localhost", "127.0.0.1", &5544, &None) + .unwrap_or_else(|e| { + unreachable!("Cannot create service config: {}", e); + }); + + let mut secondary = + ServiceConfig::new("secondary", "http://localhost", "127.0.0.1", &5545, &None) + .unwrap_or_else(|e| { + unreachable!("Cannot create service config: {}", e); + }); // introduce the secondary to the primary let invite = primary diff --git a/paddington/src/eventloop.rs b/paddington/src/eventloop.rs index 943f6fe..e6b4b04 100644 --- a/paddington/src/eventloop.rs +++ b/paddington/src/eventloop.rs @@ -57,7 +57,8 @@ mod tests { async fn test_run() -> Result<()> { // this tests that the service can be configured and will run // (it will exit immediately as there are no clients or servers) - let config = ServiceConfig::new("test_server", "http://localhost", "127.0.0.1", &5544)?; + let config = + ServiceConfig::new("test_server", "http://localhost", "127.0.0.1", &5544, &None)?; run(config).await?; Ok(()) diff --git a/paddington/src/healthcheck.rs b/paddington/src/healthcheck.rs new file mode 100644 index 0000000..6438327 --- /dev/null +++ b/paddington/src/healthcheck.rs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: © 2024 Christopher Woods +// SPDX-License-Identifier: MIT + +use crate::Error; + +use anyhow::Result; +use axum::{ + extract::Json, + http::StatusCode, + response::{IntoResponse, Response}, + routing::get, + Router, +}; +use once_cell::sync::Lazy; +use serde_json::json; +use std::net::IpAddr; +use std::sync::RwLock; +use tokio::net::TcpListener; + +// +// Health check endpoint for the web API +// +#[tracing::instrument(skip_all)] +async fn health() -> Result, AppError> { + Ok(Json(json!({"status": "ok"}))) +} + +/// +/// Function spawned to run the API server in a background thread +/// +async fn run_server(app: Router, listener: TcpListener) -> Result<()> { + match axum::serve(listener, app).await { + Ok(_) => { + tracing::info!("Server ran successfully"); + } + Err(e) => { + tracing::error!("Error starting server: {}", e); + } + } + + Ok(()) +} + +static IS_RUNNING: Lazy> = Lazy::new(|| RwLock::new(false)); + +/// +/// Spawn a small http server that responds to health checks +/// +pub async fn spawn(ip: IpAddr, port: u16) -> Result<(), Error> { + // check if the server is already running + match IS_RUNNING.read() { + Ok(guard) => { + if *guard { + // already running + return Ok(()); + } + } + Err(e) => { + // not running? + tracing::error!("Error getting read lock: {}", e); + return Ok(()); + } + } + + // set the flag to indicate the server is running + match IS_RUNNING.write() { + Ok(mut guard) => { + if *guard { + // someone else set it first + return Ok(()); + } + + *guard = true; + } + Err(e) => { + // not running? + tracing::error!("Error getting write lock: {}", e); + return Ok(()); + } + } + + tracing::info!("Starting health check server on {}:{}/health", ip, port); + + // create the web API + let app = Router::new().route("/health", get(health)); + + // create a TCP listener on the specified port + let listener = tokio::net::TcpListener::bind(&std::net::SocketAddr::new(ip, port)).await?; + + // spawn a new task to run the web server to listen for requests + tokio::spawn(run_server(app, listener)); + + Ok(()) +} + +// Errors + +#[derive(Debug)] +struct AppError(anyhow::Error, Option); + +impl IntoResponse for AppError { + fn into_response(self) -> Response { + ( + self.1.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + Json(json!({"message":format!("Something went wrong: {:?}", self.0)})), + ) + .into_response() + } +} + +impl From for AppError +where + E: Into, +{ + fn from(err: E) -> Self { + Self(err.into(), None) + } +} diff --git a/paddington/src/lib.rs b/paddington/src/lib.rs index e7a0466..c3f6ad8 100644 --- a/paddington/src/lib.rs +++ b/paddington/src/lib.rs @@ -8,6 +8,7 @@ mod crypto; mod error; mod eventloop; mod exchange; +mod healthcheck; mod server; // public API diff --git a/paddington/src/server.rs b/paddington/src/server.rs index 4f99e88..e964747 100644 --- a/paddington/src/server.rs +++ b/paddington/src/server.rs @@ -7,6 +7,7 @@ use crate::config::ServiceConfig; use crate::connection::Connection; use crate::error::Error; use crate::exchange; +use crate::healthcheck; /// /// Internal function used to handle a single connection to the server. @@ -74,6 +75,11 @@ pub async fn run(config: ServiceConfig) -> Result<(), Error> { // set the name of the service in the exchange exchange::set_name(&config.name()).await?; + // spawn the healthcheck server if enabled + if let Some(healthcheck_port) = config.healthcheck_port() { + healthcheck::spawn(config.ip(), healthcheck_port).await?; + } + loop { let result = run_once(config.clone()).await; diff --git a/portal/src/main.rs b/portal/src/main.rs index a1c4549..b5e2635 100644 --- a/portal/src/main.rs +++ b/portal/src/main.rs @@ -40,6 +40,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8040".to_owned()), Some("127.0.0.1".to_owned()), Some(8040), + None, Some(AgentType::Portal), ); diff --git a/provider/src/main.rs b/provider/src/main.rs index 5d8922d..f6759de 100644 --- a/provider/src/main.rs +++ b/provider/src/main.rs @@ -40,6 +40,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8041".to_owned()), Some("127.0.0.1".to_owned()), Some(8041), + None, Some(AgentType::Provider), ); diff --git a/python/Cargo.toml b/python/Cargo.toml index 57f02aa..44fd8d9 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -33,7 +33,9 @@ tracing-subscriber = "0.3.18" url = {version="2.5.2", features=["serde"]} uuid = { version="1.10.0", features=["serde", "v4", "fast-rng", "macro-diagnostics"] } +[build-dependencies] +pyo3-build-config = "0.22.5" + [features] extension-module = ["pyo3/extension-module"] default = ["extension-module"] - diff --git a/python/build.rs b/python/build.rs new file mode 100644 index 0000000..5de5ce2 --- /dev/null +++ b/python/build.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: © 2024 Christopher Woods +// SPDX-License-Identifier: MIT + +fn main() { + pyo3_build_config::add_extension_module_link_args(); +} diff --git a/slurm/src/main.rs b/slurm/src/main.rs index 3efddbf..fa23d42 100644 --- a/slurm/src/main.rs +++ b/slurm/src/main.rs @@ -40,6 +40,7 @@ async fn main() -> Result<()> { Some("ws://localhost:8046".to_owned()), Some("127.0.0.1".to_owned()), Some(8046), + None, Some(AgentType::Instance), ); diff --git a/templemeads/src/agent_bridge.rs b/templemeads/src/agent_bridge.rs index d67ee90..9830cb2 100644 --- a/templemeads/src/agent_bridge.rs +++ b/templemeads/src/agent_bridge.rs @@ -72,12 +72,13 @@ impl Defaults { url: Option, ip: Option, port: Option, + healthcheck_port: Option, bridge_url: Option, bridge_ip: Option, bridge_port: Option, ) -> Self { Self { - service: ServiceDefaults::parse(name, config_file, url, ip, port), + service: ServiceDefaults::parse(name, config_file, url, ip, port, healthcheck_port), bridge: BridgeDefaults::parse(bridge_url, bridge_ip, bridge_port), } } @@ -117,8 +118,17 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> port, bridge_ip, bridge_port, + healthcheck_port, force, }) => { + let local_healthcheck_port; + + if let Some(healthcheck_port) = healthcheck_port { + local_healthcheck_port = Some(*healthcheck_port); + } else { + local_healthcheck_port = defaults.service.healthcheck_port(); + } + let config = Config { service: { ServiceConfig::new( @@ -129,6 +139,7 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> .parse::()? .to_string(), &port.unwrap_or_else(|| defaults.service.port()), + &local_healthcheck_port, )? }, bridge: BridgeConfig::new( @@ -374,6 +385,13 @@ enum Commands { )] bridge_port: Option, + #[arg( + long, + short = 'k', + help = "Optional port on which to listen for health checks (e.g. 3001)" + )] + healthcheck_port: Option, + #[arg(long, short = 'f', help = "Force reinitialisation")] force: bool, }, diff --git a/templemeads/src/agent_core.rs b/templemeads/src/agent_core.rs index d628861..4a4facd 100644 --- a/templemeads/src/agent_core.rs +++ b/templemeads/src/agent_core.rs @@ -80,10 +80,11 @@ impl Defaults { url: Option, ip: Option, port: Option, + healthcheck_port: Option, agent: Option, ) -> Self { Self { - service: ServiceDefaults::parse(name, config_file, url, ip, port), + service: ServiceDefaults::parse(name, config_file, url, ip, port, healthcheck_port), agent: agent.unwrap_or(AgentType::Portal), extras: HashMap::new(), } @@ -130,8 +131,17 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> url, ip, port, + healthcheck_port, force, }) => { + let local_healthcheck_port; + + if let Some(healthcheck_port) = healthcheck_port { + local_healthcheck_port = Some(*healthcheck_port); + } else { + local_healthcheck_port = defaults.service.healthcheck_port(); + } + let config = Config { service: { ServiceConfig::new( @@ -142,6 +152,7 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> .parse::()? .to_string(), &port.unwrap_or_else(|| defaults.service.port()), + &local_healthcheck_port, )? }, agent: defaults.agent.clone(), @@ -375,6 +386,13 @@ enum Commands { )] port: Option, + #[arg( + long, + short = 'k', + help = "Optional port on which to listen for health checks (e.g. 8080)" + )] + healthcheck_port: Option, + #[arg(long, short = 'f', help = "Force reinitialisation")] force: bool, }, diff --git a/templemeads/src/bridge_server.rs b/templemeads/src/bridge_server.rs index c5feb17..c5a361e 100644 --- a/templemeads/src/bridge_server.rs +++ b/templemeads/src/bridge_server.rs @@ -365,6 +365,7 @@ pub async fn spawn(config: Config) -> Result<(), Error> { Ok(()) } + // Errors #[derive(Debug)] From 95f5e0bc2f99df6e78bc291d31ea1b4c5660b0e7 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Thu, 24 Oct 2024 14:39:19 +0700 Subject: [PATCH 22/51] Helm updates [ci skip] --- CHANGELOG.md | 3 +++ helm/bridge/templates/deployment.yaml | 2 +- helm/bridge/templates/service.yaml | 6 ++++++ helm/bridge/values.yaml | 4 +++- helm/portal/templates/deployment.yaml | 2 +- helm/portal/templates/service.yaml | 3 +++ helm/portal/values.yaml | 3 ++- 7 files changed, 19 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f1cca7..7e54d7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Added an healthcheck server to simplify pod healthchecks +- Updated helm charts to use the healthcheck server, plus expose the bridge server port ## [0.0.6] - 2024-10-23 ### Added diff --git a/helm/bridge/templates/deployment.yaml b/helm/bridge/templates/deployment.yaml index 6ae4d24..33eb6eb 100644 --- a/helm/bridge/templates/deployment.yaml +++ b/helm/bridge/templates/deployment.yaml @@ -30,7 +30,7 @@ spec: readinessProbe: httpGet: path: /health - port: {{ .Values.port }} + port: {{ .Values.health_port }} volumeMounts: - mountPath: "/config" name: "config-file-volume" diff --git a/helm/bridge/templates/service.yaml b/helm/bridge/templates/service.yaml index b453f7f..886cf63 100644 --- a/helm/bridge/templates/service.yaml +++ b/helm/bridge/templates/service.yaml @@ -11,5 +11,11 @@ spec: - protocol: TCP port: {{ .Values.port }} targetPort: {{ .Values.port }} + - protocol: TCP + port: {{ .Values.bridge_port }} + targetPort: {{ .Values.bridge_port }} + - protocol: TCP + port: {{ .Values.health_port }} + targetPort: {{ .Values.health_port }} selector: app.kubernetes.io/name: op-bridge diff --git a/helm/bridge/values.yaml b/helm/bridge/values.yaml index c9cb7f2..caabc84 100644 --- a/helm/bridge/values.yaml +++ b/helm/bridge/values.yaml @@ -9,4 +9,6 @@ image: tag: # defaults to appVersion if not set log_level: info secret_name: bridge-config -port: 8080 +port: 80 +bridge_port: 8800 +health_port: 8080 diff --git a/helm/portal/templates/deployment.yaml b/helm/portal/templates/deployment.yaml index b61da76..be3270e 100644 --- a/helm/portal/templates/deployment.yaml +++ b/helm/portal/templates/deployment.yaml @@ -30,7 +30,7 @@ spec: readinessProbe: httpGet: path: /health - port: {{ .Values.port }} + port: {{ .Values.health_port }} volumeMounts: - mountPath: "/config" name: "config-file-volume" diff --git a/helm/portal/templates/service.yaml b/helm/portal/templates/service.yaml index 17026fc..2fffdc2 100644 --- a/helm/portal/templates/service.yaml +++ b/helm/portal/templates/service.yaml @@ -11,5 +11,8 @@ spec: - protocol: TCP port: {{ .Values.port }} targetPort: {{ .Values.port }} + - protocol: TCP + port: {{ .Values.health_port }} + targetPort: {{ .Values.health_port }} selector: app.kubernetes.io/name: op-portal diff --git a/helm/portal/values.yaml b/helm/portal/values.yaml index 3a552fa..42d95be 100644 --- a/helm/portal/values.yaml +++ b/helm/portal/values.yaml @@ -9,4 +9,5 @@ image: tag: # defaults to appVersion if not set log_level: info secret_name: portal-config -port: 8080 +port: 80 +health_port: 8080 From 564cd4f424f885e661c69c2baad00339c2db7ff6 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 24 Oct 2024 07:42:19 +0000 Subject: [PATCH 23/51] Release 0.0.7 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e54d7a..f8b7522 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.7] - 2024-10-24 ### Added - Added an healthcheck server to simplify pod healthchecks - Updated helm charts to use the healthcheck server, plus expose the bridge server port @@ -34,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.7]: https://github.com/isambard-sc/openportal/releases/tag/0.0.7 [0.0.6]: https://github.com/isambard-sc/openportal/releases/tag/0.0.6 [0.0.5]: https://github.com/isambard-sc/openportal/releases/tag/0.0.5 [0.0.4]: https://github.com/isambard-sc/openportal/releases/tag/0.0.4 From d30cf9bcb03d46445c48db3813a5057b4e3b9a91 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Thu, 24 Oct 2024 16:34:56 +0700 Subject: [PATCH 24/51] Added port names in the helm chart [ci skip] --- CHANGELOG.md | 4 +++- helm/bridge/templates/service.yaml | 3 +++ helm/portal/templates/service.yaml | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8b7522..8a1fe12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Added names for the ports in the helm charts ## [0.0.7] - 2024-10-24 ### Added -- Added an healthcheck server to simplify pod healthchecks +- Added a healthcheck server to simplify pod healthchecks - Updated helm charts to use the healthcheck server, plus expose the bridge server port ## [0.0.6] - 2024-10-23 diff --git a/helm/bridge/templates/service.yaml b/helm/bridge/templates/service.yaml index 886cf63..90e0d7c 100644 --- a/helm/bridge/templates/service.yaml +++ b/helm/bridge/templates/service.yaml @@ -10,12 +10,15 @@ spec: ports: - protocol: TCP port: {{ .Values.port }} + name: agent targetPort: {{ .Values.port }} - protocol: TCP port: {{ .Values.bridge_port }} + name: bridge targetPort: {{ .Values.bridge_port }} - protocol: TCP port: {{ .Values.health_port }} + name: health targetPort: {{ .Values.health_port }} selector: app.kubernetes.io/name: op-bridge diff --git a/helm/portal/templates/service.yaml b/helm/portal/templates/service.yaml index 2fffdc2..fe290dd 100644 --- a/helm/portal/templates/service.yaml +++ b/helm/portal/templates/service.yaml @@ -10,9 +10,11 @@ spec: ports: - protocol: TCP port: {{ .Values.port }} + name: agent targetPort: {{ .Values.port }} - protocol: TCP port: {{ .Values.health_port }} + name: health targetPort: {{ .Values.health_port }} selector: app.kubernetes.io/name: op-portal From 6f1742c573b6e5a0fd005646efc9777f97cdb645 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 24 Oct 2024 09:38:14 +0000 Subject: [PATCH 25/51] Release 0.0.8 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a1fe12..b362037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.8] - 2024-10-24 ### Added - Added names for the ports in the helm charts @@ -38,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.8]: https://github.com/isambard-sc/openportal/releases/tag/0.0.8 [0.0.7]: https://github.com/isambard-sc/openportal/releases/tag/0.0.7 [0.0.6]: https://github.com/isambard-sc/openportal/releases/tag/0.0.6 [0.0.5]: https://github.com/isambard-sc/openportal/releases/tag/0.0.5 From 5d22e0c68f6d1bc98682963b5bbd4c0850ca3ece Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 25 Oct 2024 13:33:48 +0700 Subject: [PATCH 26/51] Fixed bug in parsing bridge init options. Added in TLS support for wss connections. Removed port number from urls when it is the default - this really confused me when I was testing ;-) --- paddington/Cargo.toml | 2 +- paddington/src/config.rs | 21 ++++++++++++++++++++- templemeads/src/agent_bridge.rs | 12 ++++++++++-- templemeads/src/bridge_server.rs | 31 +++++++++++++++++++++++++------ 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index c6b969c..819f772 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -32,7 +32,7 @@ serde_json = "1.0.120" serde_with = { version="3.9.0", features = ["hex"] } thiserror = "1.0.63" tokio = { version = "1.0", features = ["full", "tracing"] } -tokio-tungstenite = "0.23.1" +tokio-tungstenite = { version = "0.23.1", features = ["rustls-tls-native-roots"] } toml = "0.8.16" tracing = "0.1.40" url = {version="2.5.2", features=["serde"]} diff --git a/paddington/src/config.rs b/paddington/src/config.rs index d6f812b..6ee5bd1 100644 --- a/paddington/src/config.rs +++ b/paddington/src/config.rs @@ -162,9 +162,28 @@ fn create_websocket_url(url: &str) -> Result { }; let host = url.host_str().unwrap_or("localhost"); - let port = url.port().unwrap_or(8080); + let port = url.port().unwrap_or(match scheme { + "ws" => 80, + "wss" => 443, + _ => 443, + }); let path = url.path(); + // don't specify the port if it's the default for the protocol + match scheme { + "ws" => { + if port == 80 { + return Ok(format!("{}://{}", scheme, host)); + } + } + "wss" => { + if port == 443 { + return Ok(format!("{}://{}", scheme, host)); + } + } + _ => {} + } + Ok(format!("{}://{}:{}{}", scheme, host, port, path)) } diff --git a/templemeads/src/agent_bridge.rs b/templemeads/src/agent_bridge.rs index 9830cb2..62be82e 100644 --- a/templemeads/src/agent_bridge.rs +++ b/templemeads/src/agent_bridge.rs @@ -116,6 +116,7 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> url, ip, port, + bridge_url, bridge_ip, bridge_port, healthcheck_port, @@ -143,7 +144,7 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> )? }, bridge: BridgeConfig::new( - &bridge_ip.clone().unwrap_or(defaults.bridge.url()), + &bridge_url.clone().unwrap_or(defaults.bridge.url()), bridge_ip .clone() .unwrap_or(defaults.bridge.ip()) @@ -371,10 +372,17 @@ enum Commands { )] port: Option, + #[arg( + long, + short = 'r', + help = "URL of the bridge API server including port and route (e.g. http://localhost:3000)" + )] + bridge_url: Option, + #[arg( long, short = 'b', - help = "IP address on which to listen for bridge connections (e.g. '::')" + help = "IP address on which to listen for bridge connections (e.g. '0.0.0.0')" )] bridge_ip: Option, diff --git a/templemeads/src/bridge_server.rs b/templemeads/src/bridge_server.rs index c5a361e..d368d74 100644 --- a/templemeads/src/bridge_server.rs +++ b/templemeads/src/bridge_server.rs @@ -70,9 +70,28 @@ fn create_webserver_url(url: &str) -> Result { }; let host = url.host_str().unwrap_or("localhost"); - let port = url.port().unwrap_or(3000); + let port = url.port().unwrap_or(match scheme { + "http" => 80, + "https" => 443, + _ => 443, + }); let path = url.path(); + // don't add the port if it is the default for the protocol + match scheme { + "http" => { + if port == 80 { + return Ok(format!("{}://{}{}", scheme, host, path).parse::()?); + } + } + "https" => { + if port == 443 { + return Ok(format!("{}://{}{}", scheme, host, path).parse::()?); + } + } + _ => {} + } + Ok(format!("{}://{}:{}{}", scheme, host, port, path).parse::()?) } @@ -81,12 +100,12 @@ impl Config { Self { url: create_webserver_url(url).unwrap_or_else(|e| { tracing::error!( - "Could not parse URL: {} because {}. Using http://localhost:3000 instead.", - e, - url + "Could not parse URL: {} because '{}'. Using http://localhost:{port} instead.", + url, + e ); #[allow(clippy::unwrap_used)] - "http://localhost:3000".parse().unwrap() + format!("http://localhost:{port}").parse().unwrap() }), ip, port, @@ -105,7 +124,7 @@ pub struct Defaults { impl Defaults { pub fn parse(url: Option, ip: Option, port: Option) -> Self { Self { - url: url.unwrap_or("http://localhost:3000".to_owned()), + url: url.unwrap_or("http://localhost:8042".to_owned()), ip: ip.unwrap_or("127.0.0.1".to_owned()), port: port.unwrap_or(8042), } From 0536136f1ab1b93832cc9d665f8b090cd27c6d22 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 25 Oct 2024 14:08:41 +0700 Subject: [PATCH 27/51] Preparing for the next release --- CHANGELOG.md | 4 +++ bridge/Cargo.toml | 2 +- bridge/src/main.rs | 1 + cluster/Cargo.toml | 2 +- cluster/src/main.rs | 1 + docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/cluster/src/main.rs | 1 + docs/cmdline/portal/Cargo.toml | 2 +- docs/cmdline/portal/src/main.rs | 1 + docs/echo/Cargo.toml | 2 +- docs/echo/src/main.rs | 3 +- docs/job/Cargo.toml | 2 +- docs/job/src/main.rs | 3 +- filesystem/Cargo.toml | 2 +- filesystem/src/main.rs | 1 + freeipa/Cargo.toml | 2 +- freeipa/src/main.rs | 1 + paddington/Cargo.toml | 5 +-- paddington/src/config.rs | 54 +++++++++++++++++++++++------- paddington/src/connection.rs | 56 +++++++++++++++++++++++++------- paddington/src/eventloop.rs | 10 ++++-- portal/Cargo.toml | 2 +- portal/src/main.rs | 1 + provider/Cargo.toml | 2 +- provider/src/main.rs | 1 + python/Cargo.toml | 2 +- slurm/Cargo.toml | 2 +- slurm/src/main.rs | 1 + templemeads/Cargo.toml | 2 +- templemeads/src/agent_bridge.rs | 20 +++++++++++- templemeads/src/agent_core.rs | 21 +++++++++++- 31 files changed, 166 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b362037..de84222 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Fixed bug in parsing command line options for bridge +- Added support for getting the client IP address from a proxy header (e.g. `X-Forwarded-For`) +- Cleaned up port handling, so URLs with default ports don't have the ports specified ## [0.0.8] - 2024-10-24 ### Added diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index 299fa7e..a1c5f4c 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.1" +version = "0.0.9" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/bridge/src/main.rs b/bridge/src/main.rs index f11d6df..5c3c18d 100644 --- a/bridge/src/main.rs +++ b/bridge/src/main.rs @@ -40,6 +40,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8044), None, + None, Some("http://localhost:3000".to_owned()), Some("127.0.0.1".to_owned()), Some(3000), diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index c436c77..52c013d 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.1" +version = "0.0.9" description = "An example of an OpenPortal HPC cluster platform agent" edition = "2021" license = "MIT" diff --git a/cluster/src/main.rs b/cluster/src/main.rs index b43a44d..860bf59 100644 --- a/cluster/src/main.rs +++ b/cluster/src/main.rs @@ -35,6 +35,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8045), None, + None, Some(AgentType::Platform), ); diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index 3700f5f..d632896 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.1" +version = "0.0.9" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/src/main.rs b/docs/cmdline/cluster/src/main.rs index fd942f0..80900e2 100644 --- a/docs/cmdline/cluster/src/main.rs +++ b/docs/cmdline/cluster/src/main.rs @@ -25,6 +25,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8091), None, + None, Some(AgentType::Instance), ); diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index 1219c28..042e1c6 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.1" +version = "0.0.9" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/src/main.rs b/docs/cmdline/portal/src/main.rs index 91d7b7c..e51931e 100644 --- a/docs/cmdline/portal/src/main.rs +++ b/docs/cmdline/portal/src/main.rs @@ -21,6 +21,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8090), None, + None, Some(AgentType::Portal), ); diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index 78caf65..4f5b164 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.1" +version = "0.0.9" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/echo/src/main.rs b/docs/echo/src/main.rs index 84bd08c..1a819c6 100644 --- a/docs/echo/src/main.rs +++ b/docs/echo/src/main.rs @@ -187,6 +187,7 @@ async fn run_client(invitation: &Path) -> Result<(), Error> { "127.0.0.1", &6502, &None, + &None, )?; // now give the invitation to connect to the server to the client @@ -262,7 +263,7 @@ async fn run_server( invitation: &Path, ) -> Result<(), Error> { // create the echo-server service - let mut service = ServiceConfig::new("echo-server", url, ip, port, &None)?; + let mut service = ServiceConfig::new("echo-server", url, ip, port, &None, &None)?; let invite = service.add_client("echo-client", range)?; diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index f86d909..a0e1f6f 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.1" +version = "0.0.9" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/docs/job/src/main.rs b/docs/job/src/main.rs index f303a6d..5164423 100644 --- a/docs/job/src/main.rs +++ b/docs/job/src/main.rs @@ -205,6 +205,7 @@ async fn run_cluster(invitation: &Path) -> Result<(), Error> { "127.0.0.1", &6502, &None, + &None, )?; // now give the invitation to connect to the server to the client @@ -260,7 +261,7 @@ async fn run_portal( invitation: &Path, ) -> Result<(), Error> { // create a paddington service configuration for the portal agent - let mut service = ServiceConfig::new("portal", url, ip, port, &None)?; + let mut service = ServiceConfig::new("portal", url, ip, port, &None, &None)?; // add the cluster to the portal, returning an invitation let invite = service.add_client("cluster", range)?; diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index 9d80506..12a7180 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.1" +version = "0.0.9" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/filesystem/src/main.rs b/filesystem/src/main.rs index 36cd475..ae01436 100644 --- a/filesystem/src/main.rs +++ b/filesystem/src/main.rs @@ -40,6 +40,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8047), None, + None, Some(AgentType::Filesystem), ); diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index 5539041..f0a73c3 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.1" +version = "0.0.9" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/freeipa/src/main.rs b/freeipa/src/main.rs index f4f7ccc..dc0404b 100644 --- a/freeipa/src/main.rs +++ b/freeipa/src/main.rs @@ -45,6 +45,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8046), None, + None, Some(AgentType::Account), ); diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index 819f772..35ae750 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.1" +version = "0.0.9" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" @@ -32,9 +32,10 @@ serde_json = "1.0.120" serde_with = { version="3.9.0", features = ["hex"] } thiserror = "1.0.63" tokio = { version = "1.0", features = ["full", "tracing"] } -tokio-tungstenite = { version = "0.23.1", features = ["rustls-tls-native-roots"] } +tokio-tungstenite = { version = "0.24.0", features = ["rustls-tls-native-roots"] } toml = "0.8.16" tracing = "0.1.40" +tungstenite = "0.24.0" url = {version="2.5.2", features=["serde"]} [lints.rust] diff --git a/paddington/src/config.rs b/paddington/src/config.rs index 6ee5bd1..04aa634 100644 --- a/paddington/src/config.rs +++ b/paddington/src/config.rs @@ -74,6 +74,7 @@ pub struct Defaults { ip: String, port: u16, healthcheck_port: Option, + proxy_header: Option, } impl Defaults { @@ -84,6 +85,7 @@ impl Defaults { ip: Option, port: Option, healthcheck_port: Option, + proxy_header: Option, ) -> Self { let config_file = config_file.unwrap_or( dirs::config_local_dir() @@ -102,6 +104,7 @@ impl Defaults { ip: ip.unwrap_or("127.0.0.1".to_owned()), port: port.unwrap_or(8042), healthcheck_port, + proxy_header, } } @@ -439,6 +442,7 @@ pub struct ServiceConfig { ip: IpAddr, port: u16, heathcheck_port: Option, + proxy_header: Option, servers: Vec, clients: Vec, @@ -452,6 +456,7 @@ impl ServiceConfig { ip: &str, port: &u16, healthcheck_port: &Option, + proxy_header: &Option, ) -> Result { Ok(ServiceConfig { name: name.to_string(), @@ -461,6 +466,7 @@ impl ServiceConfig { .with_context(|| format!("Could not parse IP address: {}", ip))?, port: *port, heathcheck_port: *healthcheck_port, + proxy_header: proxy_header.clone(), servers: Vec::new(), clients: Vec::new(), encryption: None, @@ -531,6 +537,10 @@ impl ServiceConfig { self.heathcheck_port } + pub fn proxy_header(&self) -> Option { + self.proxy_header.clone() + } + pub fn add_client(&mut self, name: &str, ip: &str) -> Result { let ip = IpOrRange::new(ip) .with_context(|| format!("Could not parse into an IP address or IP range: {}", ip))?; @@ -612,6 +622,7 @@ impl ServiceConfig { ip: IpAddr, port: u16, healthcheck_port: &Option, + proxy_header: &Option, ) -> Result { // see if this config_dir exists - return an error if it does let config_file = path::absolute(config_file).with_context(|| { @@ -625,7 +636,14 @@ impl ServiceConfig { return Err(Error::NotExists(config_file.to_string_lossy().to_string())); } - let config = ServiceConfig::new(&name, &url, &ip.to_string(), &port, healthcheck_port)?; + let config = ServiceConfig::new( + &name, + &url, + &ip.to_string(), + &port, + healthcheck_port, + proxy_header, + )?; save::(config.clone(), &config_file)?; // check we can read the config and return it @@ -688,17 +706,29 @@ mod tests { #[test] fn test_invitations() { - let mut primary = - ServiceConfig::new("primary", "http://localhost", "127.0.0.1", &5544, &None) - .unwrap_or_else(|e| { - unreachable!("Cannot create service config: {}", e); - }); - - let mut secondary = - ServiceConfig::new("secondary", "http://localhost", "127.0.0.1", &5545, &None) - .unwrap_or_else(|e| { - unreachable!("Cannot create service config: {}", e); - }); + let mut primary = ServiceConfig::new( + "primary", + "http://localhost", + "127.0.0.1", + &5544, + &None, + &None, + ) + .unwrap_or_else(|e| { + unreachable!("Cannot create service config: {}", e); + }); + + let mut secondary = ServiceConfig::new( + "secondary", + "http://localhost", + "127.0.0.1", + &5545, + &None, + &None, + ) + .unwrap_or_else(|e| { + unreachable!("Cannot create service config: {}", e); + }); // introduce the secondary to the primary let invite = primary diff --git a/paddington/src/connection.rs b/paddington/src/connection.rs index 94ba593..2525dd3 100644 --- a/paddington/src/connection.rs +++ b/paddington/src/connection.rs @@ -9,12 +9,15 @@ use futures_channel::mpsc::{unbounded, UnboundedSender}; use futures_util::{future, pin_mut, stream::TryStreamExt}; use secrecy::ExposeSecret; use serde::{de::DeserializeOwned, Serialize}; +use std::sync::Arc; use tokio::net::TcpStream; use tokio::sync::Mutex as TokioMutex; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::protocol::Message as TokioMessage; - -use std::sync::Arc; +use tungstenite::handshake::server::{ + ErrorResponse as HandshakeErrorResponse, Request as HandshakeRequest, + Response as HandshakeResponse, +}; use crate::command::Command; use crate::config::{ClientConfig, PeerConfig, ServiceConfig}; @@ -358,10 +361,48 @@ impl Connection { // we now know we are the only ones handling the connection, // and are safe to update the keys etc. - let addr: std::net::SocketAddr = stream + let mut addr: std::net::SocketAddr = stream .peer_addr() .with_context(|| "Error getting the peer address. Ensure the connection is open.")?; + let proxy_header = self.config.proxy_header(); + let mut proxy_client = None; + + let process_headers = |request: &HandshakeRequest, + response: HandshakeResponse| + -> Result { + if let Some(proxy_header) = proxy_header { + if let Some(value) = request + .headers() + .get(proxy_header) + .and_then(|value| value.to_str().ok()) + { + proxy_client = Some(value.to_string()); + } + } + + Ok(response) + }; + + let ws_stream = tokio_tungstenite::accept_hdr_async(stream, process_headers) + .await + .with_context(|| { + format!( + "Error accepting WebSocket connection from: {}. Closing connection.", + addr + ) + })?; + + if let Some(proxy_client) = proxy_client { + tracing::info!("Proxy client: {:?}", proxy_client); + addr = proxy_client.parse().with_context(|| { + format!( + "Error parsing proxy client address: {}. Closing connection.", + proxy_client + ) + })?; + } + tracing::info!("Accepted connection from peer: {}", addr); let clients: Vec = self @@ -379,15 +420,6 @@ impl Connection { )); } - let ws_stream = tokio_tungstenite::accept_async(stream) - .await - .with_context(|| { - format!( - "Error accepting WebSocket connection from: {}. Closing connection.", - addr - ) - })?; - // Split the WebSocket stream into incoming and outgoing parts let (mut outgoing, mut incoming) = ws_stream.split(); diff --git a/paddington/src/eventloop.rs b/paddington/src/eventloop.rs index e6b4b04..bf24b2f 100644 --- a/paddington/src/eventloop.rs +++ b/paddington/src/eventloop.rs @@ -57,8 +57,14 @@ mod tests { async fn test_run() -> Result<()> { // this tests that the service can be configured and will run // (it will exit immediately as there are no clients or servers) - let config = - ServiceConfig::new("test_server", "http://localhost", "127.0.0.1", &5544, &None)?; + let config = ServiceConfig::new( + "test_server", + "http://localhost", + "127.0.0.1", + &5544, + &None, + &None, + )?; run(config).await?; Ok(()) diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 71b9a35..a5c2374 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.1" +version = "0.0.9" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/portal/src/main.rs b/portal/src/main.rs index b5e2635..d5749b0 100644 --- a/portal/src/main.rs +++ b/portal/src/main.rs @@ -41,6 +41,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8040), None, + None, Some(AgentType::Portal), ); diff --git a/provider/Cargo.toml b/provider/Cargo.toml index 4f61812..bf6b940 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.1" +version = "0.0.9" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/provider/src/main.rs b/provider/src/main.rs index f6759de..02b59ff 100644 --- a/provider/src/main.rs +++ b/provider/src/main.rs @@ -41,6 +41,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8041), None, + None, Some(AgentType::Provider), ); diff --git a/python/Cargo.toml b/python/Cargo.toml index 44fd8d9..dc1ea86 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.1" +version = "0.0.9" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/slurm/Cargo.toml b/slurm/Cargo.toml index 69e634f..46af131 100644 --- a/slurm/Cargo.toml +++ b/slurm/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-slurm" -version = "0.0.1" +version = "0.0.9" description = "An example of an OpenPortal Slurm cluster instance agent" edition = "2021" license = "MIT" diff --git a/slurm/src/main.rs b/slurm/src/main.rs index fa23d42..07780b9 100644 --- a/slurm/src/main.rs +++ b/slurm/src/main.rs @@ -41,6 +41,7 @@ async fn main() -> Result<()> { Some("127.0.0.1".to_owned()), Some(8046), None, + None, Some(AgentType::Instance), ); diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index 3edbd7d..e24ea6c 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.1" +version = "0.0.9" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" diff --git a/templemeads/src/agent_bridge.rs b/templemeads/src/agent_bridge.rs index 62be82e..12b15ed 100644 --- a/templemeads/src/agent_bridge.rs +++ b/templemeads/src/agent_bridge.rs @@ -73,12 +73,21 @@ impl Defaults { ip: Option, port: Option, healthcheck_port: Option, + proxy_header: Option, bridge_url: Option, bridge_ip: Option, bridge_port: Option, ) -> Self { Self { - service: ServiceDefaults::parse(name, config_file, url, ip, port, healthcheck_port), + service: ServiceDefaults::parse( + name, + config_file, + url, + ip, + port, + healthcheck_port, + proxy_header, + ), bridge: BridgeDefaults::parse(bridge_url, bridge_ip, bridge_port), } } @@ -120,6 +129,7 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> bridge_ip, bridge_port, healthcheck_port, + proxy_header, force, }) => { let local_healthcheck_port; @@ -141,6 +151,7 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> .to_string(), &port.unwrap_or_else(|| defaults.service.port()), &local_healthcheck_port, + proxy_header, )? }, bridge: BridgeConfig::new( @@ -400,6 +411,13 @@ enum Commands { )] healthcheck_port: Option, + #[arg( + long, + short = 'x', + help = "Optional header to use for proxying requests - look in this for the client IP address" + )] + proxy_header: Option, + #[arg(long, short = 'f', help = "Force reinitialisation")] force: bool, }, diff --git a/templemeads/src/agent_core.rs b/templemeads/src/agent_core.rs index 4a4facd..e477b0f 100644 --- a/templemeads/src/agent_core.rs +++ b/templemeads/src/agent_core.rs @@ -74,6 +74,7 @@ pub struct Defaults { } impl Defaults { + #[allow(clippy::too_many_arguments)] pub fn parse( name: Option, config_file: Option, @@ -81,10 +82,19 @@ impl Defaults { ip: Option, port: Option, healthcheck_port: Option, + proxy_header: Option, agent: Option, ) -> Self { Self { - service: ServiceDefaults::parse(name, config_file, url, ip, port, healthcheck_port), + service: ServiceDefaults::parse( + name, + config_file, + url, + ip, + port, + healthcheck_port, + proxy_header, + ), agent: agent.unwrap_or(AgentType::Portal), extras: HashMap::new(), } @@ -132,6 +142,7 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> ip, port, healthcheck_port, + proxy_header, force, }) => { let local_healthcheck_port; @@ -153,6 +164,7 @@ pub async fn process_args(defaults: &Defaults) -> Result, Error> .to_string(), &port.unwrap_or_else(|| defaults.service.port()), &local_healthcheck_port, + proxy_header, )? }, agent: defaults.agent.clone(), @@ -393,6 +405,13 @@ enum Commands { )] healthcheck_port: Option, + #[arg( + long, + short = 'x', + help = "Proxy header to use for the client IP address - look here for the client IP address" + )] + proxy_header: Option, + #[arg(long, short = 'f', help = "Force reinitialisation")] force: bool, }, From 1d1dd4c969c9cd035dc4b0726ef5cb9542fb2fda Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 25 Oct 2024 07:14:50 +0000 Subject: [PATCH 28/51] Release 0.0.9 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de84222..54c7a0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.9] - 2024-10-25 ### Added - Fixed bug in parsing command line options for bridge - Added support for getting the client IP address from a proxy header (e.g. `X-Forwarded-For`) @@ -44,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.9]: https://github.com/isambard-sc/openportal/releases/tag/0.0.9 [0.0.8]: https://github.com/isambard-sc/openportal/releases/tag/0.0.8 [0.0.7]: https://github.com/isambard-sc/openportal/releases/tag/0.0.7 [0.0.6]: https://github.com/isambard-sc/openportal/releases/tag/0.0.6 From cb93d6ddcf5396860a4935721c413db7564ca752 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 25 Oct 2024 15:28:29 +0700 Subject: [PATCH 29/51] Fixed IP parsing bug [ci skip] --- CHANGELOG.md | 2 ++ bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- paddington/Cargo.toml | 2 +- paddington/src/connection.rs | 9 +++------ paddington/src/server.rs | 2 -- portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- slurm/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- 17 files changed, 19 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54c7a0a..488b6a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Fixed bug in parsing header proxy IP address ## [0.0.9] - 2024-10-25 ### Added diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index a1c5f4c..4c36a16 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.9" +version = "0.0.10" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index 52c013d..dde3d8b 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.9" +version = "0.0.10" description = "An example of an OpenPortal HPC cluster platform agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index d632896..5b759fc 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.9" +version = "0.0.10" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index 042e1c6..94c1012 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.9" +version = "0.0.10" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index 4f5b164..3c2de3b 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.9" +version = "0.0.10" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index a0e1f6f..a179030 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.9" +version = "0.0.10" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index 12a7180..2eafbaa 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.9" +version = "0.0.10" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index f0a73c3..0f89d5e 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.9" +version = "0.0.10" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index 35ae750..d80bc7e 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.9" +version = "0.0.10" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" diff --git a/paddington/src/connection.rs b/paddington/src/connection.rs index 2525dd3..bb5b26f 100644 --- a/paddington/src/connection.rs +++ b/paddington/src/connection.rs @@ -9,6 +9,7 @@ use futures_channel::mpsc::{unbounded, UnboundedSender}; use futures_util::{future, pin_mut, stream::TryStreamExt}; use secrecy::ExposeSecret; use serde::{de::DeserializeOwned, Serialize}; +use std::str::FromStr; use std::sync::Arc; use tokio::net::TcpStream; use tokio::sync::Mutex as TokioMutex; @@ -395,12 +396,8 @@ impl Connection { if let Some(proxy_client) = proxy_client { tracing::info!("Proxy client: {:?}", proxy_client); - addr = proxy_client.parse().with_context(|| { - format!( - "Error parsing proxy client address: {}. Closing connection.", - proxy_client - ) - })?; + addr = std::net::SocketAddr::from_str(&proxy_client) + .with_context(|| "Error parsing proxy client address")?; } tracing::info!("Accepted connection from peer: {}", addr); diff --git a/paddington/src/server.rs b/paddington/src/server.rs index e964747..6cb5925 100644 --- a/paddington/src/server.rs +++ b/paddington/src/server.rs @@ -53,8 +53,6 @@ pub async fn run_once(config: ServiceConfig) -> Result<(), Error> { // Let's spawn the handling of each connection in a separate task. loop { - tracing::info!("Awaiting the next connection..."); - match listener.accept().await { Ok((stream, addr)) => { tracing::info!("New connection from: {}", addr); diff --git a/portal/Cargo.toml b/portal/Cargo.toml index a5c2374..800a36f 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.9" +version = "0.0.10" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index bf6b940..71dbe35 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.9" +version = "0.0.10" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index dc1ea86..b7f4825 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.9" +version = "0.0.10" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/slurm/Cargo.toml b/slurm/Cargo.toml index 46af131..3f2768f 100644 --- a/slurm/Cargo.toml +++ b/slurm/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-slurm" -version = "0.0.9" +version = "0.0.10" description = "An example of an OpenPortal Slurm cluster instance agent" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index e24ea6c..561b017 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.9" +version = "0.0.10" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" From 2d7755b51c1c34f0b4ff6738381094cc883e8ddf Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 25 Oct 2024 08:31:55 +0000 Subject: [PATCH 30/51] Release 0.0.10 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 488b6a4..c260a16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.10] - 2024-10-25 ### Added - Fixed bug in parsing header proxy IP address @@ -48,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.10]: https://github.com/isambard-sc/openportal/releases/tag/0.0.10 [0.0.9]: https://github.com/isambard-sc/openportal/releases/tag/0.0.9 [0.0.8]: https://github.com/isambard-sc/openportal/releases/tag/0.0.8 [0.0.7]: https://github.com/isambard-sc/openportal/releases/tag/0.0.7 From e4c4943a6d6d49d2ff3e12ab3647c81f46437c6f Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Mon, 28 Oct 2024 14:00:54 +0700 Subject: [PATCH 31/51] Fix client ip parsing issue [ci skip] --- CHANGELOG.md | 2 ++ paddington/src/connection.rs | 31 +++++++++++++++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c260a16..d33f838 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Fixed bug in handling of client proxy IP - need to use IP not port ;-) ## [0.0.10] - 2024-10-25 ### Added diff --git a/paddington/src/connection.rs b/paddington/src/connection.rs index bb5b26f..a63173a 100644 --- a/paddington/src/connection.rs +++ b/paddington/src/connection.rs @@ -9,7 +9,6 @@ use futures_channel::mpsc::{unbounded, UnboundedSender}; use futures_util::{future, pin_mut, stream::TryStreamExt}; use secrecy::ExposeSecret; use serde::{de::DeserializeOwned, Serialize}; -use std::str::FromStr; use std::sync::Arc; use tokio::net::TcpStream; use tokio::sync::Mutex as TokioMutex; @@ -362,9 +361,10 @@ impl Connection { // we now know we are the only ones handling the connection, // and are safe to update the keys etc. - let mut addr: std::net::SocketAddr = stream + let mut client_ip: std::net::IpAddr = stream .peer_addr() - .with_context(|| "Error getting the peer address. Ensure the connection is open.")?; + .with_context(|| "Error getting the peer address. Ensure the connection is open.")? + .ip(); let proxy_header = self.config.proxy_header(); let mut proxy_client = None; @@ -390,28 +390,32 @@ impl Connection { .with_context(|| { format!( "Error accepting WebSocket connection from: {}. Closing connection.", - addr + client_ip ) })?; if let Some(proxy_client) = proxy_client { tracing::info!("Proxy client: {:?}", proxy_client); - addr = std::net::SocketAddr::from_str(&proxy_client) + client_ip = proxy_client + .parse() .with_context(|| "Error parsing proxy client address")?; } - tracing::info!("Accepted connection from peer: {}", addr); + // this doesn't need to be mutable any more + let client_ip = client_ip; + + tracing::info!("Accepted connection from peer: {}", client_ip); let clients: Vec = self .config .clients() .iter() - .filter(|client| client.matches(addr.ip())) + .filter(|client| client.matches(client_ip)) .cloned() .collect(); if clients.is_empty() { - tracing::warn!("No matching peer found for address: {}", addr); + tracing::warn!("No matching peer found for address: {}", client_ip); return Err(Error::InvalidPeer( "No matching peer found for address.".to_string(), )); @@ -457,7 +461,7 @@ impl Connection { tracing::info!( "Client {:?} authenticated for address: {}", client.name().unwrap_or_default(), - addr + client_ip ); true } @@ -468,7 +472,10 @@ impl Connection { .collect(); if clients.is_empty() { - tracing::warn!("No matching peer could authenticate for address: {}", addr); + tracing::warn!( + "No matching peer could authenticate for address: {}", + client_ip + ); return Err(Error::InvalidPeer( "No matching peer could authenticate for address.".to_string(), )); @@ -478,7 +485,7 @@ impl Connection { tracing::warn!( "Multiple matching peers found for address: {} - \ {:?}. Ignoring all but the first...", - addr, + client_ip, clients ); } @@ -570,7 +577,7 @@ impl Connection { pin_mut!(received_from_peer, send_to_peer); future::select(received_from_peer, send_to_peer).await; - tracing::info!("{} disconnected", &addr); + tracing::info!("{} disconnected", &client_ip); // we've exited, meaning that this connection is now closed self.closed_connection().await; From f89ecac4d6310adc71154e80533721a8ab042925 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Mon, 28 Oct 2024 14:02:03 +0700 Subject: [PATCH 32/51] Updated version number for next release [ci skip] --- bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- paddington/Cargo.toml | 2 +- portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- slurm/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index 4c36a16..a714919 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.10" +version = "0.0.11" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index dde3d8b..3933820 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.10" +version = "0.0.11" description = "An example of an OpenPortal HPC cluster platform agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index 5b759fc..b0ba791 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.10" +version = "0.0.11" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index 94c1012..0a4ba20 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.10" +version = "0.0.11" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index 3c2de3b..3d243ea 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.10" +version = "0.0.11" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index a179030..fd80112 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.10" +version = "0.0.11" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index 2eafbaa..9665228 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.10" +version = "0.0.11" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index 0f89d5e..484e25c 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.10" +version = "0.0.11" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index d80bc7e..44a9b98 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.10" +version = "0.0.11" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 800a36f..8a34e2a 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.10" +version = "0.0.11" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index 71dbe35..3a1bb6e 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.10" +version = "0.0.11" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index b7f4825..c8ef3a1 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.10" +version = "0.0.11" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/slurm/Cargo.toml b/slurm/Cargo.toml index 3f2768f..d8aa51e 100644 --- a/slurm/Cargo.toml +++ b/slurm/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-slurm" -version = "0.0.10" +version = "0.0.11" description = "An example of an OpenPortal Slurm cluster instance agent" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index 561b017..80c1a9b 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.10" +version = "0.0.11" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" From 86fdd9d05a455e51e7d1312d72a1368e8cfb3c5d Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 28 Oct 2024 07:05:07 +0000 Subject: [PATCH 33/51] Release 0.0.11 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d33f838..95acdca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.11] - 2024-10-28 ### Added - Fixed bug in handling of client proxy IP - need to use IP not port ;-) @@ -52,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.11]: https://github.com/isambard-sc/openportal/releases/tag/0.0.11 [0.0.10]: https://github.com/isambard-sc/openportal/releases/tag/0.0.10 [0.0.9]: https://github.com/isambard-sc/openportal/releases/tag/0.0.9 [0.0.8]: https://github.com/isambard-sc/openportal/releases/tag/0.0.8 From a8aa587ec7dba9d9126aac9de2854619fa7d88d9 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Mon, 28 Oct 2024 15:36:40 +0700 Subject: [PATCH 34/51] Added a keepalive message to keep the websocket open [ci skip] --- CHANGELOG.md | 2 ++ bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- paddington/Cargo.toml | 2 +- paddington/src/connection.rs | 7 +++++++ paddington/src/message.rs | 32 ++++++++++++++++++++++++++++++++ portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- slurm/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- templemeads/src/handler.rs | 32 +++++++++++++++++++++++++++----- 18 files changed, 82 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95acdca..0bea4c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Added support for keepalive messages so that connections are kept open ## [0.0.11] - 2024-10-28 ### Added diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index a714919..330a416 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.11" +version = "0.0.12" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index 3933820..c6c945c 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.11" +version = "0.0.12" description = "An example of an OpenPortal HPC cluster platform agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index b0ba791..e3be4b7 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.11" +version = "0.0.12" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index 0a4ba20..f4f7b3d 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.11" +version = "0.0.12" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index 3d243ea..ab555da 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.11" +version = "0.0.12" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index fd80112..3845562 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.11" +version = "0.0.12" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index 9665228..7ed45eb 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.11" +version = "0.0.12" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index 484e25c..ea2ac33 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.11" +version = "0.0.12" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index 44a9b98..ddc548f 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.11" +version = "0.0.12" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" diff --git a/paddington/src/connection.rs b/paddington/src/connection.rs index a63173a..b9286e9 100644 --- a/paddington/src/connection.rs +++ b/paddington/src/connection.rs @@ -321,6 +321,13 @@ impl Connection { exchange::received(Command::connected(peer_name.clone()).into()) .with_context(|| "Error triggering /connected control message")?; + // finally, send a keepalive message to the peer - this will start + // a ping-pong with the peer that should keep it open + // (client sends, as the server should already be set up now) + exchange::send(Message::keepalive(&peer_name)) + .await + .with_context(|| "Error sending keepalive message to peer")?; + pin_mut!(received_from_peer, send_to_peer); future::select(received_from_peer, send_to_peer).await; diff --git a/paddington/src/message.rs b/paddington/src/message.rs index a809f68..5071390 100644 --- a/paddington/src/message.rs +++ b/paddington/src/message.rs @@ -24,6 +24,12 @@ impl Display for Message { } } +pub enum MessageType { + Control, + KeepAlive, + Message, +} + impl Message { pub fn new(sender: &str, payload: &str) -> Self { Self { @@ -45,6 +51,32 @@ impl Message { self.sender.is_empty() } + pub fn keepalive(sender: &str) -> Self { + Self { + sender: sender.to_owned(), + recipient: "".to_owned(), + payload: "KEEPALIVE".to_owned(), + } + } + + pub fn is_keepalive(&self) -> bool { + self.payload == "KEEPALIVE" + } + + pub fn is_message(&self) -> bool { + !self.is_control() && !self.is_keepalive() + } + + pub fn typ(&self) -> MessageType { + if self.is_control() { + MessageType::Control + } else if self.is_keepalive() { + MessageType::KeepAlive + } else { + MessageType::Message + } + } + pub fn set_recipient(&mut self, recipient: &str) { self.recipient = recipient.to_owned(); } diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 8a34e2a..6901174 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.11" +version = "0.0.12" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index 3a1bb6e..b112698 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.11" +version = "0.0.12" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index c8ef3a1..cc41eac 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.11" +version = "0.0.12" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/slurm/Cargo.toml b/slurm/Cargo.toml index d8aa51e..97c1e7b 100644 --- a/slurm/Cargo.toml +++ b/slurm/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-slurm" -version = "0.0.11" +version = "0.0.12" description = "An example of an OpenPortal Slurm cluster instance agent" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index 80c1a9b..c19e838 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.11" +version = "0.0.12" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" diff --git a/templemeads/src/handler.rs b/templemeads/src/handler.rs index 5ca8bf9..76af446 100644 --- a/templemeads/src/handler.rs +++ b/templemeads/src/handler.rs @@ -13,7 +13,7 @@ use crate::runnable::{default_runner, AsyncRunnable}; use anyhow::Result; use once_cell::sync::Lazy; use paddington::async_message_handler; -use paddington::message::Message; +use paddington::message::{Message, MessageType}; use std::boxed::Box; use tokio::sync::RwLock; @@ -176,9 +176,29 @@ async_message_handler! { pub async fn process_message(message: Message) -> Result<(), paddington::Error> { let service_info: ServiceDetails = SERVICE_DETAILS.read().await.to_owned(); - match message.is_control() { - true => Ok(process_control_message(&service_info.agent_type, message.into()).await?), - false => { + match message.typ() { + MessageType::Control => { + process_control_message(&service_info.agent_type, message.into()).await?; + + Ok(()) + } + MessageType::KeepAlive => { + let sender: String = message.sender().to_owned(); + let recipient: String = message.recipient().to_owned(); + + if (recipient != service_info.service) { + return Err(Error::Delivery(format!("Recipient {} does not match service {}", recipient, service_info.service)).into()); + } + + tracing::info!("Keep alive message received from {}", sender); + + // wait 20 seconds and send a keep alive message back + tokio::time::sleep(tokio::time::Duration::from_secs(20)).await; + paddington::send(Message::keepalive(&sender)).await?; + + Ok(()) + } + MessageType::Message => { let sender: String = message.sender().to_owned(); let recipient: String = message.recipient().to_owned(); let command: Command = message.into(); @@ -187,7 +207,9 @@ async_message_handler! { return Err(Error::Delivery(format!("Recipient {} does not match service {}", recipient, service_info.service)).into()); } - Ok(process_command(&recipient, &sender, &command, &service_info.runner).await?) + process_command(&recipient, &sender, &command, &service_info.runner).await?; + + Ok(()) } } } From 31476f188bb2d435e612a52e25137ac72ba38c24 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 28 Oct 2024 08:39:47 +0000 Subject: [PATCH 35/51] Release 0.0.12 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bea4c1..2898c7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.12] - 2024-10-28 ### Added - Added support for keepalive messages so that connections are kept open @@ -56,6 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.12]: https://github.com/isambard-sc/openportal/releases/tag/0.0.12 [0.0.11]: https://github.com/isambard-sc/openportal/releases/tag/0.0.11 [0.0.10]: https://github.com/isambard-sc/openportal/releases/tag/0.0.10 [0.0.9]: https://github.com/isambard-sc/openportal/releases/tag/0.0.9 From 2a0566de7bc0c03e27e023ef55901cf3d226ebb0 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Mon, 28 Oct 2024 16:28:24 +0700 Subject: [PATCH 36/51] Renamed the slurm and cluster agents to cluster and clusters. Adding in containers for all of the agents, as they will be deployed as containerised workloads --- Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- cluster/src/main.rs | 181 ++++++++++++++++++++++++++-- {slurm => clusters}/Cargo.toml | 4 +- clusters/src/main.rs | 55 +++++++++ oci/cluster/Containerfile | 8 ++ oci/cluster/build.sh | 29 +++++ oci/clusters/Containerfile | 8 ++ oci/clusters/build.sh | 29 +++++ oci/filesystem/Containerfile | 8 ++ oci/filesystem/build.sh | 29 +++++ oci/freeipa/Containerfile | 8 ++ oci/freeipa/build.sh | 29 +++++ oci/platform/Containerfile | 8 ++ oci/platform/build.sh | 29 +++++ oci/provider/Containerfile | 8 ++ oci/provider/build.sh | 29 +++++ slurm/src/main.rs | 214 --------------------------------- templemeads/src/handler.rs | 10 +- 19 files changed, 458 insertions(+), 232 deletions(-) rename {slurm => clusters}/Cargo.toml (88%) create mode 100644 clusters/src/main.rs create mode 100644 oci/cluster/Containerfile create mode 100755 oci/cluster/build.sh create mode 100644 oci/clusters/Containerfile create mode 100755 oci/clusters/build.sh create mode 100644 oci/filesystem/Containerfile create mode 100755 oci/filesystem/build.sh create mode 100644 oci/freeipa/Containerfile create mode 100755 oci/freeipa/build.sh create mode 100644 oci/platform/Containerfile create mode 100755 oci/platform/build.sh create mode 100644 oci/provider/Containerfile create mode 100755 oci/provider/build.sh delete mode 100644 slurm/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index f7c5a24..17fe23d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ "bridge", "cluster", "filesystem", "freeipa", "paddington", "portal", - "provider", "python", "slurm", "templemeads", + "provider", "python", "cluster", "clusters", "templemeads", "docs/echo", "docs/job", "docs/cmdline/portal", "docs/cmdline/cluster" ] diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index c6c945c..460aeb3 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "op-cluster" version = "0.0.12" -description = "An example of an OpenPortal HPC cluster platform agent" +description = "An example of an OpenPortal cluster instance agent" edition = "2021" license = "MIT" homepage = "https://github.com/chryswoods/openportal/" diff --git a/cluster/src/main.rs b/cluster/src/main.rs index 860bf59..07780b9 100644 --- a/cluster/src/main.rs +++ b/cluster/src/main.rs @@ -3,15 +3,21 @@ use anyhow::Result; -use templemeads::agent::platform::{process_args, run, Defaults}; +use templemeads::agent; +use templemeads::agent::instance::{process_args, run, Defaults}; use templemeads::agent::Type as AgentType; +use templemeads::async_runnable; +use templemeads::grammar::Instruction::{AddUser, RemoveUser}; +use templemeads::grammar::{UserIdentifier, UserMapping}; +use templemeads::job::{Envelope, Job}; +use templemeads::Error; /// -/// Main function for the cluster platform agent +/// Main function for the slurm cluster instance agent /// -/// This purpose of this agent is to manage clusters, defined -/// as HPC batch clusters. It will manage the lifecycle of -/// the cluster, including creating and deleting the cluster +/// This purpose of this agent is to manage an individual instance +/// of a slurm batch cluster. It will manage the lifecycle of +/// users and projects on the cluster. /// #[tokio::main] async fn main() -> Result<()> { @@ -21,7 +27,7 @@ async fn main() -> Result<()> { // create the OpenPortal paddington defaults let defaults = Defaults::parse( - Some("cluster".to_owned()), + Some("slurm".to_owned()), Some( dirs::config_local_dir() .unwrap_or( @@ -29,14 +35,14 @@ async fn main() -> Result<()> { .expect("Could not parse fallback config directory."), ) .join("openportal") - .join("cluster-config.toml"), + .join("slurm-config.toml"), ), - Some("ws://localhost:8045".to_owned()), + Some("ws://localhost:8046".to_owned()), Some("127.0.0.1".to_owned()), - Some(8045), + Some(8046), None, None, - Some(AgentType::Platform), + Some(AgentType::Instance), ); // now parse the command line arguments to get the service configuration @@ -48,8 +54,161 @@ async fn main() -> Result<()> { } }; + async_runnable! { + /// + /// Runnable function that will be called when a job is received + /// by the agent + /// + pub async fn slurm_runner(envelope: Envelope) -> Result + { + tracing::info!("Using the slurm runner"); + + let me = envelope.recipient(); + let sender = envelope.sender(); + let mut job = envelope.job(); + + match job.instruction() { + AddUser(user) => { + // add the user to the slurm cluster + tracing::info!("Adding user to slurm cluster: {}", user); + let mapping = create_account(&me, &user).await?; + + job = job.running(Some("Step 1/3: Account created".to_string()))?; + job = job.update(&sender).await?; + + let homedir = create_directories(&me, &mapping).await?; + + job = job.running(Some("Step 2/3: Directories created".to_string()))?; + job = job.update(&sender).await?; + + let _ = update_homedir(&me, &user, &homedir).await?; + + job = job.completed(mapping)?; + } + RemoveUser(user) => { + // remove the user from the slurm cluster + tracing::info!("Removing user from slurm cluster: {}", user); + job = job.completed("User removed")?; + } + _ => { + tracing::error!("Unknown instruction: {:?}", job.instruction()); + return Err(Error::UnknownInstruction( + format!("Unknown instruction: {:?}", job.instruction()).to_string(), + )); + } + } + + Ok(job) + } + } + // run the agent - run(config).await?; + run(config, slurm_runner).await?; Ok(()) } + +async fn create_account(me: &str, user: &UserIdentifier) -> Result { + // find the Account agent + match agent::account().await { + Some(account) => { + // send the add_job to the account agent + let job = Job::parse(&format!("{}.{} add_user {}", me, account, user))? + .put(&account) + .await?; + + // Wait for the add_job to complete + let result = job.wait().await?.result::()?; + + match result { + Some(mapping) => { + tracing::info!("User added to account agent: {:?}", mapping); + Ok(mapping) + } + None => { + tracing::error!("Error creating the user's account: {:?}", job); + Err(Error::Call( + format!("Error creating the user's account: {:?}", job).to_string(), + )) + } + } + } + None => { + tracing::error!("No account agent found"); + Err(Error::MissingAgent( + "Cannot run the job because there is no account agent".to_string(), + )) + } + } +} + +async fn create_directories(me: &str, mapping: &UserMapping) -> Result { + // find the Filesystem agent + match agent::filesystem().await { + Some(filesystem) => { + // send the add_job to the filesystem agent + let job = Job::parse(&format!("{}.{} add_local_user {}", me, filesystem, mapping))? + .put(&filesystem) + .await?; + + // Wait for the add_job to complete + let result = job.wait().await?.result::()?; + + match result { + Some(homedir) => { + tracing::info!("Directories created for user: {:?}", mapping); + Ok(homedir) + } + None => { + tracing::error!("Error creating the user's directories: {:?}", job); + Err(Error::Call( + format!("Error creating the user's directories: {:?}", job).to_string(), + )) + } + } + } + None => { + tracing::error!("No filesystem agent found"); + Err(Error::MissingAgent( + "Cannot run the job because there is no filesystem agent".to_string(), + )) + } + } +} + +async fn update_homedir(me: &str, user: &UserIdentifier, homedir: &str) -> Result { + // find the Account agent + match agent::account().await { + Some(account) => { + // send the add_job to the account agent + let job = Job::parse(&format!( + "{}.{} update_homedir {} {}", + me, account, user, homedir + ))? + .put(&account) + .await?; + + // Wait for the add_job to complete + let result = job.wait().await?.result::()?; + + match result { + Some(homedir) => { + tracing::info!("User {} homedir updated: {:?}", user, homedir); + Ok(homedir) + } + None => { + tracing::error!("Error updating the user's homedir: {:?}", job); + Err(Error::Call( + format!("Error updating the user's homedir: {:?}", job).to_string(), + )) + } + } + } + None => { + tracing::error!("No account agent found"); + Err(Error::MissingAgent( + "Cannot run the job because there is no account agent".to_string(), + )) + } + } +} diff --git a/slurm/Cargo.toml b/clusters/Cargo.toml similarity index 88% rename from slurm/Cargo.toml rename to clusters/Cargo.toml index 97c1e7b..afc0e06 100644 --- a/slurm/Cargo.toml +++ b/clusters/Cargo.toml @@ -2,9 +2,9 @@ # SPDX-License-Identifier: CC0-1.0 [package] -name = "op-slurm" +name = "op-clusters" version = "0.0.12" -description = "An example of an OpenPortal Slurm cluster instance agent" +description = "An example of an OpenPortal cluster platform agent" edition = "2021" license = "MIT" homepage = "https://github.com/chryswoods/openportal/" diff --git a/clusters/src/main.rs b/clusters/src/main.rs new file mode 100644 index 0000000..860bf59 --- /dev/null +++ b/clusters/src/main.rs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: © 2024 Christopher Woods +// SPDX-License-Identifier: MIT + +use anyhow::Result; + +use templemeads::agent::platform::{process_args, run, Defaults}; +use templemeads::agent::Type as AgentType; + +/// +/// Main function for the cluster platform agent +/// +/// This purpose of this agent is to manage clusters, defined +/// as HPC batch clusters. It will manage the lifecycle of +/// the cluster, including creating and deleting the cluster +/// +#[tokio::main] +async fn main() -> Result<()> { + // start tracing + let subscriber = tracing_subscriber::FmtSubscriber::new(); + tracing::subscriber::set_global_default(subscriber)?; + + // create the OpenPortal paddington defaults + let defaults = Defaults::parse( + Some("cluster".to_owned()), + Some( + dirs::config_local_dir() + .unwrap_or( + ".".parse() + .expect("Could not parse fallback config directory."), + ) + .join("openportal") + .join("cluster-config.toml"), + ), + Some("ws://localhost:8045".to_owned()), + Some("127.0.0.1".to_owned()), + Some(8045), + None, + None, + Some(AgentType::Platform), + ); + + // now parse the command line arguments to get the service configuration + let config = match process_args(&defaults).await? { + Some(config) => config, + None => { + // Not running the service, so can safely exit + return Ok(()); + } + }; + + // run the agent + run(config).await?; + + Ok(()) +} diff --git a/oci/cluster/Containerfile b/oci/cluster/Containerfile new file mode 100644 index 0000000..94bf78a --- /dev/null +++ b/oci/cluster/Containerfile @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT + +FROM gcr.io/distroless/static-debian12 +COPY op-cluster / +USER 65534:65534 +ENTRYPOINT ["/op-cluster"] diff --git a/oci/cluster/build.sh b/oci/cluster/build.sh new file mode 100755 index 0000000..6240f84 --- /dev/null +++ b/oci/cluster/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +set -euo pipefail + +# Build the project and create an OCI image containing it. + +function artifact_path { + echo "${1}" | jq --raw-output 'select(.reason == "compiler-artifact") | select(.target.name == "'"${2}"'") | .executable' +} + +out=$(cargo build --package op-cluster --target=x86_64-unknown-linux-musl --message-format=json ${@-}) +cp "$(artifact_path "${out}" "op-cluster")" oci/cluster + +cd oci/cluster + +version=$(./op-cluster --version | tail -n1 | cut -d' ' -f 2) +image_id=$( + podman build . --tag=op-cluster:latest --tag=op-cluster:"${version}" \ + --annotation="org.opencontainers.image.source=https://github.com/isambard-sc/openportal" \ + --annotation="org.opencontainers.image.description=OpenPortal" \ + --annotation="org.opencontainers.image.licenses=MIT" \ + | tee /dev/fd/2 \ + | tail -n1 +) +rm op-cluster +echo "Built op-cluster image:" 1>&2 +echo "${image_id}" diff --git a/oci/clusters/Containerfile b/oci/clusters/Containerfile new file mode 100644 index 0000000..d02ebf0 --- /dev/null +++ b/oci/clusters/Containerfile @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT + +FROM gcr.io/distroless/static-debian12 +COPY op-clusters / +USER 65534:65534 +ENTRYPOINT ["/op-clusters"] diff --git a/oci/clusters/build.sh b/oci/clusters/build.sh new file mode 100755 index 0000000..3a16855 --- /dev/null +++ b/oci/clusters/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +set -euo pipefail + +# Build the project and create an OCI image containing it. + +function artifact_path { + echo "${1}" | jq --raw-output 'select(.reason == "compiler-artifact") | select(.target.name == "'"${2}"'") | .executable' +} + +out=$(cargo build --package op-clusters --target=x86_64-unknown-linux-musl --message-format=json ${@-}) +cp "$(artifact_path "${out}" "op-clusters")" oci/clusters + +cd oci/clusters + +version=$(./op-clusters --version | tail -n1 | cut -d' ' -f 2) +image_id=$( + podman build . --tag=op-clusters:latest --tag=op-clusters:"${version}" \ + --annotation="org.opencontainers.image.source=https://github.com/isambard-sc/openportal" \ + --annotation="org.opencontainers.image.description=OpenPortal" \ + --annotation="org.opencontainers.image.licenses=MIT" \ + | tee /dev/fd/2 \ + | tail -n1 +) +rm op-clusters +echo "Built op-clusters image:" 1>&2 +echo "${image_id}" diff --git a/oci/filesystem/Containerfile b/oci/filesystem/Containerfile new file mode 100644 index 0000000..470f826 --- /dev/null +++ b/oci/filesystem/Containerfile @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT + +FROM gcr.io/distroless/static-debian12 +COPY op-filesystem / +USER 65534:65534 +ENTRYPOINT ["/op-filesystem"] diff --git a/oci/filesystem/build.sh b/oci/filesystem/build.sh new file mode 100755 index 0000000..51edc73 --- /dev/null +++ b/oci/filesystem/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +set -euo pipefail + +# Build the project and create an OCI image containing it. + +function artifact_path { + echo "${1}" | jq --raw-output 'select(.reason == "compiler-artifact") | select(.target.name == "'"${2}"'") | .executable' +} + +out=$(cargo build --package op-filesystem --target=x86_64-unknown-linux-musl --message-format=json ${@-}) +cp "$(artifact_path "${out}" "op-filesystem")" oci/filesystem + +cd oci/filesystem + +version=$(./op-filesystem --version | tail -n1 | cut -d' ' -f 2) +image_id=$( + podman build . --tag=op-filesystem:latest --tag=op-filesystem:"${version}" \ + --annotation="org.opencontainers.image.source=https://github.com/isambard-sc/openportal" \ + --annotation="org.opencontainers.image.description=OpenPortal" \ + --annotation="org.opencontainers.image.licenses=MIT" \ + | tee /dev/fd/2 \ + | tail -n1 +) +rm op-filesystem +echo "Built op-filesystem image:" 1>&2 +echo "${image_id}" diff --git a/oci/freeipa/Containerfile b/oci/freeipa/Containerfile new file mode 100644 index 0000000..5260be5 --- /dev/null +++ b/oci/freeipa/Containerfile @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT + +FROM gcr.io/distroless/static-debian12 +COPY op-freeipa / +USER 65534:65534 +ENTRYPOINT ["/op-freeipa"] diff --git a/oci/freeipa/build.sh b/oci/freeipa/build.sh new file mode 100755 index 0000000..d026a35 --- /dev/null +++ b/oci/freeipa/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +set -euo pipefail + +# Build the project and create an OCI image containing it. + +function artifact_path { + echo "${1}" | jq --raw-output 'select(.reason == "compiler-artifact") | select(.target.name == "'"${2}"'") | .executable' +} + +out=$(cargo build --package op-freeipa --target=x86_64-unknown-linux-musl --message-format=json ${@-}) +cp "$(artifact_path "${out}" "op-freeipa")" oci/freeipa + +cd oci/freeipa + +version=$(./op-freeipa --version | tail -n1 | cut -d' ' -f 2) +image_id=$( + podman build . --tag=op-freeipa:latest --tag=op-freeipa:"${version}" \ + --annotation="org.opencontainers.image.source=https://github.com/isambard-sc/openportal" \ + --annotation="org.opencontainers.image.description=OpenPortal" \ + --annotation="org.opencontainers.image.licenses=MIT" \ + | tee /dev/fd/2 \ + | tail -n1 +) +rm op-freeipa +echo "Built op-freeipa image:" 1>&2 +echo "${image_id}" diff --git a/oci/platform/Containerfile b/oci/platform/Containerfile new file mode 100644 index 0000000..556b12d --- /dev/null +++ b/oci/platform/Containerfile @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT + +FROM gcr.io/distroless/static-debian12 +COPY op-platform / +USER 65534:65534 +ENTRYPOINT ["/op-platform"] diff --git a/oci/platform/build.sh b/oci/platform/build.sh new file mode 100755 index 0000000..79123a0 --- /dev/null +++ b/oci/platform/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +set -euo pipefail + +# Build the project and create an OCI image containing it. + +function artifact_path { + echo "${1}" | jq --raw-output 'select(.reason == "compiler-artifact") | select(.target.name == "'"${2}"'") | .executable' +} + +out=$(cargo build --package op-platform --target=x86_64-unknown-linux-musl --message-format=json ${@-}) +cp "$(artifact_path "${out}" "op-platform")" oci/platform + +cd oci/platform + +version=$(./op-platform --version | tail -n1 | cut -d' ' -f 2) +image_id=$( + podman build . --tag=op-platform:latest --tag=op-platform:"${version}" \ + --annotation="org.opencontainers.image.source=https://github.com/isambard-sc/openportal" \ + --annotation="org.opencontainers.image.description=OpenPortal" \ + --annotation="org.opencontainers.image.licenses=MIT" \ + | tee /dev/fd/2 \ + | tail -n1 +) +rm op-platform +echo "Built op-platform image:" 1>&2 +echo "${image_id}" diff --git a/oci/provider/Containerfile b/oci/provider/Containerfile new file mode 100644 index 0000000..2b5e771 --- /dev/null +++ b/oci/provider/Containerfile @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT + +FROM gcr.io/distroless/static-debian12 +COPY op-provider / +USER 65534:65534 +ENTRYPOINT ["/op-provider"] diff --git a/oci/provider/build.sh b/oci/provider/build.sh new file mode 100755 index 0000000..ff37267 --- /dev/null +++ b/oci/provider/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +set -euo pipefail + +# Build the project and create an OCI image containing it. + +function artifact_path { + echo "${1}" | jq --raw-output 'select(.reason == "compiler-artifact") | select(.target.name == "'"${2}"'") | .executable' +} + +out=$(cargo build --package op-provider --target=x86_64-unknown-linux-musl --message-format=json ${@-}) +cp "$(artifact_path "${out}" "op-provider")" oci/provider + +cd oci/provider + +version=$(./op-provider --version | tail -n1 | cut -d' ' -f 2) +image_id=$( + podman build . --tag=op-provider:latest --tag=op-provider:"${version}" \ + --annotation="org.opencontainers.image.source=https://github.com/isambard-sc/openportal" \ + --annotation="org.opencontainers.image.description=OpenPortal" \ + --annotation="org.opencontainers.image.licenses=MIT" \ + | tee /dev/fd/2 \ + | tail -n1 +) +rm op-provider +echo "Built op-provider image:" 1>&2 +echo "${image_id}" diff --git a/slurm/src/main.rs b/slurm/src/main.rs deleted file mode 100644 index 07780b9..0000000 --- a/slurm/src/main.rs +++ /dev/null @@ -1,214 +0,0 @@ -// SPDX-FileCopyrightText: © 2024 Christopher Woods -// SPDX-License-Identifier: MIT - -use anyhow::Result; - -use templemeads::agent; -use templemeads::agent::instance::{process_args, run, Defaults}; -use templemeads::agent::Type as AgentType; -use templemeads::async_runnable; -use templemeads::grammar::Instruction::{AddUser, RemoveUser}; -use templemeads::grammar::{UserIdentifier, UserMapping}; -use templemeads::job::{Envelope, Job}; -use templemeads::Error; - -/// -/// Main function for the slurm cluster instance agent -/// -/// This purpose of this agent is to manage an individual instance -/// of a slurm batch cluster. It will manage the lifecycle of -/// users and projects on the cluster. -/// -#[tokio::main] -async fn main() -> Result<()> { - // start tracing - let subscriber = tracing_subscriber::FmtSubscriber::new(); - tracing::subscriber::set_global_default(subscriber)?; - - // create the OpenPortal paddington defaults - let defaults = Defaults::parse( - Some("slurm".to_owned()), - Some( - dirs::config_local_dir() - .unwrap_or( - ".".parse() - .expect("Could not parse fallback config directory."), - ) - .join("openportal") - .join("slurm-config.toml"), - ), - Some("ws://localhost:8046".to_owned()), - Some("127.0.0.1".to_owned()), - Some(8046), - None, - None, - Some(AgentType::Instance), - ); - - // now parse the command line arguments to get the service configuration - let config = match process_args(&defaults).await? { - Some(config) => config, - None => { - // Not running the service, so can safely exit - return Ok(()); - } - }; - - async_runnable! { - /// - /// Runnable function that will be called when a job is received - /// by the agent - /// - pub async fn slurm_runner(envelope: Envelope) -> Result - { - tracing::info!("Using the slurm runner"); - - let me = envelope.recipient(); - let sender = envelope.sender(); - let mut job = envelope.job(); - - match job.instruction() { - AddUser(user) => { - // add the user to the slurm cluster - tracing::info!("Adding user to slurm cluster: {}", user); - let mapping = create_account(&me, &user).await?; - - job = job.running(Some("Step 1/3: Account created".to_string()))?; - job = job.update(&sender).await?; - - let homedir = create_directories(&me, &mapping).await?; - - job = job.running(Some("Step 2/3: Directories created".to_string()))?; - job = job.update(&sender).await?; - - let _ = update_homedir(&me, &user, &homedir).await?; - - job = job.completed(mapping)?; - } - RemoveUser(user) => { - // remove the user from the slurm cluster - tracing::info!("Removing user from slurm cluster: {}", user); - job = job.completed("User removed")?; - } - _ => { - tracing::error!("Unknown instruction: {:?}", job.instruction()); - return Err(Error::UnknownInstruction( - format!("Unknown instruction: {:?}", job.instruction()).to_string(), - )); - } - } - - Ok(job) - } - } - - // run the agent - run(config, slurm_runner).await?; - - Ok(()) -} - -async fn create_account(me: &str, user: &UserIdentifier) -> Result { - // find the Account agent - match agent::account().await { - Some(account) => { - // send the add_job to the account agent - let job = Job::parse(&format!("{}.{} add_user {}", me, account, user))? - .put(&account) - .await?; - - // Wait for the add_job to complete - let result = job.wait().await?.result::()?; - - match result { - Some(mapping) => { - tracing::info!("User added to account agent: {:?}", mapping); - Ok(mapping) - } - None => { - tracing::error!("Error creating the user's account: {:?}", job); - Err(Error::Call( - format!("Error creating the user's account: {:?}", job).to_string(), - )) - } - } - } - None => { - tracing::error!("No account agent found"); - Err(Error::MissingAgent( - "Cannot run the job because there is no account agent".to_string(), - )) - } - } -} - -async fn create_directories(me: &str, mapping: &UserMapping) -> Result { - // find the Filesystem agent - match agent::filesystem().await { - Some(filesystem) => { - // send the add_job to the filesystem agent - let job = Job::parse(&format!("{}.{} add_local_user {}", me, filesystem, mapping))? - .put(&filesystem) - .await?; - - // Wait for the add_job to complete - let result = job.wait().await?.result::()?; - - match result { - Some(homedir) => { - tracing::info!("Directories created for user: {:?}", mapping); - Ok(homedir) - } - None => { - tracing::error!("Error creating the user's directories: {:?}", job); - Err(Error::Call( - format!("Error creating the user's directories: {:?}", job).to_string(), - )) - } - } - } - None => { - tracing::error!("No filesystem agent found"); - Err(Error::MissingAgent( - "Cannot run the job because there is no filesystem agent".to_string(), - )) - } - } -} - -async fn update_homedir(me: &str, user: &UserIdentifier, homedir: &str) -> Result { - // find the Account agent - match agent::account().await { - Some(account) => { - // send the add_job to the account agent - let job = Job::parse(&format!( - "{}.{} update_homedir {} {}", - me, account, user, homedir - ))? - .put(&account) - .await?; - - // Wait for the add_job to complete - let result = job.wait().await?.result::()?; - - match result { - Some(homedir) => { - tracing::info!("User {} homedir updated: {:?}", user, homedir); - Ok(homedir) - } - None => { - tracing::error!("Error updating the user's homedir: {:?}", job); - Err(Error::Call( - format!("Error updating the user's homedir: {:?}", job).to_string(), - )) - } - } - } - None => { - tracing::error!("No account agent found"); - Err(Error::MissingAgent( - "Cannot run the job because there is no account agent".to_string(), - )) - } - } -} diff --git a/templemeads/src/handler.rs b/templemeads/src/handler.rs index 76af446..d3f85ff 100644 --- a/templemeads/src/handler.rs +++ b/templemeads/src/handler.rs @@ -190,11 +190,15 @@ async_message_handler! { return Err(Error::Delivery(format!("Recipient {} does not match service {}", recipient, service_info.service)).into()); } - tracing::info!("Keep alive message received from {}", sender); - // wait 20 seconds and send a keep alive message back tokio::time::sleep(tokio::time::Duration::from_secs(20)).await; - paddington::send(Message::keepalive(&sender)).await?; + + match paddington::send(Message::keepalive(&sender)).await { + Ok(_) => {} + Err(e) => { + tracing::warn!("Error sending keepalive message to {} : {}", sender, e); + } + } Ok(()) } From b6db6a5efd6aa3d56a4311b7ad564e55bfc074f6 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 1 Nov 2024 12:36:22 +0700 Subject: [PATCH 37/51] Added helm charts for everything and finished renaming some of the agents --- .github/workflows/build.yml | 173 ++++++++++++++--- .github/workflows/release.yml | 224 +++++++++++++++++++++- CHANGELOG.md | 5 + bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- cluster/src/main.rs | 22 +-- clusters/Cargo.toml | 2 +- clusters/src/main.rs | 4 +- docs/README.md | 8 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- freeipa/README.md | 30 +++ helm/cluster/Chart.yaml | 10 + helm/cluster/templates/_helpers.yaml | 10 + helm/cluster/templates/deployment.yaml | 59 ++++++ helm/cluster/templates/service.yaml | 20 ++ helm/cluster/values.yaml | 13 ++ helm/clusters/Chart.yaml | 10 + helm/clusters/templates/_helpers.yaml | 10 + helm/clusters/templates/deployment.yaml | 59 ++++++ helm/clusters/templates/service.yaml | 20 ++ helm/clusters/values.yaml | 13 ++ helm/filesystem/Chart.yaml | 10 + helm/filesystem/templates/_helpers.yaml | 10 + helm/filesystem/templates/deployment.yaml | 59 ++++++ helm/filesystem/templates/service.yaml | 20 ++ helm/filesystem/values.yaml | 13 ++ helm/freeipa/Chart.yaml | 10 + helm/freeipa/templates/_helpers.yaml | 10 + helm/freeipa/templates/deployment.yaml | 59 ++++++ helm/freeipa/templates/service.yaml | 20 ++ helm/freeipa/values.yaml | 13 ++ helm/platform/Chart.yaml | 10 + helm/platform/templates/_helpers.yaml | 10 + helm/platform/templates/deployment.yaml | 59 ++++++ helm/platform/templates/service.yaml | 20 ++ helm/platform/values.yaml | 13 ++ helm/provider/Chart.yaml | 10 + helm/provider/templates/_helpers.yaml | 10 + helm/provider/templates/deployment.yaml | 59 ++++++ helm/provider/templates/service.yaml | 20 ++ helm/provider/values.yaml | 13 ++ paddington/Cargo.toml | 2 +- portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- 51 files changed, 1098 insertions(+), 68 deletions(-) create mode 100644 freeipa/README.md create mode 100644 helm/cluster/Chart.yaml create mode 100644 helm/cluster/templates/_helpers.yaml create mode 100644 helm/cluster/templates/deployment.yaml create mode 100644 helm/cluster/templates/service.yaml create mode 100644 helm/cluster/values.yaml create mode 100644 helm/clusters/Chart.yaml create mode 100644 helm/clusters/templates/_helpers.yaml create mode 100644 helm/clusters/templates/deployment.yaml create mode 100644 helm/clusters/templates/service.yaml create mode 100644 helm/clusters/values.yaml create mode 100644 helm/filesystem/Chart.yaml create mode 100644 helm/filesystem/templates/_helpers.yaml create mode 100644 helm/filesystem/templates/deployment.yaml create mode 100644 helm/filesystem/templates/service.yaml create mode 100644 helm/filesystem/values.yaml create mode 100644 helm/freeipa/Chart.yaml create mode 100644 helm/freeipa/templates/_helpers.yaml create mode 100644 helm/freeipa/templates/deployment.yaml create mode 100644 helm/freeipa/templates/service.yaml create mode 100644 helm/freeipa/values.yaml create mode 100644 helm/platform/Chart.yaml create mode 100644 helm/platform/templates/_helpers.yaml create mode 100644 helm/platform/templates/deployment.yaml create mode 100644 helm/platform/templates/service.yaml create mode 100644 helm/platform/values.yaml create mode 100644 helm/provider/Chart.yaml create mode 100644 helm/provider/templates/_helpers.yaml create mode 100644 helm/provider/templates/deployment.yaml create mode 100644 helm/provider/templates/service.yaml create mode 100644 helm/provider/values.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c58ad33..1064cfa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,10 +37,22 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build run: cargo build --release - - name: Build op-portal OCI image - run: oci/portal/build.sh --release - name: Build op-bridge OCI image run: oci/bridge/build.sh --release + - name: Build op-cluster OCI image + run: oci/cluster/build.sh --release + - name: Build op-clusters OCI image + run: oci/clusters/build.sh --release + - name: Build op-filesystem OCI image + run: oci/filesystem/build.sh --release + - name: Build op-freeipa OCI image + run: oci/freeipa/build.sh --release + - name: Build op-platform OCI image + run: oci/platform/build.sh --release + - name: Build op-portal OCI image + run: oci/portal/build.sh --release + - name: Build op-provider OCI image + run: oci/provider/build.sh --release - name: Get version id: get_version run: | @@ -53,12 +65,30 @@ jobs: else echo version="${{ steps.get_version.outputs.version }}.${{ github.ref_name }}" >> "${GITHUB_OUTPUT}" fi + - name: package op-bridge helm chart + run: | + helm package helm/bridge --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" + - name: package op-cluster helm chart + run: | + helm package helm/cluster --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" + - name: package op-clusters helm chart + run: | + helm package helm/clusters --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" + - name: package op-filesystem helm chart + run: | + helm package helm/filesystem --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" + - name: package op-freeipa helm chart + run: | + helm package helm/freeipa --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" + - name: package op-platform helm chart + run: | + helm package helm/platform --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" - name: package op-portal helm chart run: | helm package helm/portal --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" - - name: package op-bridge helm chart + - name: package op-provider helm chart run: | - helm package helm/bridge --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" + helm package helm/provider --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" - name: Log in to GHCR uses: redhat-actions/podman-login@v1 with: @@ -69,6 +99,48 @@ jobs: run: echo $GITHUB_TOKEN | helm registry login "ghcr.io/${{ github.repository_owner }}" --username "${{ github.actor }}" --password-stdin env: GITHUB_TOKEN: "${{ github.token }}" + - name: Publish op-bridge OCI image + id: push-bridge-to-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + image: op-bridge + tags: ${{ steps.get_version.outputs.version }} + registry: ghcr.io/${{ github.repository_owner }} + - name: Publish op-cluster OCI image + id: push-cluster-to-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + image: op-cluster + tags: ${{ steps.get_version.outputs.version }} + registry: ghcr.io/${{ github.repository_owner }} + - name: Publish op-clusters OCI image + id: push-clusters-to-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + image: op-clusters + tags: ${{ steps.get_version.outputs.version }} + registry: ghcr.io/${{ github.repository_owner }} + - name: Publish op-filesystem OCI image + id: push-filesystem-to-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + image: op-filesystem + tags: ${{ steps.get_version.outputs.version }} + registry: ghcr.io/${{ github.repository_owner }} + - name: Publish op-freeipa OCI image + id: push-freeipa-to-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + image: op-freeipa + tags: ${{ steps.get_version.outputs.version }} + registry: ghcr.io/${{ github.repository_owner }} + - name: Publish op-platform OCI image + id: push-platform-to-ghcr + uses: redhat-actions/push-to-registry@v2 + with: + image: op-platform + tags: ${{ steps.get_version.outputs.version }} + registry: ghcr.io/${{ github.repository_owner }} - name: Publish op-portal OCI image id: push-portal-to-ghcr uses: redhat-actions/push-to-registry@v2 @@ -76,50 +148,95 @@ jobs: image: op-portal tags: ${{ steps.get_version.outputs.version }} registry: ghcr.io/${{ github.repository_owner }} - - name: Publish op-bridge OCI image - id: push-bridge-to-ghcr + - name: Publish op-provider OCI image + id: push-provider-to-ghcr uses: redhat-actions/push-to-registry@v2 with: - image: op-bridge + image: op-provider tags: ${{ steps.get_version.outputs.version }} registry: ghcr.io/${{ github.repository_owner }} + - name: Attest op-bridge image + uses: actions/attest-build-provenance@v1 + id: attest-bridge + with: + subject-name: ghcr.io/${{ github.repository_owner }}/op-bridge + subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + push-to-registry: true + - name: Attest op-cluster image + uses: actions/attest-build-provenance@v1 + id: attest-cluster + with: + subject-name: ghcr.io/${{ github.repository_owner }}/op-cluster + subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + push-to-registry: true + - name: Attest op-clusters image + uses: actions/attest-build-provenance@v1 + id: attest-clusters + with: + subject-name: ghcr.io/${{ github.repository_owner }}/op-clusters + subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + push-to-registry: true + - name: Attest op-filesystem image + uses: actions/attest-build-provenance@v1 + id: attest-filesystem + with: + subject-name: ghcr.io/${{ github.repository_owner }}/op-filesystem + subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + push-to-registry: true + - name: Attest op-freeipa image + uses: actions/attest-build-provenance@v1 + id: attest-freeipa + with: + subject-name: ghcr.io/${{ github.repository_owner }}/op-freeipa + subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + push-to-registry: true + - name: Attest op-platform image + uses: actions/attest-build-provenance@v1 + id: attest-platform + with: + subject-name: ghcr.io/${{ github.repository_owner }}/op-platform + subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + push-to-registry: true - name: Attest op-portal image uses: actions/attest-build-provenance@v1 id: attest-portal with: subject-name: ghcr.io/${{ github.repository_owner }}/op-portal - subject-digest: ${{ steps.push-portal-to-ghcr.outputs.digest }} + subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} push-to-registry: true - - name: Attest op-bridge image + - name: Attest op-provider image uses: actions/attest-build-provenance@v1 - id: attest-bridge + id: attest-provider with: - subject-name: ghcr.io/${{ github.repository_owner }}/op-bridge + subject-name: ghcr.io/${{ github.repository_owner }}/op-provider subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} push-to-registry: true - - name: Push op-portal Helm chart - run: helm push "./op-portal-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - name: Push op-bridge Helm chart run: helm push "./op-bridge-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - - name: Store portal artefact - uses: actions/upload-artifact@v4 - with: - name: op-portal - path: | - target/release/op-portal - - name: Store bridge artefact - uses: actions/upload-artifact@v4 - with: - name: op-bridge - path: | - target/release/op-bridge - - name: Store other agent artefacts + - name: Push op-cluster Helm chart + run: helm push "./op-cluster-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + - name: Push op-clusters Helm chart + run: helm push "./op-clusters-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + - name: Push op-filesystem Helm chart + run: helm push "./op-filesystem-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + - name: Push op-freeipa Helm chart + run: helm push "./op-freeipa-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + - name: Push op-platform Helm chart + run: helm push "./op-platform-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + - name: Push op-portal Helm chart + run: helm push "./op-portal-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + - name: Push op-provider Helm chart + run: helm push "./op-provider-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" + - name: Store agent artefacts uses: actions/upload-artifact@v4 with: name: openportal-agents path: | - target/release/op-provider + target/release/op-bridge target/release/op-cluster - target/release/op-slurm + target/release/op-clusters target/release/op-filesystem target/release/op-freeipa + target/release/op-platform + target/release/op-portal + target/release/op-provider diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0276112..c190eac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -116,6 +116,210 @@ jobs: attestations: write id-token: write + attest-bridge: + name: Attest Bridge + needs: build-release + runs-on: ubuntu-latest + permissions: + contents: read + attestations: write + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install cargo-sbom + uses: taiki-e/install-action@v2 + with: + tool: cargo-sbom + - name: Generate SBOM + run: cargo sbom --cargo-package op-bridge --output-format=spdx_json_2_3 > sbom-bridge.spdx.json + - name: Fetch release artefacts + uses: actions/download-artifact@v4 + with: + pattern: op-bridge + merge-multiple: true + - name: Attest SBOM + uses: actions/attest-sbom@v1 + with: + subject-path: op-bridge + sbom-path: sbom-bridge.spdx.json + - name: Store SBOM + uses: actions/upload-artifact@v4 + with: + name: sbom-bridge.spdx.json + path: sbom-bridge.spdx.json + + attest-cluster: + name: Attest Cluster + needs: build-release + runs-on: ubuntu-latest + permissions: + contents: read + attestations: write + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install cargo-sbom + uses: taiki-e/install-action@v2 + with: + tool: cargo-sbom + - name: Generate SBOM + run: cargo sbom --cargo-package op-cluster --output-format=spdx_json_2_3 > sbom-cluster.spdx.json + - name: Fetch release artefacts + uses: actions/download-artifact@v4 + with: + pattern: op-cluster + merge-multiple: true + - name: Attest SBOM + uses: actions/attest-sbom@v1 + with: + subject-path: op-cluster + sbom-path: sbom-cluster.spdx.json + - name: Store SBOM + uses: actions/upload-artifact@v4 + with: + name: sbom-cluster.spdx.json + path: sbom-cluster.spdx.json + + attest-clusters: + name: Attest Clusters + needs: build-release + runs-on: ubuntu-latest + permissions: + contents: read + attestations: write + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install cargo-sbom + uses: taiki-e/install-action@v2 + with: + tool: cargo-sbom + - name: Generate SBOM + run: cargo sbom --cargo-package op-clusters --output-format=spdx_json_2_3 > sbom-clusters.spdx.json + - name: Fetch release artefacts + uses: actions/download-artifact@v4 + with: + pattern: op-clusters + merge-multiple: true + - name: Attest SBOM + uses: actions/attest-sbom@v1 + with: + subject-path: op-clusters + sbom-path: sbom-clusters.spdx.json + - name: Store SBOM + uses: actions/upload-artifact@v4 + with: + name: sbom-clusters.spdx.json + path: sbom-clusters.spdx.json + + attest-filesystem: + name: Attest Filesystem + needs: build-release + runs-on: ubuntu-latest + permissions: + contents: read + attestations: write + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install cargo-sbom + uses: taiki-e/install-action@v2 + with: + tool: cargo-sbom + - name: Generate SBOM + run: cargo sbom --cargo-package op-filesystem --output-format=spdx_json_2_3 > sbom-filesystem.spdx.json + - name: Fetch release artefacts + uses: actions/download-artifact@v4 + with: + pattern: op-filesystem + merge-multiple: true + - name: Attest SBOM + uses: actions/attest-sbom@v1 + with: + subject-path: op-filesystem + sbom-path: sbom-filesystem.spdx.json + - name: Store SBOM + uses: actions/upload-artifact@v4 + with: + name: sbom-filesystem.spdx.json + path: sbom-filesystem.spdx.json + + attest-freeipa: + name: Attest FreeIPA + needs: build-release + runs-on: ubuntu-latest + permissions: + contents: read + attestations: write + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install cargo-sbom + uses: taiki-e/install-action@v2 + with: + tool: cargo-sbom + - name: Generate SBOM + run: cargo sbom --cargo-package op-freeipa --output-format=spdx_json_2_3 > sbom-freeipa.spdx.json + - name: Fetch release artefacts + uses: actions/download-artifact@v4 + with: + pattern: op-freeipa + merge-multiple: true + - name: Attest SBOM + uses: actions/attest-sbom@v1 + with: + subject-path: op-freeipa + sbom-path: sbom-freeipa.spdx.json + - name: Store SBOM + uses: actions/upload-artifact@v4 + with: + name: sbom-freeipa.spdx.json + path: sbom-freeipa.spdx.json + + attest-platform: + name: Attest Platform + needs: build-release + runs-on: ubuntu-latest + permissions: + contents: read + attestations: write + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Install toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install cargo-sbom + uses: taiki-e/install-action@v2 + with: + tool: cargo-sbom + - name: Generate SBOM + run: cargo sbom --cargo-package op-platform --output-format=spdx_json_2_3 > sbom-platform.spdx.json + - name: Fetch release artefacts + uses: actions/download-artifact@v4 + with: + pattern: op-platform + merge-multiple: true + - name: Attest SBOM + uses: actions/attest-sbom@v1 + with: + subject-path: op-platform + sbom-path: sbom-platform.spdx.json + - name: Store SBOM + uses: actions/upload-artifact@v4 + with: + name: sbom-platform.spdx.json + path: sbom-platform.spdx.json + attest-portal: name: Attest Portal needs: build-release @@ -150,8 +354,8 @@ jobs: name: sbom-portal.spdx.json path: sbom-portal.spdx.json - attest-bridge: - name: Attest Bridge + attest-provider: + name: Attest Provider needs: build-release runs-on: ubuntu-latest permissions: @@ -167,26 +371,28 @@ jobs: with: tool: cargo-sbom - name: Generate SBOM - run: cargo sbom --cargo-package op-bridge --output-format=spdx_json_2_3 > sbom-bridge.spdx.json + run: cargo sbom --cargo-package op-provider --output-format=spdx_json_2_3 > sbom-provider.spdx.json - name: Fetch release artefacts uses: actions/download-artifact@v4 with: - pattern: op-bridge + pattern: op-provider merge-multiple: true - name: Attest SBOM uses: actions/attest-sbom@v1 with: - subject-path: op-bridge - sbom-path: sbom-bridge.spdx.json + subject-path: op-provider + sbom-path: sbom-provider.spdx.json - name: Store SBOM uses: actions/upload-artifact@v4 with: - name: sbom-bridge.spdx.json - path: sbom-bridge.spdx.json + name: sbom-provider.spdx.json + path: sbom-provider.spdx.json make-release: name: Make release ${{ needs.tag-release.outputs.ref }} - needs: [build-release, tag-release, attest-bridge, attest-portal] + needs: [build-release, tag-release, attest-bridge, attest-cluster, + attest-clusters, attest-filesystem, attest-freeipa, attest-platform, + attest-portal, attest-provider] runs-on: ubuntu-latest permissions: contents: write diff --git a/CHANGELOG.md b/CHANGELOG.md index 2898c7a..8b295f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Added +- Changed the names of the cluster instance and platform agents to `cluster` and `clusters`, + as they don't need to be named after slurm (and would cause confusion with the slurm agent). +- Added OCI images and helm charts for all agents +- Added instructions on how to configure the freeipa agent ## [0.0.12] - 2024-10-28 ### Added diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index 330a416..81332de 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.12" +version = "0.0.14" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index 460aeb3..b18d693 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.12" +version = "0.0.14" description = "An example of an OpenPortal cluster instance agent" edition = "2021" license = "MIT" diff --git a/cluster/src/main.rs b/cluster/src/main.rs index 07780b9..3d06e63 100644 --- a/cluster/src/main.rs +++ b/cluster/src/main.rs @@ -13,10 +13,10 @@ use templemeads::job::{Envelope, Job}; use templemeads::Error; /// -/// Main function for the slurm cluster instance agent +/// Main function for the cluster instance agent /// /// This purpose of this agent is to manage an individual instance -/// of a slurm batch cluster. It will manage the lifecycle of +/// of a batch cluster. It will manage the lifecycle of /// users and projects on the cluster. /// #[tokio::main] @@ -27,7 +27,7 @@ async fn main() -> Result<()> { // create the OpenPortal paddington defaults let defaults = Defaults::parse( - Some("slurm".to_owned()), + Some("cluster".to_owned()), Some( dirs::config_local_dir() .unwrap_or( @@ -35,7 +35,7 @@ async fn main() -> Result<()> { .expect("Could not parse fallback config directory."), ) .join("openportal") - .join("slurm-config.toml"), + .join("cluster-config.toml"), ), Some("ws://localhost:8046".to_owned()), Some("127.0.0.1".to_owned()), @@ -59,9 +59,9 @@ async fn main() -> Result<()> { /// Runnable function that will be called when a job is received /// by the agent /// - pub async fn slurm_runner(envelope: Envelope) -> Result + pub async fn cluster_runner(envelope: Envelope) -> Result { - tracing::info!("Using the slurm runner"); + tracing::info!("Using the cluster runner"); let me = envelope.recipient(); let sender = envelope.sender(); @@ -69,8 +69,8 @@ async fn main() -> Result<()> { match job.instruction() { AddUser(user) => { - // add the user to the slurm cluster - tracing::info!("Adding user to slurm cluster: {}", user); + // add the user to the cluster + tracing::info!("Adding user to cluster: {}", user); let mapping = create_account(&me, &user).await?; job = job.running(Some("Step 1/3: Account created".to_string()))?; @@ -86,8 +86,8 @@ async fn main() -> Result<()> { job = job.completed(mapping)?; } RemoveUser(user) => { - // remove the user from the slurm cluster - tracing::info!("Removing user from slurm cluster: {}", user); + // remove the user from the cluster + tracing::info!("Removing user from cluster: {}", user); job = job.completed("User removed")?; } _ => { @@ -103,7 +103,7 @@ async fn main() -> Result<()> { } // run the agent - run(config, slurm_runner).await?; + run(config, cluster_runner).await?; Ok(()) } diff --git a/clusters/Cargo.toml b/clusters/Cargo.toml index afc0e06..ea5822c 100644 --- a/clusters/Cargo.toml +++ b/clusters/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-clusters" -version = "0.0.12" +version = "0.0.14" description = "An example of an OpenPortal cluster platform agent" edition = "2021" license = "MIT" diff --git a/clusters/src/main.rs b/clusters/src/main.rs index 860bf59..fd46df2 100644 --- a/clusters/src/main.rs +++ b/clusters/src/main.rs @@ -21,7 +21,7 @@ async fn main() -> Result<()> { // create the OpenPortal paddington defaults let defaults = Defaults::parse( - Some("cluster".to_owned()), + Some("clusters".to_owned()), Some( dirs::config_local_dir() .unwrap_or( @@ -29,7 +29,7 @@ async fn main() -> Result<()> { .expect("Could not parse fallback config directory."), ) .join("openportal") - .join("cluster-config.toml"), + .join("clusters-config.toml"), ), Some("ws://localhost:8045".to_owned()), Some("127.0.0.1".to_owned()), diff --git a/docs/README.md b/docs/README.md index ad4ceef..8bac9f1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -98,8 +98,8 @@ The key types of Agent are: adding the colleague to the project may require adding them to a slurm cluster. So the `provider` Agent will send a Job to the `platform` Agent to tell it to add the colleague to the slurm cluster. - The `op-cluster` executable implements the `platform` Agent - for clusters, with source code in the [cluster](../cluster) directory. + The `op-clusters` executable implements the `platform` Agent + for clusters, with source code in the [clusters](../clusters) directory. 4. `instance` - these are agents that represent individual instances of a platform. For example, each indvidual slurm cluster or Jupyter notebook @@ -109,8 +109,8 @@ The key types of Agent are: slurm clusters would pass on the request to add the colleague to the individual `instance` Agent that is responsible for managing the specific slurm cluster to which the colleague is being added. - The `op-slurm` executable implements the `instance` Agent for slurm - clusters, with source code in the [slurm](../slurm) directory. + The `op-cluster` executable implements the `instance` Agent for + clusters, with source code in the [cluster](../cluster) directory. 5. `account` - these are Agents that interface with user account management services, e.g. LDAP, FreeIPA etc. There is one `account` Agent per diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index e3be4b7..2a213d6 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.12" +version = "0.0.14" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index f4f7b3d..7e0447b 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.12" +version = "0.0.14" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index ab555da..69ed404 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.12" +version = "0.0.14" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index 3845562..08386f8 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.12" +version = "0.0.14" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index 7ed45eb..024b101 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.12" +version = "0.0.14" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index ea2ac33..7d10877 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.12" +version = "0.0.14" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/freeipa/README.md b/freeipa/README.md new file mode 100644 index 0000000..7bcf692 --- /dev/null +++ b/freeipa/README.md @@ -0,0 +1,30 @@ +# FreeIPA agent + +This requires extra configuration to set the details used to connect +to the FreeIPA server. + +To test, the demo server provided by FreeIPA is very useful. +This is at `[ipa.demo1.freeipa.org](https://ipa.demo1.freeipa.org/), +and you can use the username `admin` and password `Secret123`. + +First, turn on simple encryption for the FreeIPA password + +```bash +op-freeipa encryption --simple +``` + +You set the server details using + +```bash +op-freeipa extra -k freeipa-server -v https://ipa.demo1.freeipa.org +op-freeipa extra -k freeipa-user -v admin +op-freeipa secret -k freeipa-password -v Secret123 +``` + +You can also add the set of system groups that should always be used +when adding users to FreeIPA via this agent. This should be a +comma-separated list of group names. + +```bash +op-freeipa extra -k system-groups -v group1,group2 +``` diff --git a/helm/cluster/Chart.yaml b/helm/cluster/Chart.yaml new file mode 100644 index 0000000..31b3b13 --- /dev/null +++ b/helm/cluster/Chart.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v2 +name: op-cluster +version: "0.0.0" # Set by release script +appVersion: "0.0.6" # Set by release script +sources: + - https://github.com/isambard-sc/openportal/ diff --git a/helm/cluster/templates/_helpers.yaml b/helm/cluster/templates/_helpers.yaml new file mode 100644 index 0000000..4c76b03 --- /dev/null +++ b/helm/cluster/templates/_helpers.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +{{- define "op-cluster.labels" }} +app.kubernetes.io/name: "op-cluster" +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +{{- end }} diff --git a/helm/cluster/templates/deployment.yaml b/helm/cluster/templates/deployment.yaml new file mode 100644 index 0000000..037e8a3 --- /dev/null +++ b/helm/cluster/templates/deployment.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: op-cluster + labels: + {{- include "op-cluster.labels" . | indent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: op-cluster + template: + metadata: + labels: + {{- include "op-cluster.labels" . | indent 8 }} + spec: + containers: + - name: op-cluster + image: "{{ print .Values.image.registry "/" }}{{ required "image_name must be set" .Values.image.name }}:{{ default .Chart.AppVersion .Values.image.tag }}" + args: ["--config-file=/config/config.toml", "run"] + env: + - name: RUST_LOG + value: {{ .Values.log_level | quote }} + ports: + - containerPort: {{ .Values.port }} + readinessProbe: + httpGet: + path: /health + port: {{ .Values.health_port }} + volumeMounts: + - mountPath: "/config" + name: "config-file-volume" + readOnly: true + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "128Mi" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + capabilities: + drop: + - "ALL" + add: + - "NET_BIND_SERVICE" + volumes: + - name: "config-file-volume" + secret: + secretName: {{ .Values.secret_name | quote }} diff --git a/helm/cluster/templates/service.yaml b/helm/cluster/templates/service.yaml new file mode 100644 index 0000000..4b14b89 --- /dev/null +++ b/helm/cluster/templates/service.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v1 +kind: Service +metadata: + name: op-cluster +spec: + ports: + - protocol: TCP + port: {{ .Values.port }} + name: agent + targetPort: {{ .Values.port }} + - protocol: TCP + port: {{ .Values.health_port }} + name: health + targetPort: {{ .Values.health_port }} + selector: + app.kubernetes.io/name: op-cluster diff --git a/helm/cluster/values.yaml b/helm/cluster/values.yaml new file mode 100644 index 0000000..9a4d2c4 --- /dev/null +++ b/helm/cluster/values.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +config: {} +image: + registry: ghcr.io/isambard-sc + name: op-cluster + tag: # defaults to appVersion if not set +log_level: info +secret_name: cluster-config +port: 80 +health_port: 8080 diff --git a/helm/clusters/Chart.yaml b/helm/clusters/Chart.yaml new file mode 100644 index 0000000..064c3e1 --- /dev/null +++ b/helm/clusters/Chart.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v2 +name: op-clusters +version: "0.0.0" # Set by release script +appVersion: "0.0.6" # Set by release script +sources: + - https://github.com/isambard-sc/openportal/ diff --git a/helm/clusters/templates/_helpers.yaml b/helm/clusters/templates/_helpers.yaml new file mode 100644 index 0000000..cc4297c --- /dev/null +++ b/helm/clusters/templates/_helpers.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +{{- define "op-clusters.labels" }} +app.kubernetes.io/name: "op-clusters" +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +{{- end }} diff --git a/helm/clusters/templates/deployment.yaml b/helm/clusters/templates/deployment.yaml new file mode 100644 index 0000000..32257a4 --- /dev/null +++ b/helm/clusters/templates/deployment.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: op-clusters + labels: + {{- include "op-clusters.labels" . | indent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: op-clusters + template: + metadata: + labels: + {{- include "op-clusters.labels" . | indent 8 }} + spec: + containers: + - name: op-clusters + image: "{{ print .Values.image.registry "/" }}{{ required "image_name must be set" .Values.image.name }}:{{ default .Chart.AppVersion .Values.image.tag }}" + args: ["--config-file=/config/config.toml", "run"] + env: + - name: RUST_LOG + value: {{ .Values.log_level | quote }} + ports: + - containerPort: {{ .Values.port }} + readinessProbe: + httpGet: + path: /health + port: {{ .Values.health_port }} + volumeMounts: + - mountPath: "/config" + name: "config-file-volume" + readOnly: true + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "128Mi" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + capabilities: + drop: + - "ALL" + add: + - "NET_BIND_SERVICE" + volumes: + - name: "config-file-volume" + secret: + secretName: {{ .Values.secret_name | quote }} diff --git a/helm/clusters/templates/service.yaml b/helm/clusters/templates/service.yaml new file mode 100644 index 0000000..c2fb533 --- /dev/null +++ b/helm/clusters/templates/service.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v1 +kind: Service +metadata: + name: op-clusters +spec: + ports: + - protocol: TCP + port: {{ .Values.port }} + name: agent + targetPort: {{ .Values.port }} + - protocol: TCP + port: {{ .Values.health_port }} + name: health + targetPort: {{ .Values.health_port }} + selector: + app.kubernetes.io/name: op-clusters diff --git a/helm/clusters/values.yaml b/helm/clusters/values.yaml new file mode 100644 index 0000000..46eb9fc --- /dev/null +++ b/helm/clusters/values.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +config: {} +image: + registry: ghcr.io/isambard-sc + name: op-clusters + tag: # defaults to appVersion if not set +log_level: info +secret_name: clusters-config +port: 80 +health_port: 8080 diff --git a/helm/filesystem/Chart.yaml b/helm/filesystem/Chart.yaml new file mode 100644 index 0000000..70b9ffb --- /dev/null +++ b/helm/filesystem/Chart.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v2 +name: op-filesystem +version: "0.0.0" # Set by release script +appVersion: "0.0.6" # Set by release script +sources: + - https://github.com/isambard-sc/openportal/ diff --git a/helm/filesystem/templates/_helpers.yaml b/helm/filesystem/templates/_helpers.yaml new file mode 100644 index 0000000..109b128 --- /dev/null +++ b/helm/filesystem/templates/_helpers.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +{{- define "op-filesystem.labels" }} +app.kubernetes.io/name: "op-filesystem" +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +{{- end }} diff --git a/helm/filesystem/templates/deployment.yaml b/helm/filesystem/templates/deployment.yaml new file mode 100644 index 0000000..51c4856 --- /dev/null +++ b/helm/filesystem/templates/deployment.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: op-filesystem + labels: + {{- include "op-filesystem.labels" . | indent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: op-filesystem + template: + metadata: + labels: + {{- include "op-filesystem.labels" . | indent 8 }} + spec: + containers: + - name: op-filesystem + image: "{{ print .Values.image.registry "/" }}{{ required "image_name must be set" .Values.image.name }}:{{ default .Chart.AppVersion .Values.image.tag }}" + args: ["--config-file=/config/config.toml", "run"] + env: + - name: RUST_LOG + value: {{ .Values.log_level | quote }} + ports: + - containerPort: {{ .Values.port }} + readinessProbe: + httpGet: + path: /health + port: {{ .Values.health_port }} + volumeMounts: + - mountPath: "/config" + name: "config-file-volume" + readOnly: true + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "128Mi" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + capabilities: + drop: + - "ALL" + add: + - "NET_BIND_SERVICE" + volumes: + - name: "config-file-volume" + secret: + secretName: {{ .Values.secret_name | quote }} diff --git a/helm/filesystem/templates/service.yaml b/helm/filesystem/templates/service.yaml new file mode 100644 index 0000000..f4d47b2 --- /dev/null +++ b/helm/filesystem/templates/service.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v1 +kind: Service +metadata: + name: op-filesystem +spec: + ports: + - protocol: TCP + port: {{ .Values.port }} + name: agent + targetPort: {{ .Values.port }} + - protocol: TCP + port: {{ .Values.health_port }} + name: health + targetPort: {{ .Values.health_port }} + selector: + app.kubernetes.io/name: op-filesystem diff --git a/helm/filesystem/values.yaml b/helm/filesystem/values.yaml new file mode 100644 index 0000000..5480d0a --- /dev/null +++ b/helm/filesystem/values.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +config: {} +image: + registry: ghcr.io/isambard-sc + name: op-filesystem + tag: # defaults to appVersion if not set +log_level: info +secret_name: filesystem-config +port: 80 +health_port: 8080 diff --git a/helm/freeipa/Chart.yaml b/helm/freeipa/Chart.yaml new file mode 100644 index 0000000..9035989 --- /dev/null +++ b/helm/freeipa/Chart.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v2 +name: op-freeipa +version: "0.0.0" # Set by release script +appVersion: "0.0.6" # Set by release script +sources: + - https://github.com/isambard-sc/openportal/ diff --git a/helm/freeipa/templates/_helpers.yaml b/helm/freeipa/templates/_helpers.yaml new file mode 100644 index 0000000..896553e --- /dev/null +++ b/helm/freeipa/templates/_helpers.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +{{- define "op-freeipa.labels" }} +app.kubernetes.io/name: "op-freeipa" +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +{{- end }} diff --git a/helm/freeipa/templates/deployment.yaml b/helm/freeipa/templates/deployment.yaml new file mode 100644 index 0000000..af2f578 --- /dev/null +++ b/helm/freeipa/templates/deployment.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: op-freeipa + labels: + {{- include "op-freeipa.labels" . | indent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: op-freeipa + template: + metadata: + labels: + {{- include "op-freeipa.labels" . | indent 8 }} + spec: + containers: + - name: op-freeipa + image: "{{ print .Values.image.registry "/" }}{{ required "image_name must be set" .Values.image.name }}:{{ default .Chart.AppVersion .Values.image.tag }}" + args: ["--config-file=/config/config.toml", "run"] + env: + - name: RUST_LOG + value: {{ .Values.log_level | quote }} + ports: + - containerPort: {{ .Values.port }} + readinessProbe: + httpGet: + path: /health + port: {{ .Values.health_port }} + volumeMounts: + - mountPath: "/config" + name: "config-file-volume" + readOnly: true + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "128Mi" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + capabilities: + drop: + - "ALL" + add: + - "NET_BIND_SERVICE" + volumes: + - name: "config-file-volume" + secret: + secretName: {{ .Values.secret_name | quote }} diff --git a/helm/freeipa/templates/service.yaml b/helm/freeipa/templates/service.yaml new file mode 100644 index 0000000..9ac1917 --- /dev/null +++ b/helm/freeipa/templates/service.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v1 +kind: Service +metadata: + name: op-freeipa +spec: + ports: + - protocol: TCP + port: {{ .Values.port }} + name: agent + targetPort: {{ .Values.port }} + - protocol: TCP + port: {{ .Values.health_port }} + name: health + targetPort: {{ .Values.health_port }} + selector: + app.kubernetes.io/name: op-freeipa diff --git a/helm/freeipa/values.yaml b/helm/freeipa/values.yaml new file mode 100644 index 0000000..bb2217d --- /dev/null +++ b/helm/freeipa/values.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +config: {} +image: + registry: ghcr.io/isambard-sc + name: op-freeipa + tag: # defaults to appVersion if not set +log_level: info +secret_name: freeipa-config +port: 80 +health_port: 8080 diff --git a/helm/platform/Chart.yaml b/helm/platform/Chart.yaml new file mode 100644 index 0000000..23d2c4d --- /dev/null +++ b/helm/platform/Chart.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v2 +name: op-platform +version: "0.0.0" # Set by release script +appVersion: "0.0.6" # Set by release script +sources: + - https://github.com/isambard-sc/openportal/ diff --git a/helm/platform/templates/_helpers.yaml b/helm/platform/templates/_helpers.yaml new file mode 100644 index 0000000..123348d --- /dev/null +++ b/helm/platform/templates/_helpers.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +{{- define "op-platform.labels" }} +app.kubernetes.io/name: "op-platform" +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +{{- end }} diff --git a/helm/platform/templates/deployment.yaml b/helm/platform/templates/deployment.yaml new file mode 100644 index 0000000..12e9a7b --- /dev/null +++ b/helm/platform/templates/deployment.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: op-platform + labels: + {{- include "op-platform.labels" . | indent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: op-platform + template: + metadata: + labels: + {{- include "op-platform.labels" . | indent 8 }} + spec: + containers: + - name: op-platform + image: "{{ print .Values.image.registry "/" }}{{ required "image_name must be set" .Values.image.name }}:{{ default .Chart.AppVersion .Values.image.tag }}" + args: ["--config-file=/config/config.toml", "run"] + env: + - name: RUST_LOG + value: {{ .Values.log_level | quote }} + ports: + - containerPort: {{ .Values.port }} + readinessProbe: + httpGet: + path: /health + port: {{ .Values.health_port }} + volumeMounts: + - mountPath: "/config" + name: "config-file-volume" + readOnly: true + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "128Mi" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + capabilities: + drop: + - "ALL" + add: + - "NET_BIND_SERVICE" + volumes: + - name: "config-file-volume" + secret: + secretName: {{ .Values.secret_name | quote }} diff --git a/helm/platform/templates/service.yaml b/helm/platform/templates/service.yaml new file mode 100644 index 0000000..121f6fb --- /dev/null +++ b/helm/platform/templates/service.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v1 +kind: Service +metadata: + name: op-platform +spec: + ports: + - protocol: TCP + port: {{ .Values.port }} + name: agent + targetPort: {{ .Values.port }} + - protocol: TCP + port: {{ .Values.health_port }} + name: health + targetPort: {{ .Values.health_port }} + selector: + app.kubernetes.io/name: op-platform diff --git a/helm/platform/values.yaml b/helm/platform/values.yaml new file mode 100644 index 0000000..6ace4c2 --- /dev/null +++ b/helm/platform/values.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +config: {} +image: + registry: ghcr.io/isambard-sc + name: op-platform + tag: # defaults to appVersion if not set +log_level: info +secret_name: platform-config +port: 80 +health_port: 8080 diff --git a/helm/provider/Chart.yaml b/helm/provider/Chart.yaml new file mode 100644 index 0000000..cfcf420 --- /dev/null +++ b/helm/provider/Chart.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v2 +name: op-provider +version: "0.0.0" # Set by release script +appVersion: "0.0.6" # Set by release script +sources: + - https://github.com/isambard-sc/openportal/ diff --git a/helm/provider/templates/_helpers.yaml b/helm/provider/templates/_helpers.yaml new file mode 100644 index 0000000..28d0564 --- /dev/null +++ b/helm/provider/templates/_helpers.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +{{- define "op-provider.labels" }} +app.kubernetes.io/name: "op-provider" +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +{{- end }} diff --git a/helm/provider/templates/deployment.yaml b/helm/provider/templates/deployment.yaml new file mode 100644 index 0000000..bfd7070 --- /dev/null +++ b/helm/provider/templates/deployment.yaml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: op-provider + labels: + {{- include "op-provider.labels" . | indent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: op-provider + template: + metadata: + labels: + {{- include "op-provider.labels" . | indent 8 }} + spec: + containers: + - name: op-provider + image: "{{ print .Values.image.registry "/" }}{{ required "image_name must be set" .Values.image.name }}:{{ default .Chart.AppVersion .Values.image.tag }}" + args: ["--config-file=/config/config.toml", "run"] + env: + - name: RUST_LOG + value: {{ .Values.log_level | quote }} + ports: + - containerPort: {{ .Values.port }} + readinessProbe: + httpGet: + path: /health + port: {{ .Values.health_port }} + volumeMounts: + - mountPath: "/config" + name: "config-file-volume" + readOnly: true + resources: + requests: + cpu: "100m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "128Mi" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + capabilities: + drop: + - "ALL" + add: + - "NET_BIND_SERVICE" + volumes: + - name: "config-file-volume" + secret: + secretName: {{ .Values.secret_name | quote }} diff --git a/helm/provider/templates/service.yaml b/helm/provider/templates/service.yaml new file mode 100644 index 0000000..c1b5bfe --- /dev/null +++ b/helm/provider/templates/service.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +apiVersion: v1 +kind: Service +metadata: + name: op-provider +spec: + ports: + - protocol: TCP + port: {{ .Values.port }} + name: agent + targetPort: {{ .Values.port }} + - protocol: TCP + port: {{ .Values.health_port }} + name: health + targetPort: {{ .Values.health_port }} + selector: + app.kubernetes.io/name: op-provider diff --git a/helm/provider/values.yaml b/helm/provider/values.yaml new file mode 100644 index 0000000..d4d64d4 --- /dev/null +++ b/helm/provider/values.yaml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: © 2024 Christopher Woods +# SPDX-FileCopyrightText: © 2024 Matt Williams +# SPDX-License-Identifier: MIT +--- +config: {} +image: + registry: ghcr.io/isambard-sc + name: op-provider + tag: # defaults to appVersion if not set +log_level: info +secret_name: provider-config +port: 80 +health_port: 8080 diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index ddc548f..feea675 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.12" +version = "0.0.14" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 6901174..1d74214 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.12" +version = "0.0.14" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index b112698..5a4a25d 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.12" +version = "0.0.14" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index cc41eac..866d2ce 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.12" +version = "0.0.14" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index c19e838..4f6fd17 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.12" +version = "0.0.14" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" From 76565f104c1200a9fd46db0cd4deaaf326ec7a68 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 1 Nov 2024 12:50:58 +0700 Subject: [PATCH 38/51] Fix missing copyright info [ci skip] --- freeipa/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freeipa/README.md b/freeipa/README.md index 7bcf692..b378b49 100644 --- a/freeipa/README.md +++ b/freeipa/README.md @@ -1,3 +1,8 @@ + + # FreeIPA agent This requires extra configuration to set the details used to connect From 841f8392a837c028bcca667efca016b31456b5c7 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 1 Nov 2024 05:54:49 +0000 Subject: [PATCH 39/51] Release 0.0.14 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b295f3..d112a86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.14] - 2024-11-01 ### Added - Changed the names of the cluster instance and platform agents to `cluster` and `clusters`, as they don't need to be named after slurm (and would cause confusion with the slurm agent). @@ -63,6 +65,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.14]: https://github.com/isambard-sc/openportal/releases/tag/0.0.14 [0.0.12]: https://github.com/isambard-sc/openportal/releases/tag/0.0.12 [0.0.11]: https://github.com/isambard-sc/openportal/releases/tag/0.0.11 [0.0.10]: https://github.com/isambard-sc/openportal/releases/tag/0.0.10 From ff9c0ebf17ef84655dfb54fb5213f4870cf47580 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 1 Nov 2024 13:04:47 +0700 Subject: [PATCH 40/51] Fixed missing `op-platform` issue [ci skip] --- .github/workflows/build.yml | 22 --------- .github/workflows/release.yml | 36 +-------------- CHANGELOG.md | 3 ++ Cargo.toml | 4 +- bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- clusters/Cargo.toml | 2 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- helm/platform/Chart.yaml | 10 ----- helm/platform/templates/_helpers.yaml | 10 ----- helm/platform/templates/deployment.yaml | 59 ------------------------- helm/platform/templates/service.yaml | 20 --------- helm/platform/values.yaml | 13 ------ oci/platform/Containerfile | 8 ---- oci/platform/build.sh | 29 ------------ paddington/Cargo.toml | 2 +- portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- 25 files changed, 20 insertions(+), 222 deletions(-) delete mode 100644 helm/platform/Chart.yaml delete mode 100644 helm/platform/templates/_helpers.yaml delete mode 100644 helm/platform/templates/deployment.yaml delete mode 100644 helm/platform/templates/service.yaml delete mode 100644 helm/platform/values.yaml delete mode 100644 oci/platform/Containerfile delete mode 100755 oci/platform/build.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1064cfa..890da66 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,8 +47,6 @@ jobs: run: oci/filesystem/build.sh --release - name: Build op-freeipa OCI image run: oci/freeipa/build.sh --release - - name: Build op-platform OCI image - run: oci/platform/build.sh --release - name: Build op-portal OCI image run: oci/portal/build.sh --release - name: Build op-provider OCI image @@ -80,9 +78,6 @@ jobs: - name: package op-freeipa helm chart run: | helm package helm/freeipa --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" - - name: package op-platform helm chart - run: | - helm package helm/platform --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" - name: package op-portal helm chart run: | helm package helm/portal --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" @@ -134,13 +129,6 @@ jobs: image: op-freeipa tags: ${{ steps.get_version.outputs.version }} registry: ghcr.io/${{ github.repository_owner }} - - name: Publish op-platform OCI image - id: push-platform-to-ghcr - uses: redhat-actions/push-to-registry@v2 - with: - image: op-platform - tags: ${{ steps.get_version.outputs.version }} - registry: ghcr.io/${{ github.repository_owner }} - name: Publish op-portal OCI image id: push-portal-to-ghcr uses: redhat-actions/push-to-registry@v2 @@ -190,13 +178,6 @@ jobs: subject-name: ghcr.io/${{ github.repository_owner }}/op-freeipa subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} push-to-registry: true - - name: Attest op-platform image - uses: actions/attest-build-provenance@v1 - id: attest-platform - with: - subject-name: ghcr.io/${{ github.repository_owner }}/op-platform - subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} - push-to-registry: true - name: Attest op-portal image uses: actions/attest-build-provenance@v1 id: attest-portal @@ -221,8 +202,6 @@ jobs: run: helm push "./op-filesystem-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - name: Push op-freeipa Helm chart run: helm push "./op-freeipa-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - - name: Push op-platform Helm chart - run: helm push "./op-platform-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - name: Push op-portal Helm chart run: helm push "./op-portal-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - name: Push op-provider Helm chart @@ -237,6 +216,5 @@ jobs: target/release/op-clusters target/release/op-filesystem target/release/op-freeipa - target/release/op-platform target/release/op-portal target/release/op-provider diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c190eac..dbb9082 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -286,40 +286,6 @@ jobs: name: sbom-freeipa.spdx.json path: sbom-freeipa.spdx.json - attest-platform: - name: Attest Platform - needs: build-release - runs-on: ubuntu-latest - permissions: - contents: read - attestations: write - id-token: write - steps: - - uses: actions/checkout@v4 - - name: Install toolchain - uses: dtolnay/rust-toolchain@stable - - name: Install cargo-sbom - uses: taiki-e/install-action@v2 - with: - tool: cargo-sbom - - name: Generate SBOM - run: cargo sbom --cargo-package op-platform --output-format=spdx_json_2_3 > sbom-platform.spdx.json - - name: Fetch release artefacts - uses: actions/download-artifact@v4 - with: - pattern: op-platform - merge-multiple: true - - name: Attest SBOM - uses: actions/attest-sbom@v1 - with: - subject-path: op-platform - sbom-path: sbom-platform.spdx.json - - name: Store SBOM - uses: actions/upload-artifact@v4 - with: - name: sbom-platform.spdx.json - path: sbom-platform.spdx.json - attest-portal: name: Attest Portal needs: build-release @@ -391,7 +357,7 @@ jobs: make-release: name: Make release ${{ needs.tag-release.outputs.ref }} needs: [build-release, tag-release, attest-bridge, attest-cluster, - attest-clusters, attest-filesystem, attest-freeipa, attest-platform, + attest-clusters, attest-filesystem, attest-freeipa, attest-portal, attest-provider] runs-on: ubuntu-latest permissions: diff --git a/CHANGELOG.md b/CHANGELOG.md index d112a86..01d097f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Fixed +- Fixed issues with the helm charts and OCI images (removed `op-platform` as it + doesn't exist!) ## [0.0.14] - 2024-11-01 ### Added diff --git a/Cargo.toml b/Cargo.toml index 17fe23d..268e9ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,9 +4,9 @@ [workspace] members = [ - "bridge", "cluster", + "bridge", "cluster", "clusters", "filesystem", "freeipa", "paddington", "portal", - "provider", "python", "cluster", "clusters", "templemeads", + "provider", "python", "templemeads", "docs/echo", "docs/job", "docs/cmdline/portal", "docs/cmdline/cluster" ] diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index 81332de..d07496f 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.14" +version = "0.0.15" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index b18d693..923e5b5 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.14" +version = "0.0.15" description = "An example of an OpenPortal cluster instance agent" edition = "2021" license = "MIT" diff --git a/clusters/Cargo.toml b/clusters/Cargo.toml index ea5822c..ba1e8e0 100644 --- a/clusters/Cargo.toml +++ b/clusters/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-clusters" -version = "0.0.14" +version = "0.0.15" description = "An example of an OpenPortal cluster platform agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index 2a213d6..dc6fbb9 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.14" +version = "0.0.15" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index 7e0447b..07de73c 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.14" +version = "0.0.15" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index 69ed404..f696034 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.14" +version = "0.0.15" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index 08386f8..1467d73 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.14" +version = "0.0.15" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index 024b101..4825eca 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.14" +version = "0.0.15" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index 7d10877..97a915f 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.14" +version = "0.0.15" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/helm/platform/Chart.yaml b/helm/platform/Chart.yaml deleted file mode 100644 index 23d2c4d..0000000 --- a/helm/platform/Chart.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT ---- -apiVersion: v2 -name: op-platform -version: "0.0.0" # Set by release script -appVersion: "0.0.6" # Set by release script -sources: - - https://github.com/isambard-sc/openportal/ diff --git a/helm/platform/templates/_helpers.yaml b/helm/platform/templates/_helpers.yaml deleted file mode 100644 index 123348d..0000000 --- a/helm/platform/templates/_helpers.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT -{{- define "op-platform.labels" }} -app.kubernetes.io/name: "op-platform" -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -app.kubernetes.io/managed-by: {{ .Release.Service | quote }} -app.kubernetes.io/instance: {{ .Release.Name | quote }} -helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" -{{- end }} diff --git a/helm/platform/templates/deployment.yaml b/helm/platform/templates/deployment.yaml deleted file mode 100644 index 12e9a7b..0000000 --- a/helm/platform/templates/deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: op-platform - labels: - {{- include "op-platform.labels" . | indent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: op-platform - template: - metadata: - labels: - {{- include "op-platform.labels" . | indent 8 }} - spec: - containers: - - name: op-platform - image: "{{ print .Values.image.registry "/" }}{{ required "image_name must be set" .Values.image.name }}:{{ default .Chart.AppVersion .Values.image.tag }}" - args: ["--config-file=/config/config.toml", "run"] - env: - - name: RUST_LOG - value: {{ .Values.log_level | quote }} - ports: - - containerPort: {{ .Values.port }} - readinessProbe: - httpGet: - path: /health - port: {{ .Values.health_port }} - volumeMounts: - - mountPath: "/config" - name: "config-file-volume" - readOnly: true - resources: - requests: - cpu: "100m" - memory: "64Mi" - limits: - cpu: "500m" - memory: "128Mi" - securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - capabilities: - drop: - - "ALL" - add: - - "NET_BIND_SERVICE" - volumes: - - name: "config-file-volume" - secret: - secretName: {{ .Values.secret_name | quote }} diff --git a/helm/platform/templates/service.yaml b/helm/platform/templates/service.yaml deleted file mode 100644 index 121f6fb..0000000 --- a/helm/platform/templates/service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT ---- -apiVersion: v1 -kind: Service -metadata: - name: op-platform -spec: - ports: - - protocol: TCP - port: {{ .Values.port }} - name: agent - targetPort: {{ .Values.port }} - - protocol: TCP - port: {{ .Values.health_port }} - name: health - targetPort: {{ .Values.health_port }} - selector: - app.kubernetes.io/name: op-platform diff --git a/helm/platform/values.yaml b/helm/platform/values.yaml deleted file mode 100644 index 6ace4c2..0000000 --- a/helm/platform/values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT ---- -config: {} -image: - registry: ghcr.io/isambard-sc - name: op-platform - tag: # defaults to appVersion if not set -log_level: info -secret_name: platform-config -port: 80 -health_port: 8080 diff --git a/oci/platform/Containerfile b/oci/platform/Containerfile deleted file mode 100644 index 556b12d..0000000 --- a/oci/platform/Containerfile +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT - -FROM gcr.io/distroless/static-debian12 -COPY op-platform / -USER 65534:65534 -ENTRYPOINT ["/op-platform"] diff --git a/oci/platform/build.sh b/oci/platform/build.sh deleted file mode 100755 index 79123a0..0000000 --- a/oci/platform/build.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: © 2024 Christopher Woods -# SPDX-FileCopyrightText: © 2024 Matt Williams -# SPDX-License-Identifier: MIT -set -euo pipefail - -# Build the project and create an OCI image containing it. - -function artifact_path { - echo "${1}" | jq --raw-output 'select(.reason == "compiler-artifact") | select(.target.name == "'"${2}"'") | .executable' -} - -out=$(cargo build --package op-platform --target=x86_64-unknown-linux-musl --message-format=json ${@-}) -cp "$(artifact_path "${out}" "op-platform")" oci/platform - -cd oci/platform - -version=$(./op-platform --version | tail -n1 | cut -d' ' -f 2) -image_id=$( - podman build . --tag=op-platform:latest --tag=op-platform:"${version}" \ - --annotation="org.opencontainers.image.source=https://github.com/isambard-sc/openportal" \ - --annotation="org.opencontainers.image.description=OpenPortal" \ - --annotation="org.opencontainers.image.licenses=MIT" \ - | tee /dev/fd/2 \ - | tail -n1 -) -rm op-platform -echo "Built op-platform image:" 1>&2 -echo "${image_id}" diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index feea675..47eb3bb 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.14" +version = "0.0.15" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 1d74214..8aa6137 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.14" +version = "0.0.15" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index 5a4a25d..dc8088c 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.14" +version = "0.0.15" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index 866d2ce..4e55da0 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.14" +version = "0.0.15" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index 4f6fd17..de28135 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.14" +version = "0.0.15" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" From f55d2a3dbf3cfd537841b57a43d2bd37969ee2d0 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 1 Nov 2024 06:09:03 +0000 Subject: [PATCH 41/51] Release 0.0.15 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01d097f..f1d097b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.15] - 2024-11-01 ### Fixed - Fixed issues with the helm charts and OCI images (removed `op-platform` as it doesn't exist!) @@ -68,6 +70,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.15]: https://github.com/isambard-sc/openportal/releases/tag/0.0.15 [0.0.14]: https://github.com/isambard-sc/openportal/releases/tag/0.0.14 [0.0.12]: https://github.com/isambard-sc/openportal/releases/tag/0.0.12 [0.0.11]: https://github.com/isambard-sc/openportal/releases/tag/0.0.11 From 68ab53a5fa676427b6fd667a07b4c47388dc7a1b Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 1 Nov 2024 13:19:56 +0700 Subject: [PATCH 42/51] Fixed attestation issues [ci skip] --- .github/workflows/build.yml | 12 ++++++------ CHANGELOG.md | 2 ++ bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- clusters/Cargo.toml | 2 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- paddington/Cargo.toml | 2 +- portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- 16 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 890da66..a16c17d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -155,42 +155,42 @@ jobs: id: attest-cluster with: subject-name: ghcr.io/${{ github.repository_owner }}/op-cluster - subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + subject-digest: ${{ steps.push-cluster-to-ghcr.outputs.digest }} push-to-registry: true - name: Attest op-clusters image uses: actions/attest-build-provenance@v1 id: attest-clusters with: subject-name: ghcr.io/${{ github.repository_owner }}/op-clusters - subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + subject-digest: ${{ steps.push-clusters-to-ghcr.outputs.digest }} push-to-registry: true - name: Attest op-filesystem image uses: actions/attest-build-provenance@v1 id: attest-filesystem with: subject-name: ghcr.io/${{ github.repository_owner }}/op-filesystem - subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + subject-digest: ${{ steps.push-filesystem-to-ghcr.outputs.digest }} push-to-registry: true - name: Attest op-freeipa image uses: actions/attest-build-provenance@v1 id: attest-freeipa with: subject-name: ghcr.io/${{ github.repository_owner }}/op-freeipa - subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + subject-digest: ${{ steps.push-freeipa-to-ghcr.outputs.digest }} push-to-registry: true - name: Attest op-portal image uses: actions/attest-build-provenance@v1 id: attest-portal with: subject-name: ghcr.io/${{ github.repository_owner }}/op-portal - subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + subject-digest: ${{ steps.push-portal-to-ghcr.outputs.digest }} push-to-registry: true - name: Attest op-provider image uses: actions/attest-build-provenance@v1 id: attest-provider with: subject-name: ghcr.io/${{ github.repository_owner }}/op-provider - subject-digest: ${{ steps.push-bridge-to-ghcr.outputs.digest }} + subject-digest: ${{ steps.push-provider-to-ghcr.outputs.digest }} push-to-registry: true - name: Push op-bridge Helm chart run: helm push "./op-bridge-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" diff --git a/CHANGELOG.md b/CHANGELOG.md index f1d097b..6a8e4b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Fixed +- Fixed issue with attestation of OCI images ## [0.0.15] - 2024-11-01 ### Fixed diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index d07496f..29772b0 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.15" +version = "0.0.16" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index 923e5b5..9d5ab48 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.15" +version = "0.0.16" description = "An example of an OpenPortal cluster instance agent" edition = "2021" license = "MIT" diff --git a/clusters/Cargo.toml b/clusters/Cargo.toml index ba1e8e0..182b614 100644 --- a/clusters/Cargo.toml +++ b/clusters/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-clusters" -version = "0.0.15" +version = "0.0.16" description = "An example of an OpenPortal cluster platform agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index dc6fbb9..48cc187 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.15" +version = "0.0.16" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index 07de73c..110304e 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.15" +version = "0.0.16" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index f696034..361d7e1 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.15" +version = "0.0.16" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index 1467d73..b7d2c07 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.15" +version = "0.0.16" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index 4825eca..d7b974a 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.15" +version = "0.0.16" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index 97a915f..e5b21e3 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.15" +version = "0.0.16" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index 47eb3bb..ea76741 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.15" +version = "0.0.16" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 8aa6137..67c6178 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.15" +version = "0.0.16" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index dc8088c..8d5b1ae 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.15" +version = "0.0.16" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index 4e55da0..36c23b2 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.15" +version = "0.0.16" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index de28135..2545b87 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.15" +version = "0.0.16" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" From 7444a4a6cd5847a1e303c0ee79193c9b3cbabb10 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 1 Nov 2024 06:23:41 +0000 Subject: [PATCH 43/51] Release 0.0.16 --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a8e4b0..0804892 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased -### Fixed + +## [0.0.16] - 2024-11-01 +### Fixed - Fixed issue with attestation of OCI images ## [0.0.15] - 2024-11-01 @@ -72,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.16]: https://github.com/isambard-sc/openportal/releases/tag/0.0.16 [0.0.15]: https://github.com/isambard-sc/openportal/releases/tag/0.0.15 [0.0.14]: https://github.com/isambard-sc/openportal/releases/tag/0.0.14 [0.0.12]: https://github.com/isambard-sc/openportal/releases/tag/0.0.12 From fcb0eb4296986d1ae3340cf0577a0a494a3b286d Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 1 Nov 2024 14:02:15 +0700 Subject: [PATCH 44/51] Each agent is separately released [ci skip] --- .github/workflows/build.yml | 41 +++++++++++++++++++++++++++++++-- CHANGELOG.md | 3 +++ bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- clusters/Cargo.toml | 2 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- paddington/Cargo.toml | 2 +- portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- 16 files changed, 56 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a16c17d..738fe65 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,6 +37,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Build run: cargo build --release + - name: Build op-bridge OCI image run: oci/bridge/build.sh --release - name: Build op-cluster OCI image @@ -51,6 +52,7 @@ jobs: run: oci/portal/build.sh --release - name: Build op-provider OCI image run: oci/provider/build.sh --release + - name: Get version id: get_version run: | @@ -63,6 +65,7 @@ jobs: else echo version="${{ steps.get_version.outputs.version }}.${{ github.ref_name }}" >> "${GITHUB_OUTPUT}" fi + - name: package op-bridge helm chart run: | helm package helm/bridge --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" @@ -84,6 +87,7 @@ jobs: - name: package op-provider helm chart run: | helm package helm/provider --version "${{ steps.get_helm_version.outputs.version }}" --app-version "${{ steps.get_version.outputs.version }}" + - name: Log in to GHCR uses: redhat-actions/podman-login@v1 with: @@ -94,6 +98,7 @@ jobs: run: echo $GITHUB_TOKEN | helm registry login "ghcr.io/${{ github.repository_owner }}" --username "${{ github.actor }}" --password-stdin env: GITHUB_TOKEN: "${{ github.token }}" + - name: Publish op-bridge OCI image id: push-bridge-to-ghcr uses: redhat-actions/push-to-registry@v2 @@ -143,6 +148,7 @@ jobs: image: op-provider tags: ${{ steps.get_version.outputs.version }} registry: ghcr.io/${{ github.repository_owner }} + - name: Attest op-bridge image uses: actions/attest-build-provenance@v1 id: attest-bridge @@ -206,15 +212,46 @@ jobs: run: helm push "./op-portal-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - name: Push op-provider Helm chart run: helm push "./op-provider-${{ steps.get_helm_version.outputs.version }}.tgz" "oci://ghcr.io/${{ github.repository_owner }}/charts" - - name: Store agent artefacts + + - name: Store bridge artefact uses: actions/upload-artifact@v4 with: - name: openportal-agents + name: op-bridge path: | target/release/op-bridge + - name: Store cluster artefact + uses: actions/upload-artifact@v4 + with: + name: op-cluster + path: | target/release/op-cluster + - name: Store clusters artefact + uses: actions/upload-artifact@v4 + with: + name: op-clusters + path: | target/release/op-clusters + - name: Store filesystem artefact + uses: actions/upload-artifact@v4 + with: + name: op-filesystem + path: | target/release/op-filesystem + - name: Store freeipa artefact + uses: actions/upload-artifact@v4 + with: + name: op-freeipa + path: | target/release/op-freeipa + - name: Store portal artefact + uses: actions/upload-artifact@v4 + with: + name: op-portal + path: | target/release/op-portal + - name: Store provider artefact + uses: actions/upload-artifact@v4 + with: + name: op-provider + path: | target/release/op-provider diff --git a/CHANGELOG.md b/CHANGELOG.md index 0804892..e43f83c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Fixed +- Fixed issues with attestations that depended on releases. Need to release + each agent separately, which this release now does. ## [0.0.16] - 2024-11-01 ### Fixed diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index 29772b0..94e0c63 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.16" +version = "0.0.17" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index 9d5ab48..5d60bae 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.16" +version = "0.0.17" description = "An example of an OpenPortal cluster instance agent" edition = "2021" license = "MIT" diff --git a/clusters/Cargo.toml b/clusters/Cargo.toml index 182b614..27b8226 100644 --- a/clusters/Cargo.toml +++ b/clusters/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-clusters" -version = "0.0.16" +version = "0.0.17" description = "An example of an OpenPortal cluster platform agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index 48cc187..84773ce 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.16" +version = "0.0.17" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index 110304e..5f4c9e1 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.16" +version = "0.0.17" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index 361d7e1..e6798b2 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.16" +version = "0.0.17" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index b7d2c07..19ff199 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.16" +version = "0.0.17" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index d7b974a..e91e68c 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.16" +version = "0.0.17" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index e5b21e3..6267a15 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.16" +version = "0.0.17" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index ea76741..46ebb2f 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.16" +version = "0.0.17" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 67c6178..63a1b20 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.16" +version = "0.0.17" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index 8d5b1ae..37ca441 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.16" +version = "0.0.17" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index 36c23b2..32f5623 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.16" +version = "0.0.17" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index 2545b87..0f65b40 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.16" +version = "0.0.17" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" From c845ce22879c808eb4fa3a3e5364430de41dee3a Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 1 Nov 2024 07:06:09 +0000 Subject: [PATCH 45/51] Release 0.0.17 --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e43f83c..b2ab1f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased -### Fixed + +## [0.0.17] - 2024-11-01 +### Fixed - Fixed issues with attestations that depended on releases. Need to release each agent separately, which this release now does. @@ -77,6 +79,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.17]: https://github.com/isambard-sc/openportal/releases/tag/0.0.17 [0.0.16]: https://github.com/isambard-sc/openportal/releases/tag/0.0.16 [0.0.15]: https://github.com/isambard-sc/openportal/releases/tag/0.0.15 [0.0.14]: https://github.com/isambard-sc/openportal/releases/tag/0.0.14 From f556416cf4ce3a38adda74c172325a19b68107f1 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Fri, 1 Nov 2024 14:31:38 +0700 Subject: [PATCH 46/51] Working on getting pypi upload working... [ci skip] --- .github/workflows/python.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d90a1c2..3a5651a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -87,7 +87,6 @@ jobs: with: subject-path: 'wheels-*/*' - name: Publish to PyPI - if: "startsWith(github.ref, 'refs/tags/')" uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} From 7c667ee38fd5b716fee005c904dd2e74e587bfea Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Tue, 5 Nov 2024 14:55:27 +0700 Subject: [PATCH 47/51] Specified default TLS provider [ci skip] --- CHANGELOG.md | 3 +++ bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- clusters/Cargo.toml | 2 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- paddington/Cargo.toml | 3 ++- paddington/src/eventloop.rs | 10 ++++++++++ portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- 16 files changed, 28 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2ab1f6..c893cad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Fixed +- Specified default TLS provider so that containerised services can run without + panicing. ## [0.0.17] - 2024-11-01 ### Fixed diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index 94e0c63..66bb59e 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.17" +version = "0.0.18" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index 5d60bae..78a3151 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.17" +version = "0.0.18" description = "An example of an OpenPortal cluster instance agent" edition = "2021" license = "MIT" diff --git a/clusters/Cargo.toml b/clusters/Cargo.toml index 27b8226..d5c85eb 100644 --- a/clusters/Cargo.toml +++ b/clusters/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-clusters" -version = "0.0.17" +version = "0.0.18" description = "An example of an OpenPortal cluster platform agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index 84773ce..6712cdb 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.17" +version = "0.0.18" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index 5f4c9e1..e934f20 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.17" +version = "0.0.18" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index e6798b2..30e1d78 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.17" +version = "0.0.18" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index 19ff199..ef9db8f 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.17" +version = "0.0.18" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index e91e68c..68cfc8f 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.17" +version = "0.0.18" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index 6267a15..14b188d 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.17" +version = "0.0.18" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index 46ebb2f..b2412b6 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.17" +version = "0.0.18" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" @@ -26,6 +26,7 @@ hex = {version="0.4.3", features = ["serde"]} iptools = "0.2.5" once_cell = "1.19.0" orion = "0.17.6" +rustls = { version = "0.23.16", features = ["ring"] } secrecy = { version = "0.8.0", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.120" diff --git a/paddington/src/eventloop.rs b/paddington/src/eventloop.rs index bf24b2f..e34b792 100644 --- a/paddington/src/eventloop.rs +++ b/paddington/src/eventloop.rs @@ -8,6 +8,16 @@ use crate::error::Error; use crate::{client, server}; pub async fn run(config: ServiceConfig) -> Result<(), Error> { + match rustls::crypto::ring::default_provider().install_default() { + Ok(_) => {} + Err(e) => { + tracing::error!("Could not install default ring provider: {:?}", e); + return Err(Error::NotExists( + "Could not install default ring provider".to_owned(), + )); + } + } + let mut server_handles = vec![]; let mut client_handles = vec![]; diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 63a1b20..b558e6a 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.17" +version = "0.0.18" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index 37ca441..d9f2f16 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.17" +version = "0.0.18" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index 32f5623..37976b0 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.17" +version = "0.0.18" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index 0f65b40..ab74b32 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.17" +version = "0.0.18" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" From 1334be33f73f075255e82907a1e6342a51789494 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 5 Nov 2024 08:02:02 +0000 Subject: [PATCH 48/51] Release 0.0.18 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c893cad..948493f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.18] - 2024-11-05 ### Fixed - Specified default TLS provider so that containerised services can run without panicing. @@ -82,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.18]: https://github.com/isambard-sc/openportal/releases/tag/0.0.18 [0.0.17]: https://github.com/isambard-sc/openportal/releases/tag/0.0.17 [0.0.16]: https://github.com/isambard-sc/openportal/releases/tag/0.0.16 [0.0.15]: https://github.com/isambard-sc/openportal/releases/tag/0.0.15 From 97b53037d2fa292d308685e5e64f491e82d2aba9 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Thu, 7 Nov 2024 12:28:02 +0700 Subject: [PATCH 49/51] Making freeipa more robust to groups being deleted behind our back Code also now properly uses Err to signify error, not Ok(None) (what was I thinking?) [ci skip] --- freeipa/src/freeipa.rs | 54 ++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/freeipa/src/freeipa.rs b/freeipa/src/freeipa.rs index 2a4fc4e..e5b85e3 100644 --- a/freeipa/src/freeipa.rs +++ b/freeipa/src/freeipa.rs @@ -753,7 +753,7 @@ fn get_primary_group(user: &UserIdentifier) -> String { /// or removes them as necessary. Groups will match the project group, /// the system groups, and the openportal group. /// -async fn sync_groups(user: &IPAUser) -> Result, Error> { +async fn sync_groups(user: &IPAUser) -> Result { // the user probably doesn't exist, so add them, making sure they // are in the correct groups let mut groups = cache::get_system_groups().await?; @@ -833,12 +833,21 @@ async fn sync_groups(user: &IPAUser) -> Result, Error> { match call_post::("group_add_member", None, Some(kwargs)).await { Ok(_) => tracing::info!("Successfully added user {} to group {}", userid, group_cn), Err(e) => { + // this should not happen - it indicates that the group has disappeared + // since we last updated. Our cache is now likely out of date. tracing::error!( "Could not add user {} to group {}. Error: {}", userid, group_cn, e ); + tracing::info!("Clearing the cache as FreeIPA has changed behind our back."); + cache::clear().await?; + // Return None so that the caller handles this failure case + return Err(Error::InvalidState(format!( + "Could not add user {} to group {}. Error: {}. Likely freeipa was changed behind our back!", + userid, group_cn, e + ))); } } } @@ -846,7 +855,7 @@ async fn sync_groups(user: &IPAUser) -> Result, Error> { // finally - re-fetch the user from FreeIPA to make sure that we have // the correct information match force_get_user(user.identifier()).await? { - Some(user) => Ok(Some(user)), + Some(user) => Ok(user), None => { tracing::warn!( "Failed to sync groups for user {} as this user no longer exists in FreeIPA.", @@ -854,7 +863,11 @@ async fn sync_groups(user: &IPAUser) -> Result, Error> { ); tracing::info!("Clearing the cache as FreeIPA has changed behind our back."); cache::clear().await?; - Ok(None) + // Return None so that the caller handles this failure case + Err(Error::InvalidState(format!( + "Failed to sync groups for user {} as this user no longer exists in FreeIPA. Likely freeipa was changed behind our back!", + user.identifier() + ))) } } } @@ -863,9 +876,22 @@ pub async fn add_user(user: &UserIdentifier) -> Result { // return the user if they already exist if let Some(user) = get_user(user).await? { // make sure that the groups are correct - if let Some(user) = sync_groups(&user).await? { - tracing::info!("Added user [cached] {}", user); - return Ok(user); + match sync_groups(&user).await { + Ok(user) => { + tracing::info!("Added user [cached] {}", user); + return Ok(user); + } + Err(e) => { + tracing::warn!( + "Failed to sync groups for user {} after adding. Error: {}", + user.identifier(), + e + ); + tracing::info!( + "Will try to add user {} again, as the groups are not correct.", + user.identifier() + ); + } } // we get here if the user has been removed from FreeIPA behind @@ -956,19 +982,21 @@ pub async fn add_user(user: &UserIdentifier) -> Result { // now synchronise the groups - this won't do anything if another // thread has already beaten us to creating the user - match sync_groups(&user).await? { - Some(user) => { + match sync_groups(&user).await { + Ok(user) => { tracing::info!("Added user: {}", user); Ok(user) } - None => { + Err(e) => { tracing::warn!( - "Failed to add user {} - they have been removed from FreeIPA?", - user.identifier() + "Failed to add user {} - they have been removed from FreeIPA? {}", + user.identifier(), + e ); Err(Error::Call(format!( - "Failed to add user {} - they have been removed from FreeIPA?", - user.identifier() + "Failed to add user {} - they have been removed from FreeIPA? {}", + user.identifier(), + e ))) } } From b799206cec2cfdad12ebc3b086922464987a7644 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Thu, 7 Nov 2024 12:29:39 +0700 Subject: [PATCH 50/51] Updated changelog and version [ci skip] --- CHANGELOG.md | 3 +++ bridge/Cargo.toml | 2 +- cluster/Cargo.toml | 2 +- clusters/Cargo.toml | 2 +- docs/cmdline/cluster/Cargo.toml | 2 +- docs/cmdline/portal/Cargo.toml | 2 +- docs/echo/Cargo.toml | 2 +- docs/job/Cargo.toml | 2 +- filesystem/Cargo.toml | 2 +- freeipa/Cargo.toml | 2 +- paddington/Cargo.toml | 2 +- portal/Cargo.toml | 2 +- provider/Cargo.toml | 2 +- python/Cargo.toml | 2 +- templemeads/Cargo.toml | 2 +- 15 files changed, 17 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 948493f..831c528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased +### Fixed +- Made the code more robust to freeipa being cleared / having groups removed + behind our back. Also better way to handle errors. ## [0.0.18] - 2024-11-05 ### Fixed diff --git a/bridge/Cargo.toml b/bridge/Cargo.toml index 66bb59e..c7b81ad 100644 --- a/bridge/Cargo.toml +++ b/bridge/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-bridge" -version = "0.0.18" +version = "0.0.19" description = "An example of an OpenPortal user portal to OpenPortal bridge" edition = "2021" license = "MIT" diff --git a/cluster/Cargo.toml b/cluster/Cargo.toml index 78a3151..1c13992 100644 --- a/cluster/Cargo.toml +++ b/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-cluster" -version = "0.0.18" +version = "0.0.19" description = "An example of an OpenPortal cluster instance agent" edition = "2021" license = "MIT" diff --git a/clusters/Cargo.toml b/clusters/Cargo.toml index d5c85eb..8848bad 100644 --- a/clusters/Cargo.toml +++ b/clusters/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-clusters" -version = "0.0.18" +version = "0.0.19" description = "An example of an OpenPortal cluster platform agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/cluster/Cargo.toml b/docs/cmdline/cluster/Cargo.toml index 6712cdb..55eb2e7 100644 --- a/docs/cmdline/cluster/Cargo.toml +++ b/docs/cmdline/cluster/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-cluster" -version = "0.0.18" +version = "0.0.19" description = "templemeads command line example - cluster agent" edition = "2021" license = "MIT" diff --git a/docs/cmdline/portal/Cargo.toml b/docs/cmdline/portal/Cargo.toml index e934f20..792eee4 100644 --- a/docs/cmdline/portal/Cargo.toml +++ b/docs/cmdline/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-portal" -version = "0.0.18" +version = "0.0.19" description = "templemeads command line example - portal agent" edition = "2021" license = "MIT" diff --git a/docs/echo/Cargo.toml b/docs/echo/Cargo.toml index 30e1d78..ab9542a 100644 --- a/docs/echo/Cargo.toml +++ b/docs/echo/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-echo" -version = "0.0.18" +version = "0.0.19" description = "Paddington Echo Service example" edition = "2021" license = "MIT" diff --git a/docs/job/Cargo.toml b/docs/job/Cargo.toml index ef9db8f..9700224 100644 --- a/docs/job/Cargo.toml +++ b/docs/job/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "example-job" -version = "0.0.18" +version = "0.0.19" description = "templemeads job example" edition = "2021" license = "MIT" diff --git a/filesystem/Cargo.toml b/filesystem/Cargo.toml index 68cfc8f..17359ae 100644 --- a/filesystem/Cargo.toml +++ b/filesystem/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-filesystem" -version = "0.0.18" +version = "0.0.19" description = "Agent that interfaces OpenPortal with a filesystem" edition = "2021" license = "MIT" diff --git a/freeipa/Cargo.toml b/freeipa/Cargo.toml index 14b188d..6dd9f3b 100644 --- a/freeipa/Cargo.toml +++ b/freeipa/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-freeipa" -version = "0.0.18" +version = "0.0.19" description = "Agent that interfaces OpenPortal with FreeIPA" edition = "2021" license = "MIT" diff --git a/paddington/Cargo.toml b/paddington/Cargo.toml index b2412b6..a07506c 100644 --- a/paddington/Cargo.toml +++ b/paddington/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "paddington" -version = "0.0.18" +version = "0.0.19" description = "A library for implementing the OpenPortal communication protocol" edition = "2021" license = "MIT" diff --git a/portal/Cargo.toml b/portal/Cargo.toml index b558e6a..78f5be7 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-portal" -version = "0.0.18" +version = "0.0.19" description = "An example of an OpenPortal portal interface service" edition = "2021" license = "MIT" diff --git a/provider/Cargo.toml b/provider/Cargo.toml index d9f2f16..3627977 100644 --- a/provider/Cargo.toml +++ b/provider/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "op-provider" -version = "0.0.18" +version = "0.0.19" description = "An example of an OpenPortal provider interface service" edition = "2021" license = "MIT" diff --git a/python/Cargo.toml b/python/Cargo.toml index 37976b0..e975d00 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openportal" -version = "0.0.18" +version = "0.0.19" description = "Python wrappers for OpenPortal" edition = "2021" license = "MIT" diff --git a/templemeads/Cargo.toml b/templemeads/Cargo.toml index ab74b32..f09a63d 100644 --- a/templemeads/Cargo.toml +++ b/templemeads/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "templemeads" -version = "0.0.18" +version = "0.0.19" description = "A library for interfacing OpenPortal with specific portals" edition = "2021" license = "MIT" From 2301473bb900eb8f54bcc2396a555093332fa40a Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 7 Nov 2024 05:35:33 +0000 Subject: [PATCH 51/51] Release 0.0.19 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 831c528..2292087 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Unreleased + +## [0.0.19] - 2024-11-07 ### Fixed - Made the code more robust to freeipa being cleared / having groups removed behind our back. Also better way to handle errors. @@ -87,6 +89,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial release This is an initial alpha release of the OpenPortal project. It is not yet feature complete and is not recommended for production use. +[0.0.19]: https://github.com/isambard-sc/openportal/releases/tag/0.0.19 [0.0.18]: https://github.com/isambard-sc/openportal/releases/tag/0.0.18 [0.0.17]: https://github.com/isambard-sc/openportal/releases/tag/0.0.17 [0.0.16]: https://github.com/isambard-sc/openportal/releases/tag/0.0.16