Update CI files

[noissue]

.ci/scripts/changelog.py

@@ -25,7 +25,9 @@ def get_changelog_releases(changelog):
     Get all versions in changelog.
 
     """
-    versions = re.findall(r"([0-9]+)\.([0-9]+)\.([0-9]+) \(", changelog)
+    versions = re.findall(
+        r"([0-9]+)\.([0-9]+)\.([0-9][0-9ab]*) \([0-9]{4}-[0-9]{2}-[0-9]{2}\)", changelog
+    )
     return {".".join(v) for v in versions}
 
 

.ci/scripts/check_release.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_rpm' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import argparse
+import re
+import os
+import yaml
+from tempfile import TemporaryDirectory
+from packaging.version import Version
+from git import Repo
+
+UPSTREAM_REMOTE = "https://github.com/pulp/pulp_rpm.git"
+DEFAULT_BRANCH = "main"
+RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$"
+Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"]
+Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"]
+
+
+def main():
+    """Check which branches need a release."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--branches",
+        default="supported",
+        help="A comma separated list of branches to check for releases. Can also use keyword: "
+        "'supported'. Defaults to 'supported', see `ci_update_branches` in "
+        "`plugin_template.yml`.",
+    )
+    opts = parser.parse_args()
+
+    with TemporaryDirectory() as d:
+        # Clone from upstream to ensure we have updated branches & main
+        repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none")
+        heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")]
+        available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
+        available_branches.sort(key=lambda ver: Version(ver))
+        available_branches.append(DEFAULT_BRANCH)
+
+        branches = opts.branches
+        if branches == "supported":
+            with open(f"{d}/template_config.yml", mode="r") as f:
+                tc = yaml.safe_load(f)
+                branches = tc["ci_update_branches"]
+            branches.append(DEFAULT_BRANCH)
+        else:
+            branches = branches.split(",")
+
+        if diff := set(branches) - set(available_branches):
+            print(f"Supplied branches contains non-existent branches! {diff}")
+            exit(1)
+
+        print(f"Checking for releases on branches: {branches}")
+
+        releases = []
+        for branch in branches:
+            if branch != DEFAULT_BRANCH:
+                # Check if a Z release is needed
+                changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
+                z_release = False
+                for change in changes.split("\n"):
+                    # Check each changelog file to make sure everything checks out
+                    _, ext = os.path.splitext(change)
+                    if ext in Y_CHANGELOG_EXTS:
+                        print(
+                            f"Warning: A non-backported changelog ({change}) is present in the "
+                            f"{branch} release branch!"
+                        )
+                    elif ext in Z_CHANGELOG_EXTS:
+                        z_release = True
+                if z_release:
+                    # Blobless clone does not have file contents for Z branches,
+                    # check commit message for last Z bump
+                    git_branch = f"origin/{branch}"
+                    next_version = repo.git.log(
+                        "--oneline", "--grep=Bump to", "-n 1", git_branch, "--", ".bumpversion.cfg"
+                    ).split("to")[-1]
+                    next_version = Version(next_version)
+                    print(
+                        f"A Z-release is needed for {branch}, "
+                        f"New Version: {next_version.base_version}"
+                    )
+                    releases.append(next_version)
+            else:
+                # Check if a Y release is needed
+                changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/")
+                for change in changes.split("\n"):
+                    _, ext = os.path.splitext(change)
+                    if ext in Y_CHANGELOG_EXTS:
+                        # We don't put Y release bumps in the commit message, check file instead
+                        # The 'current_version' is always the next version to release
+                        next_version = repo.git.grep(
+                            "current_version", DEFAULT_BRANCH, "--", ".bumpversion.cfg"
+                        ).split("=")[-1]
+                        next_version = Version(next_version)
+                        print(
+                            f"A new Y-release is needed! New Version: {next_version.base_version}"
+                        )
+                        releases.append(next_version)
+                        break
+
+        if len(releases) == 0:
+            print("No new releases to perform.")
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/check_requirements.py

@@ -42,12 +42,13 @@
                     else:
                         errors.append(f"{filename}:{nr}: Unreadable requirement {line}")
                 else:
-                    if not line.startswith("opentelemetry"):
-                        if check_prereleases and req.specifier.prereleases:
-                            if req.name != "pulp-rpm-client":
-                                errors.append(
-                                    f"{filename}:{nr}: Prerelease versions found in {line}."
-                                )
+                    if check_prereleases and req.specifier.prereleases:
+                        # Do not even think about begging for more exceptions!
+                        if (
+                            not req.name.startswith("opentelemetry")
+                            and req.name != "pulp-rpm-client"
+                        ):
+                            errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.")
                     ops = [op for op, ver in req.specs]
                     spec = str(req.specs)
                     if "~=" in ops:

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-214-gf2ffaa4
+2021.08.26-227-g0a28ffe

.github/workflows/ci.yml

@@ -207,7 +207,7 @@ jobs:
           docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"
   deprecations:
     runs-on: ubuntu-latest
-    if: always()
+    if: github.base_ref == 'main'
     needs: test
     steps:
       - name: Fail on deprecations

.github/workflows/create-branch.yml

@@ -63,7 +63,11 @@ jobs:
           git checkout main
           bump2version --no-commit minor
 
-      - name: Make a PR with version bump
+      - name: Remove entries from CHANGES directory
+        run: |
+          find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} +
+
+      - name: Make a PR with version bump and without CHANGES/*
         uses: peter-evans/create-pull-request@v4
         with:
           token: ${{ secrets.RELEASE_TOKEN }}

.github/workflows/release.yml

@@ -15,7 +15,7 @@ on:
         required: true
       before_script:
         description: |
-          Bash code to run before script.sh is executed. This should only be used when re-running
+          Bash code to run before bindings and docs are built. This should only be used when re-running
           a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst
         required: false
 
@@ -65,21 +65,14 @@ jobs:
         with:
           name: pulp_rpm.tar
           path: pulp_rpm.tar
-  test:
-    needs: build-artifacts
 
-    runs-on: ubuntu-latest
 
-    strategy:
-      fail-fast: false
-      matrix:
-        env:
-          - TEST: pulp
-          - TEST: docs
-          - TEST: azure
-          - TEST: s3
-          - TEST: generate-bindings
-          - TEST: lowerbounds
+  build-bindings-docs:
+    needs: build-artifacts
+    runs-on: ubuntu-latest
+    # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests
+    env:
+      TEST: docs
 
     steps:
       - uses: actions/download-artifact@v3
@@ -112,10 +105,8 @@ jobs:
           echo ::endgroup::
           echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV
 
-      - name: Set environment variables
-        run: |
-          echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV
-
+      # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to
+      # setup the Pulp instance.
       - name: Before Install
         run: .github/workflows/scripts/before_install.sh
         shell: bash
@@ -144,74 +135,41 @@ jobs:
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
         shell: bash
 
-      - name: Before Script
-        run: .github/workflows/scripts/before_script.sh
+      - name: Additional before_script
+        run: ${{ github.event.inputs.before_script }}
         shell: bash
-        env:
-          PY_COLORS: '1'
-          ANSIBLE_FORCE_COLOR: '1'
-          GITHUB_PULL_REQUEST: ${{ github.event.number }}
-          GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
-          GITHUB_BRANCH: ${{ github.head_ref }}
-          GITHUB_REPO_SLUG: ${{ github.repository }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
-          REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }}
-
-      - name: Setting secrets
-        run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
-        env:
-          SECRETS_CONTEXT: ${{ toJson(secrets) }}
 
       - name: Install Python client
         run: .github/workflows/scripts/install_python_client.sh
         shell: bash
-
       - name: Install Ruby client
-        if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }}
         run: .github/workflows/scripts/install_ruby_client.sh
         shell: bash
 
-      - name: Additional before_script
-        run: ${{ github.event.inputs.before_script }}
-        shell: bash
-
-      - name: Script
-        if: ${{ env.TEST != 'generate-bindings' }}
-        run: .github/workflows/scripts/script.sh
-        shell: bash
-        env:
-          PY_COLORS: '1'
-          ANSIBLE_FORCE_COLOR: '1'
-          GITHUB_PULL_REQUEST: ${{ github.event.number }}
-          GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
-          GITHUB_BRANCH: ${{ github.head_ref }}
-          GITHUB_REPO_SLUG: ${{ github.repository }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
-
       - name: Upload python client packages
-        if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }}
         uses: actions/upload-artifact@v3
         with:
           name: python-client.tar
           path: python-client.tar
 
       - name: Upload python client docs
-        if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }}
         uses: actions/upload-artifact@v3
         with:
           name: python-client-docs.tar
           path: python-client-docs.tar
-
       - name: Upload ruby client packages
-        if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }}
         uses: actions/upload-artifact@v3
         with:
           name: ruby-client.tar
           path: ruby-client.tar
+      - name: Build docs
+        run: |
+          export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
+          export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py
+          make -C docs/ PULP_URL="https://pulp" diagrams html
+          tar -cvf docs/docs.tar docs/_build
+
       - name: Upload built docs
-        if: ${{ env.TEST == 'docs' }}
         uses: actions/upload-artifact@v3
         with:
           name: docs.tar
@@ -232,7 +190,7 @@ jobs:
 
   publish:
     runs-on: ubuntu-latest
-    needs: test
+    needs: build-bindings-docs
 
     env:
       TEST: publish
@@ -293,7 +251,7 @@ jobs:
 
       - name: Publish docs to pulpproject.org
         run: |
-          tar -xvf docs.tar -C ./docs
+          tar -xvf docs.tar
           .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }}
       - name: Deploy plugin to pypi
         run: bash .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }}
@@ -335,25 +293,3 @@ jobs:
 
       - name: Create release on GitHub
         run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }}
-
-      - name: Cleanup repository before making changelog PR
-        run: rm -rf .lock generation pulp_rpm_client* *-client.tar pulp_rpm.tar todo web *docs.tar
-
-      - name: Stage changelog for main branch
-        run: python .github/workflows/scripts/stage-changelog-for-default-branch.py ${{ github.event.inputs.release }}
-
-      - name: Create Pull Request for Changelog
-        uses: peter-evans/create-pull-request@v4
-        with:
-          token: ${{ secrets.RELEASE_TOKEN }}
-          committer: pulpbot <[email protected]>
-          author: pulpbot <[email protected]>
-          branch: changelog/${{ github.event.inputs.release }}
-          base: main
-          title: 'Cherry pick ${{ github.event.inputs.release }} changelog'
-          body: '[noissue]'
-          commit-message: |
-            ${{ github.event.inputs.release }} changelog
-
-            [noissue]
-          delete-branch: true

.github/workflows/scripts/update_ci.sh

@@ -1,31 +1,29 @@
 #!/usr/bin/env bash
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_rpm' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
 
 set -eu
 
-if [ ! -d ../plugin_template ]; then
-  echo "Checking out plugin_template"
-  git clone https://github.com/pulp/plugin_template.git ../plugin_template
-fi
-
-
 if [ ! -f "template_config.yml" ]; then
   echo "No template_config.yml detected."
   exit 1
 fi
 
 pushd ../plugin_template
-pip install -r test_requirements.txt
 ./plugin-template --github pulp_rpm
 popd
 
 # Check if only gitref file has changed, so no effect on CI workflows.
-if [[ `git diff --name-only` == ".github/template_gitref" ]]; then
-  echo "CI update has no effect on workflows, skipping PR creation."
-  git stash
-  exit 0
+if [[ $(git diff --name-only) == ".github/template_gitref" ]]; then
+  echo "No changes detected in github section."
+  git restore ".github/template_gitref"
 fi
 
-if [[ `git status --porcelain` ]]; then
+if [[ $(git status --porcelain) ]]; then
   git add -A
   git commit -m "Update CI files" -m "[noissue]"
 else