Merge branch 'main' into sphinx-deps-fix

.github/workflows/secrets-detection.yaml

@@ -0,0 +1,77 @@
+name: Secret Detection Workflow
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+        branches:
+            - main
+
+jobs:
+    secret-detection:
+        runs-on: ubuntu-latest
+        steps:
+            -
+                name: Checkout code
+                uses: actions/checkout@v4
+            -
+                name: Install necessary packages
+                run: |
+                    pip install git+https://github.com/NASA-AMMOS/slim-detect-secrets.git@exp
+                    pip install jq
+
+            -
+                name: Create an initial .secrets.baseline if .secrets.baseline does not exist
+                run: |
+                    if [ ! -f .secrets.baseline ]; then
+                        # This generated baseline file will only be temporarily available on the GitHub side and will not appear in the user's local files.
+                        # Scanning an empty folder to generate an initial .secrets.baseline without secrets in the results.
+                        echo "⚠️ No existing .secrets.baseline file detected. Creating a new blank baseline file."
+                        mkdir empty-dir
+                        detect-secrets scan empty-dir > .secrets.baseline
+                        echo "✅ Blank .secrets.baseline file created successfully."
+                        rm -r empty-dir
+                    else
+                        echo "✅ Existing .secrets.baseline file detected. No new baseline file will be created."
+                    fi
+
+            -
+                name: Scan repository for secrets
+                run: |
+                    # scripts to scan repository for new secrets
+
+                    # backup the list of known secrets
+                    cp .secrets.baseline .secrets.new
+
+                    # find the secrets in the repository
+                    detect-secrets scan --disable-plugin AbsolutePathDetectorExperimental --baseline .secrets.new \
+                        --exclude-files '\.secrets..*' \
+                        --exclude-files '\.git.*' \
+                        --exclude-files '\.pre-commit-config\.yaml' \
+                        --exclude-files '\.mypy_cache' \
+                        --exclude-files '\.pytest_cache' \
+                        --exclude-files '\.tox' \
+                        --exclude-files '\.venv' \
+                        --exclude-files 'venv' \
+                        --exclude-files 'dist' \
+                        --exclude-files 'build' \
+                        --exclude-files '.*\.egg-info'
+
+                    # if there is any difference between the known and newly detected secrets, break the build
+                    # Function to compare secrets without listing them
+                    compare_secrets() { diff <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$1" | sort) <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$2" | sort) >/dev/null; }
+
+                    # Check if there's any difference between the known and newly detected secrets
+                    if ! compare_secrets .secrets.baseline .secrets.new; then
+                        echo "⚠️ Attention Required! ⚠️" >&2
+                        echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2
+                        echo "" >&2
+                        echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2
+                        echo "" >&2
+                        echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2
+                        echo "" >&2
+                        echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2
+                        echo "" >&2
+                        echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2
+                        exit 1
+                    fi

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+    -
+        repo: https://github.com/NASA-AMMOS/slim-detect-secrets
+        # using commit id for now, will change to tag when official version is released
+        rev: 91e097ad4559ae6ab785c883dc5ed989202c7fbe
+        hooks:
+          - id: detect-secrets
+            args:
+              - '--baseline'
+              - '.secrets.baseline'
+              - --exclude-files '\.secrets..*'
+              - --exclude-files '\.git.*'
+              - --exclude-files '\.mypy_cache'
+              - --exclude-files '\.pytest_cache'
+              - --exclude-files '\.tox'
+              - --exclude-files '\.venv'
+              - --exclude-files 'venv'
+              - --exclude-files 'dist'
+              - --exclude-files 'build'
+              - --exclude-files '.*\.egg-info'

.secrets.baseline

@@ -0,0 +1,178 @@
+{
+  "version": "1.4.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AWSSensitiveInfoDetectorExperimental"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "EmailAddressDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "IPPublicDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    },
+    {
+      "path": "detect_secrets.filters.regex.should_exclude_file",
+      "pattern": [
+        "\\.secrets..*",
+        "\\.pre-commit-config\\.yaml",
+        "\\.git.*",
+        "\\.mypy_cache",
+        "\\.pytest_cache",
+        "\\.tox",
+        "\\.venv",
+        "venv",
+        "dist",
+        "build",
+        ".*\\.egg-info"
+      ]
+    }
+  ],
+  "results": {
+    "README.md": [
+      {
+        "type": "Email Address",
+        "filename": "README.md",
+        "hashed_secret": "0ebdbaa404ab765b45b3af96c0e1874401ac3ef3",
+        "is_verified": false,
+        "line_number": 94
+      }
+    ],
+    "action.yaml": [
+      {
+        "type": "Email Address",
+        "filename": "action.yaml",
+        "hashed_secret": "1b21650fca3caf5c234a77fcc47fb5f08cfcbd8a",
+        "is_verified": false,
+        "line_number": 11
+      }
+    ],
+    "src/pds/roundup/step.py": [
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "src/pds/roundup/step.py",
+        "hashed_secret": "4e8f0d758c9cb925cd6e21fe0e99d29b90417650",
+        "is_verified": false,
+        "line_number": 246
+      }
+    ],
+    "src/pds/roundup/util.py": [
+      {
+        "type": "Email Address",
+        "filename": "src/pds/roundup/util.py",
+        "hashed_secret": "2cdaeb7565d9036f422d87494886f0295a6e6cd3",
+        "is_verified": false,
+        "line_number": 111
+      }
+    ]
+  },
+  "generated_at": "2023-12-07T15:42:21Z"
+}

Dockerfile

@@ -26,5 +26,6 @@ ENTRYPOINT ["/usr/local/bin/roundup"]
 
 RUN : &&\
     pip install 'lasso.releasers~=1.0.0' 'lasso.requirements~=1.0.0' &&\
-    python3 setup.py install --optimize=2 &&\
+    pip install 'git+https://github.com/NASA-PDS/lasso-issues.git@main' &&\
+    pip install install /usr/src/roundup &&\
     :

README.md

@@ -121,12 +121,17 @@ There are several different flavors of roundups that you can specify `with` the
 
 The Roundup includes built-in support to make official releases of software, publishing artifacts to well-known repositories, and including release archives on GitHub. The [PDS Java Template Repository](https://github.com/NASA-PDS/pds-template-repo-java) (historically called the "generic template") and the [PDS Python Template Repository](https://github.com/NASA-PDS/pds-template-repo-python) (historically called the Python template) have the correct GitHub Actions workflows to support this. If you create a new PDS repository from those templates, you're all set to roundup! Yee-haw!
 
-To make an offical release of software version `VERSION`, create a tag called `release/VERSION` and push it to GitHub. For example, to release version 2.0.17 of your software based on the latest `main`:
+To make an offical release of software version `VERSION`, create a tag called `release/VERSION` and push it to GitHub. For example, to release version 2.1.0 of your software based on the latest `main`:
 ```console
 $ git checkout main
 $ git pull
-$ git tag --annotate --message "Release of 2.0.17" release/2.0.17
-$ git push origin release/2.0.17
+$ git tag --annotate --message "Release of 2.1.0" release/2.1.0
+$ git push origin release/2.1.0
+```
+If a release fails, you can retry it under some circumstances (depending on where it failed) with an invocation like:
+```console
+$ git push --delete release/2.1.0
+$ git push origin release/2.1.0
 ```
 
 

setup.cfg

@@ -24,31 +24,27 @@ packages = find_namespace:
 package_dir =
     =src
 
-# Note: the ``install_requires`` dependencies below are for development only;
-# for operations, the base Docker image has all of these pre-installed to
-# save time.
+# Note: the ``install_requires`` dependencies below must match certain
+# packages "baked into" the nasapds/github-actions-base which is used
+# by GitHub Actions to save time on spinning up the Roundup's container.
 
 install_requires =
-# later versions of alabaster require sphinx>=3.4
     alabaster <=0.7.13
-    github3.py
-    pds-github-util
-    requests==2.23.0
+    github3.py==1.3.0
+    lxml==4.6.3
     packaging==21.0
-    sphinx~=3.2.1
+    requests==2.23.0
     sphinx-rtd-theme~=0.5.0
-
-# unclear why the following pins are necessary, but without them, versions dependent on sphinx>=5.0 are installed
+    sphinx-substitution-extensions==2020.9.30.0
+    sphinx==3.2.1
     sphinxcontrib-applehelp==1.0.4
     sphinxcontrib-devhelp==1.0.2
     sphinxcontrib-htmlhelp==2.0.1
     sphinxcontrib-jsmath==1.0.1
     sphinxcontrib-qthelp==1.0.3
     sphinxcontrib-serializinghtml==1.1.5
-
-    twine
-    wheel
-    lxml
+    twine==3.4.2
+    wheel==0.40.0
 
 
 [options.package_data]

src/pds/roundup/_maven.py

@@ -6,7 +6,7 @@
 from .errors import InvokedProcessError, MissingEnvVarError, RoundupError
 from .step import ChangeLogStep as BaseChangeLogStep
 from .step import Step, StepName, NullStep, DocPublicationStep, RequirementsStep
-from .util import invoke, invokeGIT, TAG_RE, git_config, delete_tags
+from .util import invoke, invokeGIT, TAG_RE, git_config, delete_tags, add_version_label_to_open_bugs
 from lxml import etree
 import logging, os, base64, subprocess, re
 
@@ -36,6 +36,7 @@ def __init__(self, cwd, environ, args):
             StepName.requirements:        RequirementsStep,
             StepName.unitTest:            _UnitTestStep,
             StepName.versionBump:         _VersionBumpingStep,
+            StepName.versionCommit:       _VersionCommittingStep,
         }
         super(MavenContext, self).__init__(cwd, environ, args)
 
@@ -299,15 +300,30 @@ def execute(self):
         if not match:
             raise RoundupError(f'🐎 Stable workflow on tag «{tag}» but not a ``release/`` name!')
         major, minor, micro = int(match.group(1)), int(match.group(2)), match.group(4)
-        _logger.debug('🔖 So we got version %d.%d.%s', major, minor, micro)
+        full_version = f'{major}.{minor}.{micro}'
+        _logger.debug('🔖 So we got version %s', full_version)
+        add_version_label_to_open_bugs(full_version)
         if micro is None:
             raise RoundupError('Invalid release version supplied in tag name. You must supply Major.Minor.Micro')
         self.invokeMaven(['-DgenerateBackupPoms=false', f'-DnewVersion={major}.{minor}.{micro}', 'versions:set'])
         _logger.debug('❗️ After I ran `mvn versions:set`, here is what the pom.xml looks like as far as <version>')
         with open('pom.xml', 'r') as f:
             for 𝐋 in f:
                 if 'version' in 𝐋: _logger.debug(f'“{𝐋.strip()}”')
-        self.commit_poms(f'Bumping version for {major}.{minor}.{micro} release')
+
+
+class _VersionCommittingStep(_MavenStep):
+    '''Step that commits the new version, as needed.'''
+    def execute(self):
+        '''Commit the new version number.'''
+        if not self.assembly.isStable():
+            _logger.debug('Skipping version commit for unstable build')
+            return
+        _logger.debug('❗️ Inside the _VersionCommittingStep, here is what the pom.xml looks like as far as <version>')
+        with open('pom.xml', 'r') as f:
+            for 𝐋 in f:
+                if 'version' in 𝐋: _logger.debug(f'“{𝐋.strip()}”')
+        self.commit_poms('Committing poms for stable release')
 
 
 class _CleanupStep(_MavenStep):