Merge pull request #484 from jkloetzke/unify-backends

Unify backends

.github/workflows/workflow.yaml

@@ -3,7 +3,7 @@ name: CI/CD
 on: [push, pull_request]
 
 jobs:
-    build:
+    linux:
         runs-on: ubuntu-latest
         timeout-minutes: 15
         strategy:
@@ -20,6 +20,12 @@ jobs:
               with:
                 python-version: ${{ matrix.python-version }}
 
+            - name: Cache Jenkins integration test
+              uses: actions/cache@v3
+              with:
+                path: test/integration/jenkins/cache
+                key: ${{ runner.os }}
+
             - name: Install dependencies
               run: |
                 python -m pip install --upgrade pip
@@ -49,3 +55,50 @@ jobs:
                 password: ${{ secrets.PYPI_API_TOKEN }}
                 skip_existing: true
 
+    windows:
+        runs-on: windows-latest
+        timeout-minutes: 20
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v2
+
+            - name: Set up Python 3.10
+              uses: actions/setup-python@v2
+              with:
+                python-version: "3.10"
+
+            - name: Prepare MSYS2 environment
+              run: |
+                C:\msys64\usr\bin\bash -l -c "pacman -Sy --needed --noconfirm parallel || true"
+
+            - name: Cache Jenkins integration test
+              uses: actions/cache@v3
+              with:
+                path: test/integration/jenkins/cache
+                key: ${{ runner.os }}
+
+            - name: Install dependencies
+              run: |
+                python -m pip install --upgrade pip
+                pip install PyYAML coverage schema python-magic pyparsing sphinx wheel
+
+            - name: Run unit tests
+              run: |
+                git config --global init.defaultBranch master # keep the old name
+                $env:PATH += ";C:\msys64\usr\bin"
+                bash ./test/run-tests.sh -c xml
+
+            - name: Build Python package
+              run: |
+                python3 setup.py sdist bdist_wheel
+
+            - name: Upload coverage to Codecov
+              uses: codecov/codecov-action@v2
+
+            - name: Publish package
+              if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+              uses: pypa/gh-action-pypi-publish@release/v1
+              with:
+                user: __token__
+                password: ${{ secrets.PYPI_API_TOKEN }}
+                skip_existing: true

bob

@@ -7,6 +7,10 @@ scr_path = os.path.dirname(os.path.realpath(__file__))
 pym_path = os.path.join(scr_path, 'pym')
 sys.path.insert(0, pym_path)
 
+if os.environ.get("COVERAGE_PROCESS_START"):
+    import coverage
+    coverage.process_startup()
+
 from bob.scripts import bob
 
 if __name__ == '__main__':

bob.cmd

@@ -4,5 +4,4 @@ SETLOCAL ENABLEEXTENSIONS ENABLEDELAYEDEXPANSION
 SET bobPath=%~dp0
 SET bobPath=%bobPath:~0,-1%
 
-set PYTHONPATH=%PYTHONPATH%;%bobPath%\pym
-python -c "from bob.scripts import bob; bob(r\"%~f0\")" %*
+python %bobPath%\bob %*

doc/manpages/bob-jenkins.rst

@@ -21,27 +21,34 @@ Available sub-commands:
 
 ::
 
-    bob jenkins add [-h] [-n NODES] [-o OPTIONS] [-w] [-p PREFIX] [-r ROOT]
-                    [-D DEFINES] [--keep] [--download] [--upload]
+    bob jenkins add [-h] [-n NODES] [-o OPTIONS]
+                    [--host-platform {linux,msys,win32}] [-w] [-p PREFIX]
+                    [-r ROOT] [-D DEFINES] [--keep] [--download] [--upload]
                     [--no-sandbox] [--credentials CREDENTIALS] [--clean]
-                    [--shortdescription] [--longdescription]
+                    [--shortdescription | --longdescription]
                     name url
     bob jenkins export [-h] name dir
     bob jenkins graph [-h] name
     bob jenkins ls [-h] [-v]
     bob jenkins prune [-h] [--obsolete | --intermediate] [--no-ssl-verify]
-                      [-q] [-v]
+                      [--user USER] [--password PASSWORD] [-q] [-v]
                       name
-    bob jenkins push [-h] [-f] [--no-ssl-verify] [--no-trigger] [-q] [-v]
+    bob jenkins push [-h] [-f] [--no-ssl-verify] [--no-trigger]
+                     [--user USER] [--password PASSWORD] [-q] [-v]
                      name
     bob jenkins rm [-h] [-f] name
-    bob jenkins set-options [-h] [--reset] [-n NODES] [-o OPTIONS] [-p PREFIX]
-                            [--add-root ADD_ROOT] [--del-root DEL_ROOT]
-                            [-D DEFINES] [-U UNDEFINES] [--credentials CREDENTIALS]
-                            [--keep | --no-keep] [--download | --no-download]
-                            [--upload | --no-upload] [--sandbox | --no-sandbox]
-                            [--clean | --incremental] [--autotoken AUTHTOKEN]
-                            [--shortdescription]
+    bob jenkins set-options [-h] [--reset] [-n NODES] [-o OPTIONS]
+                            [--host-platform {linux,msys,win32}]
+                            [-p PREFIX] [--add-root ADD_ROOT]
+                            [--del-root DEL_ROOT] [-D DEFINES]
+                            [-U UNDEFINES] [--credentials CREDENTIALS]
+                            [--authtoken AUTHTOKEN]
+                            [--shortdescription | --longdescription]
+                            [--keep | --no-keep]
+                            [--download | --no-download]
+                            [--upload | --no-upload]
+                            [--sandbox | --no-sandbox]
+                            [--clean | --incremental]
                             name
     bob jenkins set-url [-h] name url
 
@@ -76,6 +83,13 @@ Options
 ``-f, --force``
     Overwrite existing jobs
 
+``--host-platform``
+    Jenkins host platform type. May be any of ``linux``, ``msys`` or ``win32``.
+
+    This specifies the host operating system where the Jenkins master and the
+    build slaves are running. By default this is the type of the current
+    operating system.
+
 ``--incremental``
     Reuse workspace for incremental builds
 
@@ -126,6 +140,19 @@ Options
 ``-p PREFIX, --prefix PREFIX``
     Prefix for jobs
 
+``--password``
+    Set password for Jenkins authentication.
+
+    You can also set the user name and password persistently by encoding it
+    into the Jenkins url directly, e.g. *https:://user:password@host/*.
+
+    .. attention::
+       On Linux users can usually see the program arguments of processes from
+       other users. By using the ``--password`` you could inadvertently reveal
+       the password to untrusted other users that have access to the same
+       machine.  It is safer to either enter the password manually or to pipe
+       it through stdin.
+
 ``-q, --quiet``
     Decrease verbosity (may be specified multiple times)
 
@@ -148,11 +175,18 @@ Options
 ``--upload``
     Upload to binary archive
 
+``--user``
+    Set user name for Jenkins authentication.
+
+    You can also set the user name persistently by encoding it into the Jenkins
+    url directly, e.g. *https:://user@host/*.
+
 ``-v, --verbose``
     Show additional information
 
 ``-w, --windows``
-    Jenkins is running on Windows. Produce cygwin compatible scripts.
+    Jenkins is running on Windows. Produce MSYS2 compatible scripts. This
+    option has been deprecated in favour of ``--host-platform``.
 
 Commands
 --------

doc/manual/extending.rst

@@ -214,6 +214,7 @@ Currently the following additionaly kwargs are passed:
    the job
  * ``checkoutSteps``: list of all checkout steps (:class:`bob.input.Step`) used
    in the job
+ * ``hostPlatform``: Jenkins host platform type (``linux``, ``msys`` or ``win32``)
  * ``name``: name of Jenkins job
  * ``nodes``: The nodes where the job should run
  * ``packageSteps``: list of all package steps (:class:`bob.input.Step`) used

doc/tutorial/compile.rst

@@ -240,11 +240,10 @@ requires that the following plugins are available:
 .. _Conditional BuildStep Plugin: https://wiki.jenkins-ci.org/display/JENKINS/Conditional+BuildStep+Plugin
 .. _Workspace Cleanup Plugin: https://wiki.jenkins-ci.org/display/JENKINS/Workspace+Cleanup+Plugin
 
-Additionally some of the Bob helper tools must be installed on the Jenkins
-server and be available in the PATH. The ``bob-hash-engine`` and
-``bob-audit-engine`` scripts are always needed. If you're using the sandbox
-feature ``bob-namespace-sandbox`` must be available too. To keep the setup
-simple it is recommended to install Bob entirely on the server.
+Additionally Bob must be installed on the Jenkins server and be available in
+the PATH. It is required to install the same version of Bob on the server and
+the build nodes that is used to configure the Jenkins. Otherwise the build will
+fail.
 
 Suppose you have a suitable Jenkins server located at
 http://jenkins.intranet.local:8080. Go to the project root directory and tell Bob

pym/bob/__init__.py

@@ -5,13 +5,29 @@
 
 import sys
 
+def filterDirs(dirs, exclude):
+    i = 0
+    while i < len(dirs):
+        if dirs[i] in exclude:
+            del dirs[i]
+        else:
+            i += 1
+
 def getBobInputHash():
-    from .utils import hashDirectory
-    import os
+    import hashlib
+    import os, os.path
     # we need the source hash to invalidate the cache in case of source code changes.
     # therefore it's enough to hash the pym directory without the entries of cmds-Dir
     root = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')
-    return hashDirectory(root, ignoreDirs=['__pycache__', 'cmds'])
+    exclude = frozenset(['__pycache__', 'cmds'])
+    h = hashlib.sha1()
+    for root, dirs, files in os.walk(root):
+        filterDirs(dirs, exclude)
+        dirs.sort()
+        for fn in sorted(files):
+            with open(os.path.join(root, fn), "rb") as f:
+                h.update(f.read())
+    return h.digest()
 
 # First try to see if we're running a development version. If we do we take the
 # version from git and make sure everything is up-to-date. Otherwise Bob was