Merge branch 'master' into add_env_option_serverless

.github/workflows/integration.yml

@@ -96,3 +96,39 @@ jobs:
           commit: ${{ steps.vars.output.tag_sha }}
           tag: ${{ github.event.inputs.tag }}
 
+  build-conda:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      CCTOOLS_OUTPUT: ${{ format('cctools-{0}-x86_64-linux-conda.tar.gz', github.event.inputs.version) }}
+    steps:
+      - name: checkout CCTools from branch head
+        if: github.event_name != 'workflow_dispatch'
+        uses: actions/checkout@v3
+      - name: checkout CCTools from tag
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.tag }}
+      - name: Get sha of tag
+        id: vars
+        shell: bash
+        run: echo "{tag_sha}=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: build
+        run: ${GITHUB_WORKSPACE}/packaging/build-conda/build.sh
+      - name: deploy
+        uses: ncipollo/release-action@v1
+        if: github.event_name == 'workflow_dispatch'
+        with:
+          artifacts: /tmp/${{ env.CCTOOLS_OUTPUT }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          allowUpdates: true
+          artifactContentType: application/gzip
+          draft: true
+          omitBody: true
+          omitBodyDuringUpdate: true
+          prerelease: true
+          replacesArtifacts: true
+          commit: ${{ steps.vars.output.tag_sha }}
+          tag: ${{ github.event.inputs.tag }}
+

packaging/build-conda/README.md

@@ -0,0 +1,8 @@
+This build script performs a build-and-test using only
+the standard developer dependencies from conda, so as
+to provide a test that matches what happens in conda-forge.
+
+Note that the conda packages given as developer dependencies
+here are repeated in several different places.  We should have
+a single location where they are clearly specified.
+

packaging/build-conda/build.sh

@@ -0,0 +1,51 @@
+#! /bin/bash
+
+set -xe
+
+# Save the dir from which the script was called
+ORG_DIR=$(pwd)
+
+# Find cctools src directory
+CCTOOLS_SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
+
+# Ensure we end up in the directory we started regardless of how the script
+# ends.
+function finish {
+    cd ${ORG_DIR}
+}
+trap finish EXIT
+
+# Fix for local environment at ND: unset PYTHONPATH to ignore existing python installs.
+export PYTHONPATH=
+
+# Activate the Conda shell hooks without starting a new shell.
+CONDA_BASE=$(conda info --base)
+. $CONDA_BASE/etc/profile.d/conda.sh
+
+# Install conda developer dependencies first:
+conda create -y -n cctools-dev -c conda-forge --strict-channel-priority python=3 gcc_linux-64 gxx_linux-64 gdb m4 perl swig make zlib libopenssl-static openssl conda-pack cloudpickle packaging
+conda activate cctools-dev
+
+# Leave out some items that are research prototypes.
+DISABLED_SYS=$(echo --without-system-{parrot,prune,umbrella,weaver})
+DISABLED_LIB=$(echo --with-{readline,fuse,perl}-path\ no)
+
+# Now build and configure in the normal way.
+./configure --strict ${DISABLED_SYS} ${DISABLED_LIB} "$@"
+[[ -f config.mk ]] && make clean
+echo === Contents of config.mk ===
+cat config.mk
+
+make
+
+make install
+
+if ! make test
+then
+    echo === Contents of cctools.test.fail ===
+    cat cctools.test.fail
+    exit 1
+else
+    exit 0
+fi
+

packaging/scripts/generate-images

@@ -139,17 +139,17 @@ my %postinstall_for;
 $versions_of{centos} = [ qw{ 7 } ];
 $command_for{centos} = 'yum install -y';
 
-$versions_of{almalinux} = [ qw{ 8 9 } ];
+$versions_of{almalinux} = [ qw{ 8 } ];
 $command_for{almalinux} = 'dnf install -y';
 
-$versions_of{fedora} = [ qw{ 30  } ];
-$command_for{fedora} = 'dnf install -y';
+#$versions_of{fedora} = [ qw{ 30  } ];
+#$command_for{fedora} = 'dnf install -y';
 
-$versions_of{debian} = [ qw{ 9.9 } ];
-$command_for{debian} = 'apt-get install -y';
+#$versions_of{debian} = [ qw{ 9.9 } ];
+#$command_for{debian} = 'apt-get install -y';
 
 $versions_of{ubuntu} = [ qw{ 16.04 20.04 } ];
-$command_for{ubuntu} = $command_for{debian};
+$command_for{ubuntu} = 'apt-get install -y';
 
 # e.g., $package_for{distro}{version}{dependency} == 'package'
 my %package_for;
@@ -373,7 +373,7 @@ $postinstall_for{ubuntu}{default} = $postinstall_for{debian}{default};
 
 # finishing installation steps for all:
 my $epilogue = [
-	'python3 -m pip install cloudpickle'
+	'python3 -m pip install cloudpickle conda-pack packaging'
 ];
 
 

parrot/test/TR_parrot_execve.sh

@@ -16,6 +16,7 @@ prepare()
 {
 	chirp_start local
 	cp ${exe} ${root}/hello
+	${root}/hello > expected.txt
 
 	echo "$hostport" > config.txt
 
@@ -29,7 +30,7 @@ run()
 
 	parrot --no-chirp-catalog --timeout=5 --work-dir="/chirp/${hostport}/" ./hello > output.txt
 
-	if [ "$(cat output.txt)" == "Hello, world!" ]
+	if diff output.txt expected.txt
 	then
 		return 0
 	else

parrot/test/TR_parrot_stat.sh

@@ -20,26 +20,28 @@ prepare()
 	return 0
 }
 
+FORMAT="%a %b %g %h %i %s %u %W %X %Y %Z"
+
 run()
 {
 	set -e
 
-	stat $file $link 2>/dev/null > $expected
-	parrot -- stat $file $link > $from_parrot
+	stat --format "$FORMAT" $file $link 2>/dev/null > $expected
+	parrot -- stat --format "$FORMAT" $file $link > $from_parrot
 
 	diff $expected $from_parrot
 
-	if ! parrot --check-driver cvmfs
+	if ! parrot_run --check-driver cvmfs
 	then
 		return 0
 	fi
 
 	# check that symlinks are correctly detected
 	cvmfs_symlink=/cvmfs/cms.cern.ch/bin/scramv1
-	parrot -t${tmp_dir} -- sh -c "[ -L ${cvmfs_symlink} ]";
+	parrot_run -t${tmp_dir} -- sh -c "[ -L ${cvmfs_symlink} ]";
 
 	target=$(parrot -t${tmp_dir} -- realpath $cvmfs_symlink | tail -n1)
-	parrot -t${tmp_dir}  /bin/sh -c "[ -f $target ]";
+	parrot_run -t${tmp_dir}  /bin/sh -c "[ -f $target ]";
 
 	return 0
 }

poncho/src/poncho/package_serverize.py

@@ -5,8 +5,9 @@
 # See the file COPYING for details.
 
 
-from poncho import package_analyze as analyze
-from poncho import package_create as create
+from ndcctools.poncho import package_analyze as analyze
+from ndcctools.poncho import package_create as create
+
 import argparse
 import json
 import os

poncho/src/poncho_package_analyze

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from poncho import package_analyze as analyze
+from ndcctools.poncho import package_analyze as analyze
 import argparse
 import ast
 import json

poncho/src/poncho_package_create

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from poncho import package_create as create
+from ndcctools.poncho import package_create as create
 import argparse
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Create a packed environment from a spec, a conda environment name, or a conda directory.')

poncho/src/poncho_package_serverize

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from poncho import package_serverize as serverize
+from ndcctools.poncho import package_serverize as serverize
 import argparse
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Creates Library code based on an input file and function. Allows the running of FunctionCall instances.")

taskvine/src/bindings/python3/ndcctools/taskvine/manager.py

@@ -46,12 +46,6 @@
 import time
 import weakref
 
-try:
-    from poncho import package_serverize
-    poncho_available = True
-except Exception:
-    poncho_available = False
-
 
 ##
 # @class ndcctools.taskvine.Manager
@@ -858,9 +852,13 @@ def remove_library(self, name):
     # @param add_env         Whether to automatically create and/or add environment to the library
     # @returns               A task to be used with @ref ndcctools.taskvine.manager.Manager.install_library.
     def create_library_from_functions(self, name, *function_list, poncho_env=None, init_command=None, add_env=True):
+        # Delay loading of poncho until here, to avoid bringing in conda-pack etc unless needed.
         # ensure poncho python library is available
-        if not poncho_available:
+        try:
+            from ndcctools.poncho import package_serverize
+        except:
             raise ModuleNotFoundError("The poncho module is not available. Cannot create Library.")
+
         # positional arguments are the list of functions to include in the library
         # create a unique hash of a combination of function names and bodies
         functions_hash = package_serverize.generate_functions_hash(function_list)
@@ -917,8 +915,9 @@ def create_library_from_functions(self, name, *function_list, poncho_env=None, i
     #                        to a poncho environment.
     # @returns               A task to be used with @ref ndcctools.taskvine.manager.Manager.install_library.
     def create_library_from_serverized_files(self, name, library_path, env=None):
-        if not poncho_available:
-            raise ModuleNotFoundError("The poncho module is not available. Cannot create library.")
+        # Delay loading of poncho until here, to avoid bringing in conda-pack etc unless needed.
+        from ndcctools.poncho import package_serverize
+
         t = LibraryTask("python ./library_code.py", name)
         if env:
             if isinstance(env, str):

taskvine/test/TR_vine_allocations.sh

@@ -33,7 +33,7 @@ run()
 	disk=2000
 	gpus=8
 
-	# send makeflow to the background, saving its exit status.
+	# send taskvine to the background, saving its exit status.
 	${CCTOOLS_PYTHON_TEST_EXEC} vine_alloc_test.py $PORT_FILE $cores $memory $disk $gpus; echo $? > $STATUS_FILE
 
 	# retrieve wq script exit status