Merge branch 'cooperative-computing-lab:master' into pq_ready_tasks

.github/workflows/testing.yml

@@ -28,7 +28,7 @@ jobs:
       CCTOOLS_DOCKER_GITHUB: ${{ matrix.os-name }}
     strategy: 
       matrix:
-        os-name: ['almalinux8', 'almalinux9', 'ubuntu20.04']
+        os-name: ['almalinux8', 'almalinux9', 'ubuntu20.04', 'ubuntu22.04', 'ubuntu24.04']
     steps:
       - name: checkout CCTools from branch head
         if: github.event_name != 'workflow_dispatch'

batch_job/src/vine_factory.c

@@ -136,6 +136,9 @@ struct hash_table *features_table = NULL;
 // Disable the check for invalid use of AFS with HTCondor.
 static int disable_afs_check = 0;
 
+// Remove workers upon manager disconnection
+static int single_shot = 0;
+
 /*
 In a signal handler, only a limited number of functions are safe to
 invoke, so we construct a string and emit it with a low-level write.
@@ -447,10 +450,10 @@ static int submit_worker( struct batch_queue *queue )
 	}
 
 	char *features_string = make_features_string(features_table);
-	
+
 	if(using_catalog) {
 		cmd = string_format(
-		"./%s --parent-death -M %s -t %d -C '%s' %s %s %s %s %s %s %s",
+		"./%s --parent-death -M %s -t %d -C '%s' %s %s %s %s %s %s %s %s",
 		worker_command,
 		submission_regex,
 		worker_timeout,
@@ -461,11 +464,12 @@ static int submit_worker( struct batch_queue *queue )
 		resource_args ? resource_args : "",
 		manual_ssl_option ? "--ssl" : "",
 		features_string,
+		single_shot ? "--single-shot" : "",
 		extra_worker_args ? extra_worker_args : ""
 		);
 	} else {
 		cmd = string_format(
-		"./%s --parent-death %s %d -t %d -C '%s' %s %s %s %s %s %s",
+		"./%s --parent-death %s %d -t %d -C '%s' %s %s %s %s %s %s %s",
 		worker_command,
 		manager_host,
 		manager_port,
@@ -476,6 +480,7 @@ static int submit_worker( struct batch_queue *queue )
 		resource_args ? resource_args : "",
 		manual_ssl_option ? "--ssl" : "",
 		features_string,
+		single_shot ? "--single-shot" : "",
 		extra_worker_args ? extra_worker_args : ""
 		);
 	}
@@ -996,7 +1001,7 @@ static void mainloop( struct batch_queue *queue )
 				}
 			}
 		}
-	
+
 		debug(D_VINE,"evaluating manager list...");
 		int workers_connected = count_workers_connected(managers_list);
 		int workers_needed = 0;
@@ -1237,6 +1242,7 @@ enum{   LONG_OPT_CORES = 255,
 		LONG_OPT_FACTORY_NAME,
 		LONG_OPT_DEBUG_WORKERS,
 		LONG_OPT_DISABLE_AFS_CHECK,
+		LONG_OPT_SINGLE_SHOT,
 };
 
 static const struct option long_options[] = {
@@ -1285,6 +1291,7 @@ static const struct option long_options[] = {
 	{"ssl",no_argument, 0, LONG_OPT_USE_SSL},
 	{"tls-sni", required_argument, 0, LONG_OPT_TLS_SNI},
 	{"factory-name",required_argument, 0, LONG_OPT_FACTORY_NAME},
+	{"single-shot", no_argument, 0, LONG_OPT_SINGLE_SHOT},
 	{0,0,0,0}
 };
 
@@ -1478,6 +1485,9 @@ int main(int argc, char *argv[])
 			case LONG_OPT_DISABLE_AFS_CHECK:
 				disable_afs_check = 1;
 				break;
+			case LONG_OPT_SINGLE_SHOT:
+				single_shot = 1;
+				break;
 			default:
 				show_help(argv[0]);
 				return EXIT_FAILURE;

configure

@@ -111,7 +111,7 @@ config_swig_path=auto
 config_perl_path=no
 config_golang_path=no
 config_curl_path=no
-config_cvmfs_path=no
+config_cvmfs_path=auto
 config_globus_path=no
 config_irods_path=no
 config_uuid_path=no
@@ -702,11 +702,12 @@ then
 	fuse_path=${fuse_path:-${base_root}}
 	if library_search fuse ${fuse_path} || library_search fuse /
 	then
+		fuse_ccflags="-DHAS_FUSE"
 		if [ "${fuse_path}" != / -a "${fuse_path}" != ${base_root} ]
 		then
-			fuse_ccflags="-I${fuse_path}/include"
+			fuse_ccflags="${fuse_ccflags} -I${fuse_path}/include"
 		fi
-		fuse_ldflags="-DHAS_FUSE ${library_search_result}"
+		fuse_ldflags="${library_search_result}"
 	else
 		fuse_avail=no
 	fi

doc/man/md/vine_factory.md

@@ -86,6 +86,8 @@ Also configurable through environment variables **CCTOOLS_TEMP** or **TMPDIR**
 - **--tls-sni=_&lt;&gt;_**<br />SNI domain name if different from manager hostname. Implies --ssl.
 - **-v**,**--version**<br /> Show the version string.
 - **-h**,**--help**<br /> Show this screen.
+- **--debug-workers**<br /> Bring the woker logs back to the scratch directory, usually work with **--single-shot**
+- **--single-shot**<br /> Shut down workers gracefully once their connected manager is done
 
 
 Concurrent control options:
@@ -172,6 +174,8 @@ If running on condor, you may manually specify condor requirements:
 vine_factory -T condor -M barney --condor-requirements 'MachineGroup == "disc"' --condor-requirements 'has_matlab == true'
 ```
 
+If 
+
 Repeated uses of **condor-requirements** are and-ed together. The previous example will produce a statement equivalent to:
 
 **requirements = ((MachineGroup == "disc") && (has_matlab == true))**

packaging/build-docker/configure-from-image

@@ -16,36 +16,13 @@ function finish {
 trap finish EXIT
 
 
-# dependencies to be included statically in parrot live in /opt/vc3 in the
-# container images.
-DEPS_DIR=/opt/vc3/cctools-deps
-DEPS=$(/bin/ls "$DEPS_DIR" || true)
-DEP_ARGS=""
-for dep in $DEPS; do
-    DEP_ARGS="$DEP_ARGS --with-$dep-path $DEPS_DIR/$dep"
-done
-
 cd ${CCTOOLS_SRC}
 
-if [[ -d /opt/vc3/cctools-deps/musl ]]
-then
-    # compile work_queue binaries statically
-    [[ -f config.mk ]] && make clean
-    CFLAGS=-D__MUSL__ CC=/opt/vc3/cctools-deps/musl/bin/musl-gcc LD=/opt/vc3/cctools-deps/musl/bin/musl-gcc ./configure --without-system-{doc,apps,chirp} --with-readline-path=no --static  --with-zlib-path=/opt/vc3/cctools-deps/musl-zlib "$@"
-    (cd dttools/src && make)
-    (cd work_queue/src && make)
-    (cp work_queue/src/work_queue_{worker,status,example} .)
-    STATIC_WORKER=1
-fi
-
 # disable perl bindings (to compile as we do in conda)
 perl_option='--with-perl-path no'
 
-# using rpm/deb for cvmfs from cclnd docker containers
 if [[ -n "${CCTOOLS_DOCKER_GITHUB}" ]]
 then
-    cvmfs_option='--with-cvmfs-path /usr'
-
     # ensure both python2 and python3 are built for centos7
     python_option='--with-python3-path /usr'
     if [[ "${CCTOOLS_DOCKER_GITHUB}" = centos7 ]]