From f4372b01fcac4df81cf736e5e94cc747fe94bdde Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Mon, 27 Jan 2025 15:22:21 -0500 Subject: [PATCH 1/4] add run_info_dir --- .../python3/ndcctools/taskvine/manager.py | 19 ++++++++++++++++--- taskvine/src/manager/taskvine.h | 5 +++++ taskvine/src/manager/vine_runtime_dir.c | 6 ++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py b/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py index 1aca9c3723..6e2cd722bb 100644 --- a/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py +++ b/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py @@ -65,7 +65,8 @@ class Manager(object): # @param port The port number to listen on. If zero, then a random port is chosen. A range of possible ports (low, hight) can be also specified instead of a single integer. Default is 9123 # @param name The project name to use. # @param shutdown Automatically shutdown workers when manager is finished. Disabled by default. - # @param run_info_path Directory to write log (and staging if staging_path not given) files per run. If None, defaults to "vine-run-info" + # @param run_info_path Directory to archive workflow log directories, it is the upper level directory to run_info_dir. If None, defaults to "vine-run-info" + # @param run_info_dir Directory to write log (and staging if staging_path not given) files per run. If None, defaults by a %Y-%m-%dT%H%M%S format. # @param staging_path Directory to write temporary files. Defaults to run_info_path if not given. # @param ssl A tuple of filenames (ssl_key, ssl_cert) in pem format, or True. # If not given, then TSL is not activated. If True, a self-signed temporary key and cert are generated. @@ -78,6 +79,7 @@ def __init__(self, name=None, shutdown=False, run_info_path="vine-run-info", + run_info_dir=None, staging_path=None, ssl=None, init_fn=None, @@ -113,6 +115,9 @@ def __init__(self, try: if run_info_path: self.set_runtime_info_path(run_info_path) + if run_info_dir: + os.environ["VINE_RUNTIME_INFO_DIR"] = run_info_dir + # self.set_runtime_info_dir(run_info_dir) self._stats = cvine.vine_stats() self._stats_hierarchy = cvine.vine_stats() @@ -559,8 +564,16 @@ def set_property(self, name, value): # # @param self Reference to the current manager object. # @param dirname A directory name - def set_runtime_info_path(self, dirname): - cvine.vine_set_runtime_info_path(dirname) + def set_runtime_info_path(self, path): + cvine.vine_set_runtime_info_path(path) + + ## + # Specify the runtime info directory of this workflow, by default is a %Y-%m-%dT%H%M%S format. + # + # @param self Reference to the current manager object. + # @param dirname A directory name + def set_runtime_info_dir(self, dirname): + cvine.vine_set_runtime_info_dir(dirname) ## # Add a mandatory password that each worker must present. diff --git a/taskvine/src/manager/taskvine.h b/taskvine/src/manager/taskvine.h index d5d10a39d2..b017dafa83 100644 --- a/taskvine/src/manager/taskvine.h +++ b/taskvine/src/manager/taskvine.h @@ -1489,6 +1489,11 @@ void vine_initialize_categories(struct vine_manager *m, struct rmsummary *max, c */ void vine_set_runtime_info_path(const char *path); +/** Sets the directory where a workflow-specific runtime logs are directly written into. +@param dir A directory +*/ +void vine_set_runtime_info_dir(const char *dir); + /** Adds a custom APPLICATION entry to the debug log. @param m Reference to the current manager object. @param entry A custom debug message. diff --git a/taskvine/src/manager/vine_runtime_dir.c b/taskvine/src/manager/vine_runtime_dir.c index 580149af91..c84af08b7a 100644 --- a/taskvine/src/manager/vine_runtime_dir.c +++ b/taskvine/src/manager/vine_runtime_dir.c @@ -163,3 +163,9 @@ void vine_set_runtime_info_path(const char *path) assert(path); vine_runtime_info_path = xxstrdup(path); } + +void vine_set_runtime_info_dir(const char *dir) +{ + assert(dir); + setenv("VINE_RUNTIME_INFO_DIR", dir, 1); +} \ No newline at end of file From 8c84230a92ce43f2494361184a872a0e08e0cf92 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 4 Feb 2025 13:04:40 -0500 Subject: [PATCH 2/4] vine: rename runtime_info_path to runtime_info_template --- .../python3/ndcctools/taskvine/manager.py | 36 +++++++------------ taskvine/src/manager/taskvine.h | 6 +++- taskvine/src/manager/vine_runtime_dir.c | 9 +++-- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py b/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py index 6e2cd722bb..3d861c5051 100644 --- a/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py +++ b/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py @@ -65,8 +65,8 @@ class Manager(object): # @param port The port number to listen on. If zero, then a random port is chosen. A range of possible ports (low, hight) can be also specified instead of a single integer. Default is 9123 # @param name The project name to use. # @param shutdown Automatically shutdown workers when manager is finished. Disabled by default. - # @param run_info_path Directory to archive workflow log directories, it is the upper level directory to run_info_dir. If None, defaults to "vine-run-info" - # @param run_info_dir Directory to write log (and staging if staging_path not given) files per run. If None, defaults by a %Y-%m-%dT%H%M%S format. + # @param run_info_path Directory to archive workflow log directories, it is the upper level directory to run_info_template. If None, defaults to "vine-run-info" + # @param run_info_template Directory to write log (and staging if staging_path not given) files per run. If None, defaults by a %Y-%m-%dT%H%M%S format. # @param staging_path Directory to write temporary files. Defaults to run_info_path if not given. # @param ssl A tuple of filenames (ssl_key, ssl_cert) in pem format, or True. # If not given, then TSL is not activated. If True, a self-signed temporary key and cert are generated. @@ -79,7 +79,7 @@ def __init__(self, name=None, shutdown=False, run_info_path="vine-run-info", - run_info_dir=None, + run_info_template=None, staging_path=None, ssl=None, init_fn=None, @@ -113,11 +113,17 @@ def __init__(self, self._info_widget = JupyterDisplay(interval=status_display_interval) try: + # Set an internal variable in the C code. + # No need to unset it explicitly as it doesn't rely on environment variables. if run_info_path: - self.set_runtime_info_path(run_info_path) - if run_info_dir: - os.environ["VINE_RUNTIME_INFO_DIR"] = run_info_dir - # self.set_runtime_info_dir(run_info_dir) + cvine.vine_set_runtime_info_path(run_info_path) + + # Set the environment variable VINE_RUNTIME_INFO_DIR on the C side. + # If run_info_template is not provided, unset the environment variable to prevent potential issues caused by leftover historical values from previous runs. + if run_info_template: + cvine.vine_set_runtime_info_template(run_info_template) + else: + cvine.vine_unset_runtime_info_template() self._stats = cvine.vine_stats() self._stats_hierarchy = cvine.vine_stats() @@ -559,22 +565,6 @@ def set_catalog_servers(self, catalogs): def set_property(self, name, value): cvine.vine_set_property(self._taskvine, name, value) - ## - # Specify a directory to write logs and staging files. - # - # @param self Reference to the current manager object. - # @param dirname A directory name - def set_runtime_info_path(self, path): - cvine.vine_set_runtime_info_path(path) - - ## - # Specify the runtime info directory of this workflow, by default is a %Y-%m-%dT%H%M%S format. - # - # @param self Reference to the current manager object. - # @param dirname A directory name - def set_runtime_info_dir(self, dirname): - cvine.vine_set_runtime_info_dir(dirname) - ## # Add a mandatory password that each worker must present. # diff --git a/taskvine/src/manager/taskvine.h b/taskvine/src/manager/taskvine.h index b017dafa83..fd0d7e253b 100644 --- a/taskvine/src/manager/taskvine.h +++ b/taskvine/src/manager/taskvine.h @@ -1492,7 +1492,11 @@ void vine_set_runtime_info_path(const char *path); /** Sets the directory where a workflow-specific runtime logs are directly written into. @param dir A directory */ -void vine_set_runtime_info_dir(const char *dir); +void vine_set_runtime_info_template(const char *dir); + +/** Unsets the directory where a workflow-specific runtime logs are directly written into. +*/ +void vine_unset_runtime_info_template(); /** Adds a custom APPLICATION entry to the debug log. @param m Reference to the current manager object. diff --git a/taskvine/src/manager/vine_runtime_dir.c b/taskvine/src/manager/vine_runtime_dir.c index c84af08b7a..82f86ca687 100644 --- a/taskvine/src/manager/vine_runtime_dir.c +++ b/taskvine/src/manager/vine_runtime_dir.c @@ -164,8 +164,13 @@ void vine_set_runtime_info_path(const char *path) vine_runtime_info_path = xxstrdup(path); } -void vine_set_runtime_info_dir(const char *dir) +void vine_set_runtime_info_template(const char *dir) { assert(dir); setenv("VINE_RUNTIME_INFO_DIR", dir, 1); -} \ No newline at end of file +} + +void vine_unset_runtime_info_template() +{ + unsetenv("VINE_RUNTIME_INFO_DIR"); +} From 38427df81816eac7e1cb2ea9ffc62d3f7440d309 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 4 Feb 2025 13:32:09 -0500 Subject: [PATCH 3/4] vine: check if the template has exists --- .../python3/ndcctools/taskvine/manager.py | 2 +- taskvine/src/manager/vine_runtime_dir.c | 26 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py b/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py index 3d861c5051..d2dd6c81c2 100644 --- a/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py +++ b/taskvine/src/bindings/python3/ndcctools/taskvine/manager.py @@ -66,7 +66,7 @@ class Manager(object): # @param name The project name to use. # @param shutdown Automatically shutdown workers when manager is finished. Disabled by default. # @param run_info_path Directory to archive workflow log directories, it is the upper level directory to run_info_template. If None, defaults to "vine-run-info" - # @param run_info_template Directory to write log (and staging if staging_path not given) files per run. If None, defaults by a %Y-%m-%dT%H%M%S format. + # @param run_info_template Directory to write log (and staging if staging_path not given) files per run, append a time-based suffix if the path has already exists. If None, defaults by a %Y-%m-%dT%H%M%S format. # @param staging_path Directory to write temporary files. Defaults to run_info_path if not given. # @param ssl A tuple of filenames (ssl_key, ssl_cert) in pem format, or True. # If not given, then TSL is not activated. If True, a self-signed temporary key and cert are generated. diff --git a/taskvine/src/manager/vine_runtime_dir.c b/taskvine/src/manager/vine_runtime_dir.c index 82f86ca687..b928bfbd9a 100644 --- a/taskvine/src/manager/vine_runtime_dir.c +++ b/taskvine/src/manager/vine_runtime_dir.c @@ -167,7 +167,31 @@ void vine_set_runtime_info_path(const char *path) void vine_set_runtime_info_template(const char *dir) { assert(dir); - setenv("VINE_RUNTIME_INFO_DIR", dir, 1); + + char absolute_template_path[512]; + snprintf(absolute_template_path, sizeof(absolute_template_path), "%s/%s", vine_runtime_info_path, dir); + + // Check if the template path has already exists, if yes, append a suffix with the current time. + struct stat st; + if (stat(absolute_template_path, &st) == 0 && S_ISDIR(st.st_mode)) { + char buf[20]; + time_t now = time(NULL); + struct tm *tm_info = localtime(&now); + strftime(buf, sizeof(buf), "%Y-%m-%dT%H%M%S", tm_info); + + size_t new_dir_len = strlen(dir) + strlen(buf) + 2; + char *new_dir = (char *)malloc(new_dir_len); + if (new_dir == NULL) { + perror("Error: cannot set runtime template path"); + exit(EXIT_FAILURE); + } + snprintf(new_dir, new_dir_len, "%s-%s", dir, buf); + + setenv("VINE_RUNTIME_INFO_DIR", new_dir, 1); + free(new_dir); + } else { + setenv("VINE_RUNTIME_INFO_DIR", dir, 1); + } } void vine_unset_runtime_info_template() From 81820a66bfd0b5f755c5361d74ff953dd41d3935 Mon Sep 17 00:00:00 2001 From: JinZhou5042 Date: Tue, 4 Feb 2025 13:34:04 -0500 Subject: [PATCH 4/4] lint --- taskvine/src/manager/vine_runtime_dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskvine/src/manager/vine_runtime_dir.c b/taskvine/src/manager/vine_runtime_dir.c index b928bfbd9a..dd238ecce8 100644 --- a/taskvine/src/manager/vine_runtime_dir.c +++ b/taskvine/src/manager/vine_runtime_dir.c @@ -171,7 +171,7 @@ void vine_set_runtime_info_template(const char *dir) char absolute_template_path[512]; snprintf(absolute_template_path, sizeof(absolute_template_path), "%s/%s", vine_runtime_info_path, dir); - // Check if the template path has already exists, if yes, append a suffix with the current time. + /* Check if the template path has already exists, if yes, append a suffix with the current time. */ struct stat st; if (stat(absolute_template_path, &st) == 0 && S_ISDIR(st.st_mode)) { char buf[20];