From b9f33eacd44d43d5b6c0a77f58a897ebcf57d3a0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 May 2021 12:02:38 +0200 Subject: [PATCH 01/45] Allow tweaking of ECs from different toolchains This is useful to bulk-install e.g. Python with enable_lto=False set for all ECs. The restriction is only required when changing the toolchain so move the check down and add a simple test. --- easybuild/framework/easyconfig/tweak.py | 18 ++++++----- test/framework/easyconfig.py | 40 ++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 41c688355b..c3e26ab868 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -91,15 +91,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): tweaked_ecs_path, tweaked_ecs_deps_path = None, None if targetdirs is not None: tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs - # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) - toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) - if len(toolchains) > 1: - raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s", - toolchains) - # Toolchain is unique, let's store it - source_toolchain = easyconfigs[-1]['ec']['toolchain'] modifying_toolchains_or_deps = False - target_toolchain = {} src_to_dst_tc_mapping = {} revert_to_regex = False @@ -117,6 +109,16 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): revert_to_regex = True if not revert_to_regex: + # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) + toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) + if len(toolchains) > 1: + raise EasyBuildError("Multiple toolchains featured in easyconfigs, " + "--try-X not supported in that case: %s", + toolchains) + # Toolchain is unique, let's store it + source_toolchain = easyconfigs[-1]['ec']['toolchain'] + target_toolchain = {} + # we're doing something that involves the toolchain hierarchy; # obtain full dependency graph for specified easyconfigs; # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f09a0896a6..dfcd956f02 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -57,10 +57,10 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str from easybuild.framework.easyconfig.style import check_easyconfigs_style -from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph -from easybuild.framework.easyconfig.tools import det_copy_ec_specs, find_related_easyconfigs, get_paths_for +from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, check_sha256_checksums +from easybuild.framework.easyconfig.tools import dep_graph, det_copy_ec_specs, find_related_easyconfigs, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs -from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one +from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak, tweak_one from easybuild.framework.extension import resolve_exts_filter_template from easybuild.toolchains.system import SystemToolchain from easybuild.tools.build_log import EasyBuildError @@ -73,7 +73,7 @@ from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata from easybuild.tools.py2vs3 import OrderedDict, reload -from easybuild.tools.robot import resolve_dependencies +from easybuild.tools.robot import det_robot_path, resolve_dependencies from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version @@ -728,6 +728,38 @@ def test_tweaking(self): # cleanup os.remove(tweaked_fn) + def test_tweak_multiple_tcs(self): + """Test that tweaking variables of ECs from multiple toolchains works""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + + # Create directories to store the tweaked easyconfigs + tweaked_ecs_paths, pr_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_path, auto_robot=True) + + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': robot_path, + 'check_osdeps': False, + }) + + # Allow tweaking of non-toolchain values for multiple ECs of different toolchains + untweaked_openmpi_1 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') + untweaked_openmpi_2 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb') + easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi_1, False), (untweaked_openmpi_2, False)]) + tweak_specs = {'moduleclass': 'debugger'} + easyconfigs = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths) + # Check that all expected tweaked easyconfigs exists + tweaked_openmpi_1 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_1)) + tweaked_openmpi_2 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_2)) + self.assertTrue(os.path.isfile(tweaked_openmpi_1)) + self.assertTrue(os.path.isfile(tweaked_openmpi_2)) + tweaked_openmpi_content_1 = read_file(tweaked_openmpi_1) + tweaked_openmpi_content_2 = read_file(tweaked_openmpi_2) + self.assertTrue('moduleclass = "debugger"' in tweaked_openmpi_content_1, + "Tweaked value not found in " + tweaked_openmpi_content_1) + self.assertTrue('moduleclass = "debugger"' in tweaked_openmpi_content_2, + "Tweaked value not found in " + tweaked_openmpi_content_2) + def test_installversion(self): """Test generation of install version.""" From 0915a995d93dacc5f4478dfcd7d11bdc36331815 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 12:25:16 +0200 Subject: [PATCH 02/45] Deduplicate warnings&errors found in logs and add initial newline in output It isn't useful to report duplicate errors multiple times. And e.g. configure may report a wrong option at the start and end. Hence deduplicate it. Also change the logged warning/error to include a newline & tab in front of each element which makes it nicer to read. E.g.: > build failed (first 300 chars): Found 1 error(s) in command output: > configure: WARNING: unrecognized options: --with-foo, --with-bar (took 2 mins 39 secs) --- easybuild/tools/run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 8916d80795..19a66db1ca 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -52,7 +52,7 @@ from easybuild.tools.config import ERROR, IGNORE, WARN, build_option from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook from easybuild.tools.py2vs3 import string_type -from easybuild.tools.utilities import trace_msg +from easybuild.tools.utilities import nub, trace_msg _log = fancylogger.getLogger('run', fname=False) @@ -790,7 +790,7 @@ def extract_errors_from_log(log_txt, reg_exps): elif action == WARN: warnings.append(line) break - return warnings, errors + return nub(warnings), nub(errors) def check_log_for_errors(log_txt, reg_exps): @@ -805,8 +805,8 @@ def check_log_for_errors(log_txt, reg_exps): errors_found_in_log += len(warnings) + len(errors) if warnings: - _log.warning("Found %s potential error(s) in command output (output: %s)", + _log.warning("Found %s potential error(s) in command output:\n\t%s", len(warnings), "\n\t".join(warnings)) if errors: - raise EasyBuildError("Found %s error(s) in command output (output: %s)", + raise EasyBuildError("Found %s error(s) in command output:\n\t%s", len(errors), "\n\t".join(errors)) From 96c7a04bec604ff69608b4c52cceadc1d3734c5e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Oct 2023 13:01:59 +0200 Subject: [PATCH 03/45] Adapt test to new output --- test/framework/run.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 1a93ff1a75..d529b41a3a 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -705,8 +705,9 @@ def test_check_log_for_errors(self): "enabling -Werror", "the process crashed with 0" ]) - expected_msg = r"Found 2 error\(s\) in command output "\ - r"\(output: error found\n\tthe process crashed with 0\)" + expected_msg = r"Found 2 error\(s\) in command output:\n"\ + r"\terror found\n"\ + r"\tthe process crashed with 0" # String promoted to list self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, @@ -718,14 +719,17 @@ def test_check_log_for_errors(self): self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [(r"\b(error|crashed)\b", ERROR)]) - expected_msg = "Found 2 potential error(s) in command output " \ - "(output: error found\n\tthe process crashed with 0)" + expected_msg = "Found 2 potential error(s) in command output:\n"\ + "\terror found\n"\ + "\tthe process crashed with 0" init_logging(logfile, silent=True) check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)]) stop_logging(logfile) self.assertIn(expected_msg, read_file(logfile)) - expected_msg = r"Found 2 error\(s\) in command output \(output: error found\n\ttest failed\)" + expected_msg = r"Found 2 error\(s\) in command output:\n"\ + r"\terror found\n"\ + r"\ttest failed" write_file(logfile, '') init_logging(logfile, silent=True) self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [ @@ -735,7 +739,7 @@ def test_check_log_for_errors(self): "fail" ]) stop_logging(logfile) - expected_msg = "Found 1 potential error(s) in command output (output: the process crashed with 0)" + expected_msg = "Found 1 potential error(s) in command output:\n\tthe process crashed with 0" self.assertIn(expected_msg, read_file(logfile)) def test_run_cmd_with_hooks(self): From f4dda1ea638f11f16652ab00ca88b72b96ea7762 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 11:16:30 +0100 Subject: [PATCH 04/45] Fix test_load test for EnvironmentModules 4.2+ test_load expects that version 4.2+ of Environment Modules reload an already loaded module. As of version 5.3, already loaded modules are not reloaded unless one of their dependent module is unloaded or loaded. --- test/framework/modules.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 7bf87130bd..738cc79595 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -437,9 +437,8 @@ def test_load(self): # if GCC is loaded again, $EBROOTGCC should be set again, and GCC should be listed last self.modtool.load(['GCC/6.4.0-2.28']) - # environment modules v4.0 does not reload already loaded modules, will be changed in v4.2 - modtool_ver = StrictVersion(self.modtool.version) - if not isinstance(self.modtool, EnvironmentModules) or modtool_ver >= StrictVersion('4.2'): + # environment modules v4+ does not reload already loaded modules + if not isinstance(self.modtool, EnvironmentModules): self.assertTrue(os.environ.get('EBROOTGCC')) if isinstance(self.modtool, Lmod): From 94ff50f8df8c0dfa05c6bec94bf32900112b8100 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 11:26:15 +0100 Subject: [PATCH 05/45] Ensure correct configuration for EnvironmentModules Defines environment variables when initializing EnvironmentModules object to ensure that this module tool will not be influenced by external configuration. Setup Environment Modules configuration to ensure module search behaves like EasyBuild expects (match module name start, case sensitive, return in-depth modulepath content, ignore cache file). Also defines a basic output configuration not to get unexpected content like tags or variants. This change helps to pass "test_avail" test with Environment Modules v5.0+. --- easybuild/tools/modules.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..aa739f7563 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1328,6 +1328,29 @@ class EnvironmentModules(EnvironmentModulesTcl): MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + def __init__(self, *args, **kwargs): + """Constructor, set Environment Modules-specific class variable values.""" + # ensure in-depth modulepath search (MODULES_AVAIL_INDEPTH has been introduced in v4.3) + setvar('MODULES_AVAIL_INDEPTH', '1', verbose=False) + # match against module name start (MODULES_SEARCH_MATCH has been introduced in v4.3) + setvar('MODULES_SEARCH_MATCH', 'starts_with', verbose=False) + # ensure no debug message (MODULES_VERBOSITY has been introduced in v4.3) + setvar('MODULES_VERBOSITY', 'normal', verbose=False) + # make module search case sensitive (search is case insensitive by default since v5.0) + setvar('MODULES_ICASE', 'never', verbose=False) + # disable extended default (introduced in v4.4 and enabled by default in v5.0) + setvar('MODULES_EXTENDED_DEFAULT', '0', verbose=False) + # hard disable output redirection, output messages are expected on stderr + setvar('MODULES_REDIRECT_OUTPUT', '0', verbose=False) + # make sure modulefile cache is ignored (cache mechanism supported since v5.3) + setvar('MODULES_IGNORE_CACHE', '1', verbose=False) + # ensure only module names are returned on avail (MODULES_AVAIL_TERSE_OUTPUT added in v4.7) + setvar('MODULES_AVAIL_TERSE_OUTPUT', '', verbose=False) + # ensure only module names are returned on list (MODULES_LIST_TERSE_OUTPUT added in v4.7) + setvar('MODULES_LIST_TERSE_OUTPUT', '', verbose=False) + + super(EnvironmentModules, self).__init__(*args, **kwargs) + def check_module_output(self, cmd, stdout, stderr): """Check output of 'module' command, see if if is potentially invalid.""" if "_mlstatus = False" in stdout: From f297d321b276730a6a5b806e60e266342012b999 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sun, 29 Oct 2023 11:35:32 +0100 Subject: [PATCH 06/45] Setenv value may be enclosed in curly braces on EnvironmentModules Starting Environment Modules 4.2, value of setenv statement in "module show" output is enclosed in curly braces if it contains spaces. This change helps to pass "test_get_setenv_value_from_modulefile" test with Environment Modules v4.2+ --- easybuild/tools/modules.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index aa739f7563..e0eda91aeb 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1358,6 +1358,27 @@ def check_module_output(self, cmd, stdout, stderr): else: self.log.debug("No errors detected when running module command '%s'", cmd) + def get_setenv_value_from_modulefile(self, mod_name, var_name): + """ + Get value for specific 'setenv' statement from module file for the specified module. + + :param mod_name: module name + :param var_name: name of the variable being set for which value should be returned + """ + # Tcl-based module tools produce "module show" output with setenv statements like: + # "setenv GCC_PATH /opt/gcc/8.3.0" + # "setenv VAR {some text} + # - line starts with 'setenv' + # - whitespace (spaces & tabs) around variable name + # - curly braces around value if it contain spaces + value = super(EnvironmentModules, self).get_setenv_value_from_modulefile(mod_name=mod_name, + var_name=var_name) + + if value: + value = value.strip('{}') + + return value + class Lmod(ModulesTool): """Interface to Lmod.""" From e9f190c7802d340f09b177743f77e1a022a964f7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Nov 2023 18:05:16 +0100 Subject: [PATCH 07/45] only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 --- .github/workflows/unit_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 5b09fc654b..ec98f9df03 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -103,9 +103,9 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | - # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit; + # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit; # tests that require a GitHub token are skipped automatically when no GitHub token is available - if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then + if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then if [ ! -z $GITHUB_TOKEN ]; then SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; From 13da6438282a5cc052724feb230e62285687b851 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Nov 2023 21:07:02 +0100 Subject: [PATCH 08/45] bump version to 4.9.0dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 8582267629..655f7d0a49 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.2') +VERSION = LooseVersion('4.9.0.dev0') UNKNOWN = 'UNKNOWN' From ee99d87e910a210638eb13e841c2920f152459c0 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 8 Nov 2023 16:51:16 +0000 Subject: [PATCH 09/45] Use -qopenmp instead of -fiopenmp for OpenMP in OneAPI Recommended in porting guide: qopenmp, unlike fiopenmp, works for both classic and oneapi compilers https://www.intel.com/content/www/us/en/developer/articles/guide/porting-guide-for-ifort-to-ifx.html Fixes #4376 --- easybuild/toolchains/compiler/intel_compilers.py | 5 +++-- test/framework/toolchain.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index ef537d9315..ae97dfa87d 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -109,8 +109,9 @@ def set_variables(self): self.options.options_map['loose'] = ['fp-model fast'] # fp-model fast=2 gives "warning: overriding '-ffp-model=fast=2' option with '-ffp-model=fast'" self.options.options_map['veryloose'] = ['fp-model fast'] - # recommended in porting guide - self.options.options_map['openmp'] = ['fiopenmp'] + # recommended in porting guide: qopenmp, unlike fiopenmp, works for both classic and oneapi compilers + # https://www.intel.com/content/www/us/en/developer/articles/guide/porting-guide-for-ifort-to-ifx.html + self.options.options_map['openmp'] = ['qopenmp'] # -xSSE2 is not supported by Intel oneAPI compilers, # so use -march=x86-64 -mtune=generic when using optarch=GENERIC diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index d80d41c788..f6243bd3b3 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -2036,7 +2036,7 @@ def test_independence(self): 'CrayIntel': "-O2 -ftz -fp-speculation=safe -fp-model source -fopenmp -craype-verbose", 'GCC': "-O2 -ftree-vectorize -test -fno-math-errno -fopenmp", 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source -fopenmp", - 'intel-compilers': "-O2 -test -ftz -fp-speculation=safe -fp-model precise -fiopenmp", + 'intel-compilers': "-O2 -test -ftz -fp-speculation=safe -fp-model precise -qopenmp", } toolchains = [ From 757b8ee60ebbeaf50e868f430a2f0f399bcadcd7 Mon Sep 17 00:00:00 2001 From: Xavier Delaruelle Date: Sat, 28 Oct 2023 20:55:49 +0200 Subject: [PATCH 10/45] Adapt module function check for EnvironmentModules Default check_module_function tests that module command is called from module shell function. With EnvironmentModules v4+, module command is usually called from the _module_raw shell function. This commit adds a specific version of check_module_function for EnvironmentModules class. Module command is first checked within _module_raw shell function definition. If not found, default test (that checks module function) is run. Add new unit test "test_environment_modules_specific" to specifically check module command definition with EnvironmentModules. Fixes #4368 --- easybuild/tools/modules.py | 25 +++++++++++++++++++++++++ easybuild/tools/run.py | 1 + test/framework/modulestool.py | 19 ++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..c5789f40eb 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1328,6 +1328,31 @@ class EnvironmentModules(EnvironmentModulesTcl): MAX_VERSION = None VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + def check_module_function(self, allow_mismatch=False, regex=None): + """Check whether selected module tool matches 'module' function definition.""" + # Modules 5.1.0+: module command is called from _module_raw shell function + # Modules 4.2.0..5.0.1: module command is called from _module_raw shell function if it has + # been initialized in an interactive shell session (i.e., a session attached to a tty) + if self.testing: + if '_module_raw' in os.environ: + out, ec = os.environ['_module_raw'], 0 + else: + out, ec = None, 1 + else: + cmd = "type _module_raw" + out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True, trace=False) + + if regex is None: + regex = r".*%s" % os.path.basename(self.cmd) + mod_cmd_re = re.compile(regex, re.M) + + if ec == 0 and mod_cmd_re.search(out): + self.log.debug("Found pattern '%s' in defined '_module_raw' function." % mod_cmd_re.pattern) + else: + self.log.debug("Pattern '%s' not found in '_module_raw' function, falling back to 'module' function", + mod_cmd_re.pattern) + super(EnvironmentModules, self).check_module_function(allow_mismatch, regex) + def check_module_output(self, cmd, stdout, stderr): """Check output of 'module' command, see if if is potentially invalid.""" if "_mlstatus = False" in stdout: diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 8916d80795..08b109ec15 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -71,6 +71,7 @@ "sysctl -n machdep.cpu.brand_string", # used in get_cpu_model (OS X) "sysctl -n machdep.cpu.vendor", # used in get_cpu_vendor (OS X) "type module", # used in ModulesTool.check_module_function + "type _module_raw", # used in EnvironmentModules.check_module_function "ulimit -u", # used in det_parallelism ] diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 59c6872b93..fb991ad797 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -39,7 +39,7 @@ from easybuild.tools import modules, StrictVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which, write_file -from easybuild.tools.modules import Lmod +from easybuild.tools.modules import EnvironmentModules, Lmod from test.framework.utilities import init_config @@ -192,6 +192,23 @@ def test_lmod_specific(self): # test updating local spider cache (but don't actually update the local cache file!) self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty") + def test_environment_modules_specific(self): + """Environment Modules-specific test (skipped unless installed).""" + modulecmd_abspath = which(EnvironmentModules.COMMAND) + # only run this test if 'modulecmd.tcl' is installed + if modulecmd_abspath is not None: + # redefine 'module' and '_module_raw' function (deliberate mismatch with used module + # command in EnvironmentModules) + os.environ['_module_raw'] = "() { eval `/usr/share/Modules/libexec/foo.tcl' bash $*`;\n}" + os.environ['module'] = "() { _module_raw \"$@\" 2>&1;\n}" + error_regex = ".*pattern .* not found in defined 'module' function" + self.assertErrorRegex(EasyBuildError, error_regex, EnvironmentModules, testing=True) + + # redefine '_module_raw' function with correct module command + os.environ['_module_raw'] = "() { eval `/usr/share/Modules/libexec/modulecmd.tcl' bash $*`;\n}" + mt = EnvironmentModules(testing=True) + self.assertIsInstance(mt.loaded_modules(), list) # dummy usage + def tearDown(self): """Testcase cleanup.""" super(ModulesToolTest, self).tearDown() From 0288ca35420fe54a298e547e17f1d057c96b2099 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 10 Nov 2023 13:56:14 +0100 Subject: [PATCH 11/45] fix LIBBLAS_MT for flexiblas, ensure -lpthread is included --- easybuild/toolchains/linalg/flexiblas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index 43b450b3de..c266aff248 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -70,6 +70,7 @@ class FlexiBLAS(LinAlg): """ BLAS_MODULE_NAME = ['FlexiBLAS'] BLAS_LIB = ['flexiblas'] + BLAS_LIB_MT = ['flexiblas'] BLAS_INCLUDE_DIR = [os.path.join('include', 'flexiblas')] BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS From 2ce5868740716f27bbd092bbeeaef1b67475cca4 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 21 Nov 2023 18:32:28 +0800 Subject: [PATCH 12/45] relax major version match regex in find_related_easyconfigs --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index abe8f998ea..c6649c383b 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -472,7 +472,7 @@ def find_related_easyconfigs(path, ec): if len(parsed_version) >= 2: version_patterns.append(r'%s\.%s\.\w+' % tuple(parsed_version[:2])) # major/minor version match if parsed_version != parsed_version[0]: - version_patterns.append(r'%s\.[\d-]+\.\w+' % parsed_version[0]) # major version match + version_patterns.append(r'%s\.[\d-]+(\.\w+)*' % parsed_version[0]) # major version match version_patterns.append(r'[\w.]+') # any version regexes = [] From 95e8c480b23f8e653957d0d8dc646fb877ebb1b7 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 21 Nov 2023 19:52:49 +0000 Subject: [PATCH 13/45] When combining multi_deps with iterative builds, it generates a list with duplicates. This commit fixes that --- easybuild/tools/module_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index aaf97d3194..0da350c1bd 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -666,7 +666,7 @@ def _generate_help_text(self): if multi_deps: compatible_modules_txt = '\n'.join([ "This module is compatible with the following modules, one of each line is required:", - ] + ['* %s' % d for d in multi_deps]) + ] + ['* %s' % d for d in set(multi_deps)]) lines.extend(self._generate_section("Compatible modules", compatible_modules_txt)) # Extensions (if any) From 3714b8cf30f2cb86af9e0d1b292cac97997a3769 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 22 Nov 2023 15:15:17 +0000 Subject: [PATCH 14/45] Use nub instead of set to preserve order --- easybuild/tools/module_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index 0da350c1bd..c65879009e 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -49,7 +49,7 @@ from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool from easybuild.tools.py2vs3 import string_type -from easybuild.tools.utilities import get_subclasses, quote_str +from easybuild.tools.utilities import get_subclasses, nub, quote_str _log = fancylogger.getLogger('module_generator', fname=False) @@ -666,7 +666,7 @@ def _generate_help_text(self): if multi_deps: compatible_modules_txt = '\n'.join([ "This module is compatible with the following modules, one of each line is required:", - ] + ['* %s' % d for d in set(multi_deps)]) + ] + ['* %s' % d for d in nub(multi_deps)]) lines.extend(self._generate_section("Compatible modules", compatible_modules_txt)) # Extensions (if any) From 871f955ca0317a3d5d1bcd2f5955b02d445ac62d Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:01:02 +0100 Subject: [PATCH 15/45] Add formating as json --- easybuild/tools/docs.py | 30 ++++++++++++++++++++++++++++++ easybuild/tools/options.py | 4 ++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4f4bfca99c..84666a6e10 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -39,6 +39,7 @@ import copy import inspect import os +import json from easybuild.tools import LooseVersion from easybuild.base import fancylogger @@ -75,6 +76,7 @@ FORMAT_MD = 'md' FORMAT_RST = 'rst' FORMAT_TXT = 'txt' +FORMAT_JSON = 'json' def generate_doc(name, params): @@ -1024,6 +1026,34 @@ def list_software_txt(software, detailed=False): return '\n'.join(lines) +def list_software_json(software, detailed=False): + """ + Return overview of supported software in json + + :param software: software information (strucuted like list_software does) + :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info) + :return: multi-line string presenting requested info + """ + lines = ['['] + for key in sorted(software, key=lambda x: x.lower()): + for tmp in software[key]: + if detailed: + # deep copy here to avoid modifying the original dict + x = copy.deepcopy(tmp) + x['description'] = x['description'].split('\n')[0].strip() + else: + x = {} + x['name'] = key + + lines.append(json.dumps(x, indent=4) + ",") + if detailed: + break + # remove last comma + if len(lines) > 1: + lines[-1] = lines[-1][:-1] + return '\n'.join(lines) + '\n]' + + def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e69d96b2f6..2a7d990e01 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.configobj import ConfigObj, ConfigObjError -from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT +from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains @@ -469,7 +469,7 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT]), + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), From c3bde441c3710f3dbcd8ab873385795d3ff3179e Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:26:14 +0100 Subject: [PATCH 16/45] Add json list software test --- easybuild/tools/docs.py | 2 +- easybuild/tools/options.py | 3 +- test/framework/docs.py | 88 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 84666a6e10..74684842dd 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1040,7 +1040,7 @@ def list_software_json(software, detailed=False): if detailed: # deep copy here to avoid modifying the original dict x = copy.deepcopy(tmp) - x['description'] = x['description'].split('\n')[0].strip() + x['description'] = ' '.join(x['description'].split('\n')).strip() else: x = {} x['name'] = key diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 2a7d990e01..a1f9d756eb 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -469,7 +469,8 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, + [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), diff --git a/test/framework/docs.py b/test/framework/docs.py index 84d862bd3c..829369934a 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -405,6 +405,91 @@ ``1.4``|``GCC/4.6.3``, ``system`` ``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +LIST_SOFTWARE_SIMPLE_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +[g](#g) + + +## G + +* GCC +* gzip""" + + + +LIST_SOFTWARE_DETAILED_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +[g](#g) + + +## G + + +[GCC](#gcc) - [gzip](#gzip) + + +### GCC + +%(gcc_descr)s + +*homepage*: + +version |toolchain +---------|---------- +``4.6.3``|``system`` + +### gzip + +%(gzip_descr)s + +*homepage*: + +version|toolchain +-------|------------------------------- +``1.4``|``GCC/4.6.3``, ``system`` +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + +LIST_SOFTWARE_SIMPLE_JSON = """[ +{ + "name": "GCC" +}, +{ + "name": "gzip" +}, +{ + "name": "gzip" +}, +{ + "name": "gzip" +}, +{ + "name": "gzip" +} +]""" + +LIST_SOFTWARE_DETAILED_JSON = """[ +{ + "toolchain": "system", + "description": "%(gcc_descr)s", + "homepage": "http://gcc.gnu.org/", + "version": "4.6.3", + "versionsuffix": "", + "name": "GCC" +}, +{ + "toolchain": "GCC/4.6.3", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", + "name": "gzip" +} +]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + class DocsTest(EnhancedTestCase): @@ -587,6 +672,9 @@ def test_list_software(self): self.assertEqual(list_software(output_format='md'), LIST_SOFTWARE_SIMPLE_MD) self.assertEqual(list_software(output_format='md', detailed=True), LIST_SOFTWARE_DETAILED_MD) + self.assertEqual(list_software(output_format='json'), LIST_SOFTWARE_SIMPLE_JSON) + self.assertEqual(list_software(output_format='json', detailed=True), LIST_SOFTWARE_DETAILED_JSON) + # GCC/4.6.3 is installed, no gzip module installed txt = list_software(output_format='txt', detailed=True, only_installed=True) self.assertTrue(re.search(r'^\* GCC', txt, re.M)) From 1cc58f9971498c4b175b34937dc061568cea9900 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:27:50 +0100 Subject: [PATCH 17/45] Fix format... --- easybuild/tools/options.py | 2 +- test/framework/docs.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a1f9d756eb..64c46b61b5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -469,7 +469,7 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", diff --git a/test/framework/docs.py b/test/framework/docs.py index 829369934a..5894796b05 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -417,8 +417,6 @@ * GCC * gzip""" - - LIST_SOFTWARE_DETAILED_MD = """# List of supported software EasyBuild supports 2 different software packages (incl. toolchains, bundles): From 9104b85d9efa6e1c10fbc32d52872f4a2617a770 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:35:50 +0100 Subject: [PATCH 18/45] Fix verbosity --- easybuild/tools/docs.py | 4 ++-- test/framework/docs.py | 33 ++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 74684842dd..40f53f2ef1 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1046,9 +1046,9 @@ def list_software_json(software, detailed=False): x['name'] = key lines.append(json.dumps(x, indent=4) + ",") - if detailed: + if not detailed: break - # remove last comma + # remove last line last comma if len(lines) > 1: lines[-1] = lines[-1][:-1] return '\n'.join(lines) + '\n]' diff --git a/test/framework/docs.py b/test/framework/docs.py index 5894796b05..b52e7e09ee 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -455,15 +455,6 @@ { "name": "GCC" }, -{ - "name": "gzip" -}, -{ - "name": "gzip" -}, -{ - "name": "gzip" -}, { "name": "gzip" } @@ -485,6 +476,30 @@ "version": "1.4", "versionsuffix": "", "name": "gzip" +}, +{ + "toolchain": "system", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", + "name": "gzip" +}, +{ + "toolchain": "foss/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", + "name": "gzip" +}, +{ + "toolchain": "intel/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", + "name": "gzip" } ]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} From 3538e93403c1adf70d9ab12f95537deec634e183 Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 22:59:42 +0100 Subject: [PATCH 19/45] Add not implemented exceptions for other json cases --- easybuild/tools/docs.py | 43 +++++++++++++++++++++++++++++++++++++++++ test/framework/docs.py | 28 +++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 40f53f2ef1..0c710be74e 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -117,6 +117,11 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT): return generate_doc('avail_cfgfile_constants_%s' % output_format, [go_cfg_constants]) +def avail_cfgfile_constants_json(go_cfg_constants): + """Generate documentation on constants for configuration files in json format""" + raise NotImplementedError("JSON output format not supported for avail_cfgfile_constants_json") + + def avail_cfgfile_constants_txt(go_cfg_constants): """Generate documentation on constants for configuration files in txt format""" doc = [ @@ -186,6 +191,11 @@ def avail_easyconfig_constants(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_constants_%s' % output_format, []) +def avail_easyconfig_constants_json(): + """Generate easyconfig constant documentation in json format""" + raise NotImplementedError("JSON output format not supported for avail_easyconfig_constants_json") + + def avail_easyconfig_constants_txt(): """Generate easyconfig constant documentation in txt format""" doc = ["Constants that can be used in easyconfigs"] @@ -244,6 +254,11 @@ def avail_easyconfig_licenses(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_licenses_%s' % output_format, []) +def avail_easyconfig_licenses_json(): + """Generate easyconfig license documentation in json format""" + raise NotImplementedError("JSON output format not supported for avail_easyconfig_licenses_json") + + def avail_easyconfig_licenses_txt(): """Generate easyconfig license documentation in txt format""" doc = ["License constants that can be used in easyconfigs"] @@ -356,6 +371,13 @@ def avail_easyconfig_params_rst(title, grouped_params): return '\n'.join(doc) +def avail_easyconfig_params_json(): + """ + Compose overview of available easyconfig parameters, in json format. + """ + raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json") + + def avail_easyconfig_params_txt(title, grouped_params): """ Compose overview of available easyconfig parameters, in plain text format. @@ -428,6 +450,11 @@ def avail_easyconfig_templates(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_templates_%s' % output_format, []) +def avail_easyconfig_templates_json(): + """ Returns template documentation in json text format """ + raise NotImplementedError("JSON output format not supported for avail_easyconfig_templates") + + def avail_easyconfig_templates_txt(): """ Returns template documentation in plain text format """ # This has to reflect the methods/steps used in easyconfig _generate_template_values @@ -642,6 +669,8 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT): + if output_format == FORMAT_JSON: + raise NotImplementedError("JSON output format not supported for list_easyblocks") format_strings = { FORMAT_MD: { 'det_root_templ': "- **%s** (%s%s)", @@ -1202,6 +1231,10 @@ def list_toolchains_txt(tcs): return '\n'.join(doc) +def list_toolchains_json(tcs): + """ Returns overview of all toolchains in json format """ + raise NotImplementedError("JSON output not implemented yet for --list-toolchains") + def avail_toolchain_opts(name, output_format=FORMAT_TXT): """Show list of known options for given toolchain.""" @@ -1256,6 +1289,11 @@ def avail_toolchain_opts_rst(name, tc_dict): return '\n'.join(doc) +def avail_toolchain_opts_json(name, tc_dict): + """ Returns overview of toolchain options in jsonformat """ + raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts") + + def avail_toolchain_opts_txt(name, tc_dict): """ Returns overview of toolchain options in txt format """ doc = ["Available options for %s toolchain:" % name] @@ -1281,6 +1319,11 @@ def get_easyblock_classes(package_name): return easyblocks +def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None): + """ + Compose overview of all easyblocks in the given package in json format + """ + raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview") def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None): """ diff --git a/test/framework/docs.py b/test/framework/docs.py index b52e7e09ee..a271d63cfc 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -639,6 +639,9 @@ def test_license_docs(self): regex = re.compile(r"^``GPLv3``\s*|The GNU General Public License", re.M) self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs)) + # expect NotImplementedError for JSON output + self.assertRaises(NotImplementedError, avail_easyconfig_licenses, output_format='json') + def test_list_easyblocks(self): """ Tests for list_easyblocks function @@ -667,6 +670,9 @@ def test_list_easyblocks(self): txt = list_easyblocks(list_easyblocks='detailed', output_format='md') self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_MD % {'topdir': topdir_easyblocks}) + # expect NotImplementedError for JSON output + self.assertRaises(NotImplementedError, list_easyblocks, output_format='json') + def test_list_software(self): """Test list_software* functions.""" build_options = { @@ -791,6 +797,10 @@ def test_list_toolchains(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + list_toolchains(output_format='json') + def test_avail_cfgfile_constants(self): """ Test avail_cfgfile_constants to generate overview of constants that can be used in a configuration file. @@ -835,6 +845,10 @@ def test_avail_cfgfile_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='json') + def test_avail_easyconfig_constants(self): """ Test avail_easyconfig_constants to generate overview of constants that can be used in easyconfig files. @@ -878,6 +892,10 @@ def test_avail_easyconfig_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_easyconfig_constants(output_format='json') + def test_avail_easyconfig_templates(self): """ Test avail_easyconfig_templates to generate overview of templates that can be used in easyconfig files. @@ -928,6 +946,10 @@ def test_avail_easyconfig_templates(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_easyconfig_templates(output_format='json') + def test_avail_toolchain_opts(self): """ Test avail_toolchain_opts to generate overview of supported toolchain options. @@ -1012,6 +1034,12 @@ def test_avail_toolchain_opts(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_toolchain_opts('foss', output_format='json') + with self.assertRaises(NotImplementedError): + avail_toolchain_opts('intel', output_format='json') + def test_mk_table(self): """ Tests for mk_*_table functions. From 5967a66620ab57d3c212dc1e80f99d3e60aa280a Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Sun, 26 Nov 2023 23:01:35 +0100 Subject: [PATCH 20/45] Fix format again --- easybuild/tools/docs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 0c710be74e..344123d0f7 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1231,6 +1231,7 @@ def list_toolchains_txt(tcs): return '\n'.join(doc) + def list_toolchains_json(tcs): """ Returns overview of all toolchains in json format """ raise NotImplementedError("JSON output not implemented yet for --list-toolchains") @@ -1319,12 +1320,14 @@ def get_easyblock_classes(package_name): return easyblocks + def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None): """ Compose overview of all easyblocks in the given package in json format """ raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview") + def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None): """ Compose overview of all easyblocks in the given package in MarkDown format From e97eb9b0e0198dca70a6ac89ad217c153706a0d6 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 27 Nov 2023 17:37:22 +0100 Subject: [PATCH 21/45] extend test to recent toolchain --- test/framework/toolchain.py | 79 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index f6243bd3b3..8809278193 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1840,45 +1840,46 @@ def test_old_new_iccifort(self): scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) - tc = self.get_toolchain('fosscuda', version='2018a') - tc.prepare() - self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) - self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) - self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) - - self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) - self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) - - self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) - - self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) - self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) - self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) - self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) - self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) - self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) - self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) - - self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) - self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) - self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) - self.modtool.purge() + for tc in [('fosscuda', '2018a'), ('foss', '2023a')]: + tc = self.get_toolchain(tc[0], version=tc[1]) + tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) + self.modtool.purge() tc = self.get_toolchain('intel', version='2018a') tc.prepare() From e11b9bd7ac62ebd3508dd6240e3f06b68f1a9c62 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 27 Nov 2023 17:47:53 +0100 Subject: [PATCH 22/45] fix --- test/framework/toolchain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 8809278193..bee0eec8a9 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1840,8 +1840,8 @@ def test_old_new_iccifort(self): scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) - for tc in [('fosscuda', '2018a'), ('foss', '2023a')]: - tc = self.get_toolchain(tc[0], version=tc[1]) + for toolc in [('fosscuda', '2018a'), ('foss', '2023a')]: + tc = self.get_toolchain(toolc[0], version=toolc[1]) tc.prepare() self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) From 7e6993fafdab1c78391bc8c494018cf92848cbad Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 27 Nov 2023 18:40:46 +0100 Subject: [PATCH 23/45] add foss/2023a module for testing --- test/framework/modules/foss/2023a | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 test/framework/modules/foss/2023a diff --git a/test/framework/modules/foss/2023a b/test/framework/modules/foss/2023a new file mode 100644 index 0000000000..05540a45f5 --- /dev/null +++ b/test/framework/modules/foss/2023a @@ -0,0 +1,45 @@ +#%Module + +proc ModulesHelp { } { + puts stderr { GCC based compiler toolchain including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) +} +} + +module-whatis {GCC based compiler toolchain including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none)} + +set root /prefix/software/foss/2023a + +conflict foss + +if { ![is-loaded GCC/12.3.0] } { + module load GCC/12.3.0 +} + +if { ![is-loaded OpenMPI/4.1.5-GCC-12.3.0] } { + module load OpenMPI/4.1.5-GCC-12.3.0 +} + +if { ![is-loaded FlexiBLAS/3.3.1-GCC-12.3.0] } { + module load FlexiBLAS/3.3.1-GCC-12.3.0 +} + +if { ![is-loaded FFTW/3.3.10-GCC-12.3.0] } { + module load FFTW/3.3.10-GCC-12.3.0 +} + +if { ![is-loaded FFTW.MPI/3.3.10-gompi-2023a] } { + module load FFTW.MPI/3.3.10-gompi-2023a +} + +if { ![is-loaded ScaLAPACK/2.2.0-gompi-2023a-fb] } { + module load ScaLAPACK/2.2.0-gompi-2023a-fb +} + +setenv EBROOTFOSS "$root" +setenv EBVERSIONFOSS "2023a" +setenv EBDEVELFOSS "$root/easybuild/foss-2023a-easybuild-devel" + + +# built with EasyBuild version 1.4.0dev From 737d77895d4003c277989811da6a04aee4e91208 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Tue, 28 Nov 2023 10:43:11 +0000 Subject: [PATCH 24/45] resolve templated values in name in _make_extension_list --- easybuild/framework/easyblock.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3ac26eb476..7df6d9abae 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1739,9 +1739,15 @@ def _make_extension_list(self): Each entry should be a (name, version) tuple or just (name, ) if no version exists """ - # We need only name and version, so don't resolve templates # Each extension in exts_list is either a string or a list/tuple with name, version as first entries - return [(ext, ) if isinstance(ext, string_type) else ext[:2] for ext in self.cfg.get_ref('exts_list')] + # As name can be a templated value we must resolve templates + exts_list = [] + for ext in self.cfg.get_ref('exts_list'): + if isinstance(ext, string_type): + exts_list.append((resolve_template(ext, self.cfg.template_values), )) + else: + exts_list.append((resolve_template(ext[0], self.cfg.template_values), ext[1])) + return exts_list def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True): """ From 873a001bb9bf98512ef218f9f103aa262cc218a1 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 2 Dec 2023 20:21:44 +0100 Subject: [PATCH 25/45] add deps --- .../modules/FFTW.MPI/3.3.10-gompi-2023a | 42 +++++++++++++++++ test/framework/modules/FFTW/3.3.10-GCC-12.3.0 | 43 +++++++++++++++++ .../modules/FlexiBLAS/3.3.1-GCC-12.3.0 | 47 +++++++++++++++++++ test/framework/modules/GCC/12.3.0 | 38 +++++++++++++++ test/framework/modules/GCCcore/12.3.0 | 37 +++++++++++++++ .../modules/OpenBLAS/0.3.23-GCC-12.3.0 | 38 +++++++++++++++ .../modules/ScaLAPACK/2.2.0-gompi-2023a-fb | 43 +++++++++++++++++ .../modules/binutils/2.40-GCCcore-12.3.0 | 44 +++++++++++++++++ test/framework/modules/foss/2023a | 41 +++++++++------- .../modules/zlib/1.2.13-GCCcore-12.3.0 | 44 +++++++++++++++++ 10 files changed, 401 insertions(+), 16 deletions(-) create mode 100644 test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a create mode 100644 test/framework/modules/FFTW/3.3.10-GCC-12.3.0 create mode 100644 test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 create mode 100644 test/framework/modules/GCC/12.3.0 create mode 100644 test/framework/modules/GCCcore/12.3.0 create mode 100644 test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 create mode 100644 test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb create mode 100644 test/framework/modules/binutils/2.40-GCCcore-12.3.0 create mode 100644 test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 diff --git a/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a b/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a new file mode 100644 index 0000000000..e35c12d348 --- /dev/null +++ b/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a @@ -0,0 +1,42 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. + + +More information +================ + - Homepage: https://www.fftw.org + } +} + +module-whatis {Description: FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data.} +module-whatis {Homepage: https://www.fftw.org} +module-whatis {URL: https://www.fftw.org} + +set root /prefix/software/FFTW.MPI/3.3.10-gompi-2023a + +conflict FFTW.MPI + +if { ![ is-loaded gompi/2023a ] } { + module load gompi/2023a +} + +if { ![ is-loaded FFTW/3.3.10-GCC-12.3.0 ] } { + module load FFTW/3.3.10-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +setenv EBROOTFFTWMPI "$root" +setenv EBVERSIONFFTWMPI "3.3.10" +setenv EBDEVELFFTWMPI "$root/easybuild/FFTW.MPI-3.3.10-gompi-2023a-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 b/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 new file mode 100644 index 0000000000..2ae82a0193 --- /dev/null +++ b/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 @@ -0,0 +1,43 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. + + +More information +================ + - Homepage: https://www.fftw.org + } +} + +module-whatis {Description: FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data.} +module-whatis {Homepage: https://www.fftw.org} +module-whatis {URL: https://www.fftw.org} + +set root /prefix/software/FFTW/3.3.10-GCC-12.3.0 + +conflict FFTW + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTFFTW "$root" +setenv EBVERSIONFFTW "3.3.10" +setenv EBDEVELFFTW "$root/easybuild/FFTW-3.3.10-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 b/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 new file mode 100644 index 0000000000..00f02d0460 --- /dev/null +++ b/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 @@ -0,0 +1,47 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FlexiBLAS is a wrapper library that enables the exchange of the BLAS and LAPACK implementation +used by a program without recompiling or relinking it. + + +More information +================ + - Homepage: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release + } +} + +module-whatis {Description: FlexiBLAS is a wrapper library that enables the exchange of the BLAS and LAPACK implementation +used by a program without recompiling or relinking it.} +module-whatis {Homepage: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release} +module-whatis {URL: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release} + +set root /prefix/software/FlexiBLAS/3.3.1-GCC-12.3.0 + +conflict FlexiBLAS + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded OpenBLAS/0.3.23-GCC-12.3.0 ] } { + module load OpenBLAS/0.3.23-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTFLEXIBLAS "$root" +setenv EBVERSIONFLEXIBLAS "3.3.1" +setenv EBDEVELFLEXIBLAS "$root/easybuild/FlexiBLAS-3.3.1-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/GCC/12.3.0 b/test/framework/modules/GCC/12.3.0 new file mode 100644 index 0000000000..5a21e5af1c --- /dev/null +++ b/test/framework/modules/GCC/12.3.0 @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). + + +More information +================ + - Homepage: https://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).} +module-whatis {Homepage: https://gcc.gnu.org/} +module-whatis {URL: https://gcc.gnu.org/} + +set root /prefix/software/GCC/12.3.0 + +conflict GCC + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +if { ![ is-loaded binutils/2.40-GCCcore-12.3.0 ] } { + module load binutils/2.40-GCCcore-12.3.0 +} + +setenv EBROOTGCC "$root" +setenv EBVERSIONGCC "12.3.0" +setenv EBDEVELGCC "$root/easybuild/GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/GCCcore/12.3.0 b/test/framework/modules/GCCcore/12.3.0 new file mode 100644 index 0000000000..745b111007 --- /dev/null +++ b/test/framework/modules/GCCcore/12.3.0 @@ -0,0 +1,37 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). + + +More information +================ + - Homepage: https://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).} +module-whatis {Homepage: https://gcc.gnu.org/} +module-whatis {URL: https://gcc.gnu.org/} + +set root /prefix/software/GCCcore/12.3.0 + +conflict GCCcore + +prepend-path CMAKE_LIBRARY_PATH $root/lib64 +prepend-path CMAKE_PREFIX_PATH $root +prepend-path LD_LIBRARY_PATH $root/lib64 +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTGCCCORE "$root" +setenv EBVERSIONGCCCORE "12.3.0" +setenv EBDEVELGCCCORE "$root/easybuild/GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 b/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 new file mode 100644 index 0000000000..89a72a65bc --- /dev/null +++ b/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. + + +More information +================ + - Homepage: http://www.openblas.net/ + } +} + +module-whatis {Description: OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version.} +module-whatis {Homepage: http://www.openblas.net/} +module-whatis {URL: http://www.openblas.net/} + +set root /prefix/software/OpenBLAS/0.3.23-GCC-12.3.0 + +conflict OpenBLAS + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTOPENBLAS "$root" +setenv EBVERSIONOPENBLAS "0.3.23" +setenv EBDEVELOPENBLAS "$root/easybuild/OpenBLAS-0.3.23-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb b/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb new file mode 100644 index 0000000000..7118eebf4f --- /dev/null +++ b/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb @@ -0,0 +1,43 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines + redesigned for distributed memory MIMD parallel computers. + + +More information +================ + - Homepage: https://www.netlib.org/scalapack/ + } +} + +module-whatis {Description: The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines + redesigned for distributed memory MIMD parallel computers.} +module-whatis {Homepage: https://www.netlib.org/scalapack/} +module-whatis {URL: https://www.netlib.org/scalapack/} + +set root /prefix/software/ScaLAPACK/2.2.0-gompi-2023a-fb + +conflict ScaLAPACK + +if { ![ is-loaded gompi/2023a ] } { + module load gompi/2023a +} + +if { ![ is-loaded FlexiBLAS/3.3.1-GCC-12.3.0 ] } { + module load FlexiBLAS/3.3.1-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTSCALAPACK "$root" +setenv EBVERSIONSCALAPACK "2.2.0" +setenv EBDEVELSCALAPACK "$root/easybuild/ScaLAPACK-2.2.0-gompi-2023a-fb-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/binutils/2.40-GCCcore-12.3.0 b/test/framework/modules/binutils/2.40-GCCcore-12.3.0 new file mode 100644 index 0000000000..7975adcb60 --- /dev/null +++ b/test/framework/modules/binutils/2.40-GCCcore-12.3.0 @@ -0,0 +1,44 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +binutils: GNU binary utilities + + +More information +================ + - Homepage: https://directory.fsf.org/project/binutils/ + } +} + +module-whatis {Description: binutils: GNU binary utilities} +module-whatis {Homepage: https://directory.fsf.org/project/binutils/} +module-whatis {URL: https://directory.fsf.org/project/binutils/} + +set root /prefix/software/binutils/2.40-GCCcore-12.3.0 + +conflict binutils + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +if { ![ is-loaded zlib/1.2.13-GCCcore-12.3.0 ] } { + module load zlib/1.2.13-GCCcore-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTBINUTILS "$root" +setenv EBVERSIONBINUTILS "2.40" +setenv EBDEVELBINUTILS "$root/easybuild/binutils-2.40-GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/foss/2023a b/test/framework/modules/foss/2023a index 05540a45f5..448c5a77be 100644 --- a/test/framework/modules/foss/2023a +++ b/test/framework/modules/foss/2023a @@ -1,39 +1,49 @@ #%Module - proc ModulesHelp { } { - puts stderr { GCC based compiler toolchain including - OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none) -} + puts stderr { + +Description +=========== +GNU Compiler Collection (GCC) based compiler toolchain, including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. + + +More information +================ + - Homepage: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain + } } -module-whatis {GCC based compiler toolchain including - OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. - Homepage: (none)} +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK.} +module-whatis {Homepage: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain} +module-whatis {URL: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain} -set root /prefix/software/foss/2023a +set root /prefix/software/foss/2023a -conflict foss +conflict foss -if { ![is-loaded GCC/12.3.0] } { +if { ![ is-loaded GCC/12.3.0 ] } { module load GCC/12.3.0 } -if { ![is-loaded OpenMPI/4.1.5-GCC-12.3.0] } { +if { ![ is-loaded OpenMPI/4.1.5-GCC-12.3.0 ] } { module load OpenMPI/4.1.5-GCC-12.3.0 } -if { ![is-loaded FlexiBLAS/3.3.1-GCC-12.3.0] } { +if { ![ is-loaded FlexiBLAS/3.3.1-GCC-12.3.0 ] } { module load FlexiBLAS/3.3.1-GCC-12.3.0 } -if { ![is-loaded FFTW/3.3.10-GCC-12.3.0] } { +if { ![ is-loaded FFTW/3.3.10-GCC-12.3.0 ] } { module load FFTW/3.3.10-GCC-12.3.0 } -if { ![is-loaded FFTW.MPI/3.3.10-gompi-2023a] } { +if { ![ is-loaded FFTW.MPI/3.3.10-gompi-2023a ] } { module load FFTW.MPI/3.3.10-gompi-2023a } -if { ![is-loaded ScaLAPACK/2.2.0-gompi-2023a-fb] } { +if { ![ is-loaded ScaLAPACK/2.2.0-gompi-2023a-fb ] } { module load ScaLAPACK/2.2.0-gompi-2023a-fb } @@ -41,5 +51,4 @@ setenv EBROOTFOSS "$root" setenv EBVERSIONFOSS "2023a" setenv EBDEVELFOSS "$root/easybuild/foss-2023a-easybuild-devel" - -# built with EasyBuild version 1.4.0dev +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 b/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 new file mode 100644 index 0000000000..a82945bd0c --- /dev/null +++ b/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 @@ -0,0 +1,44 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +zlib is designed to be a free, general-purpose, legally unencumbered -- that is, + not covered by any patents -- lossless data-compression library for use on virtually any + computer hardware and operating system. + + +More information +================ + - Homepage: https://www.zlib.net/ + } +} + +module-whatis {Description: zlib is designed to be a free, general-purpose, legally unencumbered -- that is, + not covered by any patents -- lossless data-compression library for use on virtually any + computer hardware and operating system.} +module-whatis {Homepage: https://www.zlib.net/} +module-whatis {URL: https://www.zlib.net/} + +set root /prefix/software/zlib/1.2.13-GCCcore-12.3.0 + +conflict zlib + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTZLIB "$root" +setenv EBVERSIONZLIB "1.2.13" +setenv EBDEVELZLIB "$root/easybuild/zlib-1.2.13-GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 From edde7796fde519b7b75058d52295729fdfd8a4e0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 5 Dec 2023 10:10:07 +0100 Subject: [PATCH 26/45] use source toolchain version when passing only `--try-toolchain` `process_software_build_specs` ialways adds a `toolchain` key even when only the name of the toolchain is available. This makes the code in `tweak` ignore the code path in using `toolchain_name` and/or `toolchain_version` with the fallback to using the values of the source toolchain. This in turn leads to failures further down when a value of `None` is used where an actual version is expected. Fix by only adding the `toolchain` key when we have both the name and version and hence a complete, valid toolchain spec. --- easybuild/tools/options.py | 6 ++-- .../test_ecs/i/iimpi/iimpi-2018a.eb | 20 +++++++++++ .../test_ecs/i/intel/intel-2018a.eb | 15 +++++---- .../test_ecs/t/toy/toy-0.0-foss-2018a.eb | 33 +++++++++++++++++++ test/framework/options.py | 12 +++++++ 5 files changed, 78 insertions(+), 8 deletions(-) create mode 100644 test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb create mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e69d96b2f6..eeba800c09 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1735,11 +1735,13 @@ def process_software_build_specs(options): }) # provide both toolchain and toolchain_name/toolchain_version keys - if 'toolchain_name' in build_specs: + try: build_specs['toolchain'] = { 'name': build_specs['toolchain_name'], - 'version': build_specs.get('toolchain_version', None), + 'version': build_specs['toolchain_version'], } + except KeyError: + pass # Don't set toolchain key if we don't have both keys # process --amend and --try-amend if options.amend or options.try_amend: diff --git a/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb new file mode 100644 index 0000000000..008b76a1b4 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb @@ -0,0 +1,20 @@ +# This is an easyconfig file for EasyBuild, see http://easybuilders.github.io/easybuild +easyblock = 'Toolchain' + +name = 'iimpi' +version = '2018a' + +homepage = 'https://software.intel.com/parallel-studio-xe' +description = """Intel C/C++ and Fortran compilers, alongside Intel MPI.""" + +toolchain = SYSTEM + +local_compver = '2016.1.150' +local_suff = '-GCC-4.9.3-2.25' +dependencies = [ + ('icc', local_compver, local_suff), + ('ifort', local_compver, local_suff), + ('impi', '5.1.2.150', '', ('iccifort', '%s%s' % (local_compver, local_suff))), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb b/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb index af32ecb585..8fdf23171b 100644 --- a/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb +++ b/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb @@ -8,14 +8,17 @@ description = """Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and toolchain = SYSTEM +local_compver = '2016.1.150' +local_gccver = '4.9.3' +local_binutilsver = '2.25' +local_gccsuff = '-GCC-%s-%s' % (local_gccver, local_binutilsver) # fake intel toolchain easyconfig, no dependencies (good enough for testing) local_fake_dependencies = [ - ('GCCcore', '6.4.0'), - ('binutils', '2.28', '-GCCcore-6.4.0'), - ('icc', '2018.1.163', '-GCCcore-6.4.0'), - ('ifort', '2018.1.163', '-GCCcore-6.4.0'), - ('impi', '2018.1.163', '', ('iccifort', '2018.1.163-GCCcore-6.4.0')), - ('imkl', '2018.1.163', '', ('iimpi', version)), + ('GCCcore', local_gccver), + ('binutils', local_binutilsver, '-GCCcore-%s' % local_gccver), + ('icc', local_compver, local_gccsuff), + ('ifort', local_compver, local_gccsuff), + ('impi', '5.1.2.150', '', ('iccifort', '%s%s' % (local_compver, local_gccsuff))), ] moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb new file mode 100644 index 0000000000..5f95d5d108 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb @@ -0,0 +1,33 @@ +name = 'toy' +version = '0.0' + +homepage = 'https://easybuilders.github.io/easybuild' +description = "Toy C program, 100% toy." + +toolchain = {'name': 'foss', 'version': '2018a'} +toolchainopts = {'pic': True, 'opt': True, 'optarch': True} + +sources = [SOURCE_TAR_GZ] +checksums = [[ + 'be662daa971a640e40be5c804d9d7d10', # default (MD5) + '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) + ('adler32', '0x998410035'), + ('crc32', '0x1553842328'), + ('md5', 'be662daa971a640e40be5c804d9d7d10'), + ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), + ('size', 273), +]] +patches = [ + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', + ('toy-extra.txt', 'toy-0.0'), +] + +sanity_check_paths = { + 'files': [('bin/yot', 'bin/toy')], + 'dirs': ['bin'], +} + +postinstallcmds = ["echo TOY > %(installdir)s/README"] + +moduleclass = 'tools' +# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/options.py b/test/framework/options.py index 51b67e39e5..050c123e0d 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2420,6 +2420,18 @@ def test_try(self): allargs = args + ['--software-version=1.2.3', '--toolchain=gompi,2018a'] self.assertErrorRegex(EasyBuildError, "version .* not available", self.eb_main, allargs, raise_error=True) + # Try changing only name or version of toolchain + args.pop(0) # Remove EC filename + test_cases = [ + (['toy-0.0-gompi-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-iimpi-2018a'), + (['toy-0.0-foss-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-intel-2018a'), + (['toy-0.0-gompi-2018a.eb', '--try-toolchain-version=2018b'], 'toy/0.0-gompi-2018b'), + ] + for extra_args, mod in test_cases: + outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True) + mod_regex = re.compile(r"\(module: %s\)$" % mod, re.M) + self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + def test_try_with_copy(self): """Test whether --try options are taken into account.""" ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 9613d9f10bee1408b473637d029535327db875c8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 5 Dec 2023 11:13:43 +0100 Subject: [PATCH 27/45] Avoid interferring with other tests --- .../test_ecs/t/toy/toy-0.0-foss-2018a.eb | 33 ------------------- test/framework/filetools.py | 2 +- test/framework/options.py | 6 +++- 3 files changed, 6 insertions(+), 35 deletions(-) delete mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb deleted file mode 100644 index 5f95d5d108..0000000000 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-foss-2018a.eb +++ /dev/null @@ -1,33 +0,0 @@ -name = 'toy' -version = '0.0' - -homepage = 'https://easybuilders.github.io/easybuild' -description = "Toy C program, 100% toy." - -toolchain = {'name': 'foss', 'version': '2018a'} -toolchainopts = {'pic': True, 'opt': True, 'optarch': True} - -sources = [SOURCE_TAR_GZ] -checksums = [[ - 'be662daa971a640e40be5c804d9d7d10', # default (MD5) - '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) - ('adler32', '0x998410035'), - ('crc32', '0x1553842328'), - ('md5', 'be662daa971a640e40be5c804d9d7d10'), - ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), - ('size', 273), -]] -patches = [ - 'toy-0.0_fix-silly-typo-in-printf-statement.patch', - ('toy-extra.txt', 'toy-0.0'), -] - -sanity_check_paths = { - 'files': [('bin/yot', 'bin/toy')], - 'dirs': ['bin'], -} - -postinstallcmds = ["echo TOY > %(installdir)s/README"] - -moduleclass = 'tools' -# trailing comment, leave this here, it may trigger bugs with extract_comments() diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 400a05c1b3..3b879bef85 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2401,7 +2401,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 91) + self.assertEqual(len(index), 92) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), diff --git a/test/framework/options.py b/test/framework/options.py index 050c123e0d..e6be49ab4f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2422,9 +2422,13 @@ def test_try(self): # Try changing only name or version of toolchain args.pop(0) # Remove EC filename + foss_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-foss-2018a.eb') + copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), foss_toy_ec) + write_file(foss_toy_ec, "toolchain['name'] = 'foss'", append=True) + test_cases = [ (['toy-0.0-gompi-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-iimpi-2018a'), - (['toy-0.0-foss-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-intel-2018a'), + ([foss_toy_ec, '--try-toolchain-name=intel'], 'toy/0.0-intel-2018a'), (['toy-0.0-gompi-2018a.eb', '--try-toolchain-version=2018b'], 'toy/0.0-gompi-2018b'), ] for extra_args, mod in test_cases: From aaeaa62bb72b522d0769105b750ba62a5679547e Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Wed, 6 Dec 2023 11:53:37 +0100 Subject: [PATCH 28/45] Add trailing spaces --- test/framework/docs.py | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index a271d63cfc..4f878fc728 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -462,43 +462,43 @@ LIST_SOFTWARE_DETAILED_JSON = """[ { - "toolchain": "system", - "description": "%(gcc_descr)s", - "homepage": "http://gcc.gnu.org/", - "version": "4.6.3", - "versionsuffix": "", + "toolchain": "system", + "description": "%(gcc_descr)s", + "homepage": "http://gcc.gnu.org/", + "version": "4.6.3", + "versionsuffix": "", "name": "GCC" }, { - "toolchain": "GCC/4.6.3", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.4", - "versionsuffix": "", + "toolchain": "GCC/4.6.3", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "system", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.4", - "versionsuffix": "", + "toolchain": "system", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "foss/2018a", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.5", - "versionsuffix": "", + "toolchain": "foss/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "intel/2018a", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.5", - "versionsuffix": "", + "toolchain": "intel/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", "name": "gzip" } ]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} From d2ccb1aa3364bf251b3a50cb973b98184e3977ed Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Wed, 6 Dec 2023 12:22:00 +0100 Subject: [PATCH 29/45] Enforce no space after comma from json.dumps (for older pythons) --- easybuild/tools/docs.py | 2 +- test/framework/docs.py | 50 ++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 344123d0f7..1bbb4c2d59 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1074,7 +1074,7 @@ def list_software_json(software, detailed=False): x = {} x['name'] = key - lines.append(json.dumps(x, indent=4) + ",") + lines.append(json.dumps(x, indent=4, separators=(',', ': ')) + ",") if not detailed: break # remove last line last comma diff --git a/test/framework/docs.py b/test/framework/docs.py index 4f878fc728..a271d63cfc 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -462,43 +462,43 @@ LIST_SOFTWARE_DETAILED_JSON = """[ { - "toolchain": "system", - "description": "%(gcc_descr)s", - "homepage": "http://gcc.gnu.org/", - "version": "4.6.3", - "versionsuffix": "", + "toolchain": "system", + "description": "%(gcc_descr)s", + "homepage": "http://gcc.gnu.org/", + "version": "4.6.3", + "versionsuffix": "", "name": "GCC" }, { - "toolchain": "GCC/4.6.3", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.4", - "versionsuffix": "", + "toolchain": "GCC/4.6.3", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "system", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.4", - "versionsuffix": "", + "toolchain": "system", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.4", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "foss/2018a", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.5", - "versionsuffix": "", + "toolchain": "foss/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", "name": "gzip" }, { - "toolchain": "intel/2018a", - "description": "%(gzip_descr)s", - "homepage": "http://www.gzip.org/", - "version": "1.5", - "versionsuffix": "", + "toolchain": "intel/2018a", + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "version": "1.5", + "versionsuffix": "", "name": "gzip" } ]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} From 5423a6fcedd1617ef5cd30883a8c0485abb8b62b Mon Sep 17 00:00:00 2001 From: Alexander Puck Neuwirth Date: Wed, 6 Dec 2023 13:38:52 +0100 Subject: [PATCH 30/45] Sort JSON keys for older python versions --- easybuild/tools/docs.py | 2 +- test/framework/docs.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 1bbb4c2d59..43f6038b26 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -1074,7 +1074,7 @@ def list_software_json(software, detailed=False): x = {} x['name'] = key - lines.append(json.dumps(x, indent=4, separators=(',', ': ')) + ",") + lines.append(json.dumps(x, indent=4, sort_keys=True, separators=(',', ': ')) + ",") if not detailed: break # remove last line last comma diff --git a/test/framework/docs.py b/test/framework/docs.py index a271d63cfc..70280892e4 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -462,44 +462,44 @@ LIST_SOFTWARE_DETAILED_JSON = """[ { - "toolchain": "system", "description": "%(gcc_descr)s", "homepage": "http://gcc.gnu.org/", + "name": "GCC", + "toolchain": "system", "version": "4.6.3", - "versionsuffix": "", - "name": "GCC" + "versionsuffix": "" }, { - "toolchain": "GCC/4.6.3", "description": "%(gzip_descr)s", "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "GCC/4.6.3", "version": "1.4", - "versionsuffix": "", - "name": "gzip" + "versionsuffix": "" }, { - "toolchain": "system", "description": "%(gzip_descr)s", "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "system", "version": "1.4", - "versionsuffix": "", - "name": "gzip" + "versionsuffix": "" }, { - "toolchain": "foss/2018a", "description": "%(gzip_descr)s", "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "foss/2018a", "version": "1.5", - "versionsuffix": "", - "name": "gzip" + "versionsuffix": "" }, { - "toolchain": "intel/2018a", "description": "%(gzip_descr)s", "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "intel/2018a", "version": "1.5", - "versionsuffix": "", - "name": "gzip" + "versionsuffix": "" } ]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} From 62810326b1d8947c108594af831364c9648dd55f Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 21:28:59 +0100 Subject: [PATCH 31/45] more deps --- .../modules/OpenMPI/4.1.5-GCC-12.3.0 | 70 +++++++++++++++++++ .../modules/PMIx/4.2.4-GCCcore-12.3.0 | 0 .../modules/UCC/1.2.0-GCCcore-12.3.0 | 0 .../modules/UCX/1.14.1-GCCcore-12.3.0 | 0 test/framework/modules/gompi/2023a | 38 ++++++++++ .../modules/hwloc/2.9.1-GCCcore-12.3.0 | 0 .../modules/libevent/2.1.12-GCCcore-12.3.0 | 0 7 files changed, 108 insertions(+) create mode 100644 test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 create mode 100644 test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 create mode 100644 test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 create mode 100644 test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 create mode 100644 test/framework/modules/gompi/2023a create mode 100644 test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 create mode 100644 test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 diff --git a/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 new file mode 100644 index 0000000000..eb1563c578 --- /dev/null +++ b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 @@ -0,0 +1,70 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The Open MPI Project is an open source MPI-3 implementation. + + +More information +================ + - Homepage: https://www.open-mpi.org/ + } +} + +module-whatis {Description: The Open MPI Project is an open source MPI-3 implementation.} +module-whatis {Homepage: https://www.open-mpi.org/} +module-whatis {URL: https://www.open-mpi.org/} + +set root /scratch/brussel/vo/000/bvo00005/vsc10009/ebtest/tclmodules/software/OpenMPI/4.1.5-GCC-12.3.0 + +conflict OpenMPI + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded zlib/1.2.13-GCCcore-12.3.0 ] } { + module load zlib/1.2.13-GCCcore-12.3.0 +} + +if { ![ is-loaded hwloc/2.9.1-GCCcore-12.3.0 ] } { + module load hwloc/2.9.1-GCCcore-12.3.0 +} + +if { ![ is-loaded libevent/2.1.12-GCCcore-12.3.0 ] } { + module load libevent/2.1.12-GCCcore-12.3.0 +} + +if { ![ is-loaded UCX/1.14.1-GCCcore-12.3.0 ] } { + module load UCX/1.14.1-GCCcore-12.3.0 +} + +if { ![ is-loaded libfabric/1.18.0-GCCcore-12.3.0 ] } { + module load libfabric/1.18.0-GCCcore-12.3.0 +} + +if { ![ is-loaded PMIx/4.2.4-GCCcore-12.3.0 ] } { + module load PMIx/4.2.4-GCCcore-12.3.0 +} + +if { ![ is-loaded UCC/1.2.0-GCCcore-12.3.0 ] } { + module load UCC/1.2.0-GCCcore-12.3.0 +} + +prepend_path("CMAKE_PREFIX_PATH", root) +prepend_path("CPATH", pathJoin(root, "include")) +prepend_path("LD_LIBRARY_PATH", pathJoin(root, "lib")) +prepend_path("LIBRARY_PATH", pathJoin(root, "lib")) +prepend_path("MANPATH", pathJoin(root, "share/man")) +prepend_path("PATH", pathJoin(root, "bin")) +prepend_path("PKG_CONFIG_PATH", pathJoin(root, "lib/pkgconfig")) +prepend_path("XDG_DATA_DIRS", pathJoin(root, "share")) + +setenv EBROOTOPENMPI "$root" +setenv EBVERSIONOPENMPI "4.1.5" +setenv EBDEVELOPENMPI "$root/easybuild/OpenMPI-4.1.5-GCC-12.3.0-easybuild-devel" + +setenv SLURM_MPI_TYPE "pmix" +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 b/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 b/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 b/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/modules/gompi/2023a b/test/framework/modules/gompi/2023a new file mode 100644 index 0000000000..c81eeff8df --- /dev/null +++ b/test/framework/modules/gompi/2023a @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. + + +More information +================ + - Homepage: (none) + } +} + +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support.} +module-whatis {Homepage: (none)} +module-whatis {URL: (none)} + +set root /prefix/software/gompi/2023a + +conflict gompi + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded OpenMPI/4.1.5-GCC-12.3.0 ] } { + module load OpenMPI/4.1.5-GCC-12.3.0 +} + +setenv EBROOTGOMPI "$root" +setenv EBVERSIONGOMPI "2023a" +setenv EBDEVELGOMPI "$root/easybuild/gompi-2023a-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 b/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 b/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 new file mode 100644 index 0000000000..e69de29bb2 From 0b702ee4cbee3128b3766897dec019ae0bcc9186 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 22:03:52 +0100 Subject: [PATCH 32/45] add magic cookies --- test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 | 1 + test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 | 1 + test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 | 1 + test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 | 1 + test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 | 1 + 5 files changed, 5 insertions(+) diff --git a/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 b/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 +++ b/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 b/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 +++ b/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 b/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 +++ b/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 b/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 +++ b/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 b/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 index e69de29bb2..1c148cdd28 100644 --- a/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 +++ b/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module From ce64b015430e339a86d9bfe8624e2fc774d0615d Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 22:10:54 +0100 Subject: [PATCH 33/45] fix test_avail --- test/framework/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 7bf87130bd..2d0c52606e 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -197,12 +197,12 @@ def test_avail(self): # test modules include 3 GCC modules and one GCCcore module ms = self.modtool.available('GCC') - expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30'] + expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30', 'GCC/12.3.0'] # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl., # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules): - expected.append('GCCcore/6.2.0') + expected.extend(['GCCcore/6.2.0', 'GCCcore/12.3.0']) self.assertEqual(ms, expected) # test modules include one GCC/4.6.3 module From 21c15f579a1f3e14ef1afd9463d59613aba2db91 Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 22:44:10 +0100 Subject: [PATCH 34/45] more fixes --- test/framework/modules.py | 4 ++-- test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 2d0c52606e..916e407d92 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -197,12 +197,12 @@ def test_avail(self): # test modules include 3 GCC modules and one GCCcore module ms = self.modtool.available('GCC') - expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30', 'GCC/12.3.0'] + expected = ['GCC/12.3.0', 'GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30'] # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl., # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules): - expected.extend(['GCCcore/6.2.0', 'GCCcore/12.3.0']) + expected.extend(['GCCcore/12.3.0', 'GCCcore/6.2.0']) self.assertEqual(ms, expected) # test modules include one GCC/4.6.3 module diff --git a/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 index eb1563c578..481f81e627 100644 --- a/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 +++ b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 @@ -53,14 +53,14 @@ if { ![ is-loaded UCC/1.2.0-GCCcore-12.3.0 ] } { module load UCC/1.2.0-GCCcore-12.3.0 } -prepend_path("CMAKE_PREFIX_PATH", root) -prepend_path("CPATH", pathJoin(root, "include")) -prepend_path("LD_LIBRARY_PATH", pathJoin(root, "lib")) -prepend_path("LIBRARY_PATH", pathJoin(root, "lib")) -prepend_path("MANPATH", pathJoin(root, "share/man")) -prepend_path("PATH", pathJoin(root, "bin")) -prepend_path("PKG_CONFIG_PATH", pathJoin(root, "lib/pkgconfig")) -prepend_path("XDG_DATA_DIRS", pathJoin(root, "share")) +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share setenv EBROOTOPENMPI "$root" setenv EBVERSIONOPENMPI "4.1.5" From 3f05f7ed5b4545c1acc78817d63d1b692252abeb Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 22:59:21 +0100 Subject: [PATCH 35/45] add another dep --- test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 | 1 + 1 file changed, 1 insertion(+) create mode 100644 test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 diff --git a/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 b/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 new file mode 100644 index 0000000000..1c148cdd28 --- /dev/null +++ b/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module From 646f7a60310757ebb9a15f3d6659fd2c076bf8aa Mon Sep 17 00:00:00 2001 From: sam Date: Wed, 6 Dec 2023 23:47:05 +0100 Subject: [PATCH 36/45] more test fixes --- test/framework/modules.py | 2 +- test/framework/toolchain.py | 131 +++++++++++++++++++++++++----------- 2 files changed, 92 insertions(+), 41 deletions(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 916e407d92..4a8a6b15df 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -54,7 +54,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 92 +TEST_MODULES_COUNT = 110 class ModulesTest(EnhancedTestCase): diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index bee0eec8a9..aacde3baf7 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1840,46 +1840,97 @@ def test_old_new_iccifort(self): scalapack_mt_static_libs_fosscuda = "libscalapack.a,libopenblas.a,libgfortran.a,libpthread.a" scalapack_mt_shared_libs_fosscuda = scalapack_mt_static_libs_fosscuda.replace('.a', '.' + shlib_ext) - for toolc in [('fosscuda', '2018a'), ('foss', '2023a')]: - tc = self.get_toolchain(toolc[0], version=toolc[1]) - tc.prepare() - self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) - self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) - self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) - - self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) - self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) - - self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) - - self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) - self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) - self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) - self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) - self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) - self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) - self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) - self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) - - self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) - self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) - self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) - self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) - self.modtool.purge() + tc = self.get_toolchain('fosscuda', version='2018a') + tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda) + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], lapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], lapack_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], blas_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], blas_static_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], blas_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], blas_mt_static_libs_fosscuda) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], scalapack_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], scalapack_static_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], scalapack_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) + self.modtool.purge() + + tc = self.get_toolchain('foss', version='2023a') + tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], + blas_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], + blas_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], + lapack_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], + lapack_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], + blas_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], + blas_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], + blas_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], + blas_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], + scalapack_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], + scalapack_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], + scalapack_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], + scalapack_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.modtool.purge() tc = self.get_toolchain('intel', version='2018a') tc.prepare() From 06461ea123980c124d4fd73fbe6037f6df0b8fe3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 10:53:15 +0100 Subject: [PATCH 37/45] Introduce `--module-cache-suffix` to allow multiple caches This is useful when having multiple module trees (i.e. different `--installprefix`) to not overwrite the cache file of one with the other when using `--update-modules-tool-cache`. Inspired by how LMod itself creates the cache, e.g. `~/.cache/lmod/spiderT.rapids_x86_64_Linux.lua` --- easybuild/tools/config.py | 1 + easybuild/tools/modules.py | 3 ++- easybuild/tools/options.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6c0a173fe4..ee8eb5fc53 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -239,6 +239,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'job_polling_interval', 'job_target_resource', 'locks_dir', + 'module_cache_suffix', 'modules_footer', 'modules_header', 'mpi_cmd_template', diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..f2d74c2137 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1436,7 +1436,8 @@ def update(self): # don't actually update local cache when testing, just return the cache contents return stdout else: - cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') + suffix = build_option('module_cache_suffix') or '' + cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT%s.lua' % suffix) self.log.debug("Updating Lmod spider cache %s with output from '%s'" % (cache_fp, ' '.join(cmd))) cache_dir = os.path.dirname(cache_fp) if not os.path.exists(cache_dir): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index eeba800c09..b840bc4d53 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -461,6 +461,9 @@ def override_options(self): "environment variable and its value separated by a colon (':')", None, 'store', DEFAULT_MINIMAL_BUILD_ENV), 'minimal-toolchains': ("Use minimal toolchain when resolving dependencies", None, 'store_true', False), + 'module-cache-suffix': ("Suffix to add to the cache file name (before the extension) " + "when updating the modules tool cache", + None, 'store', None), 'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS), None, 'store_true', False), 'modules-tool-version-check': ("Check version of modules tool being used", None, 'store_true', True), From b4488e0d7083d82ac05f01e4f11c5b32aa4516e1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 10:40:20 +0100 Subject: [PATCH 38/45] Fix writing spider cache for LMod >= 8.7.12 --- easybuild/tools/modules.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 386643e706..a4c234c352 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1345,7 +1345,6 @@ class Lmod(ModulesTool): DEPR_VERSION = '7.0.0' REQ_VERSION_DEPENDS_ON = '7.6.1' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" - USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') SHOW_HIDDEN_OPTION = '--show-hidden' @@ -1361,7 +1360,14 @@ def __init__(self, *args, **kwargs): setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False) super(Lmod, self).__init__(*args, **kwargs) - self.supports_depends_on = StrictVersion(self.version) >= StrictVersion(self.REQ_VERSION_DEPENDS_ON) + version = StrictVersion(self.version) + + self.supports_depends_on = version >= self.REQ_VERSION_DEPENDS_ON + # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html + if version >= '8.7.12': + self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod') + else: + self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') def check_module_function(self, *args, **kwargs): """Check whether selected module tool matches 'module' function definition.""" From 7f73abf27baf6d2c3aba12ed631bc433f846a143 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 12:34:23 +0100 Subject: [PATCH 39/45] Fix `--inject-checksums` when extension has tuple-format patch `make_list_lines` had a `"'%s'" % x` part where `x` is the patch entry. However that could be a tuple (or even dict) to designate extra options such as the subdir or level where the patch should be applied. This then failed with "TypeError: not all arguments converted during string formatting" Fix by checking the type first and add a test for this case. --- easybuild/framework/easyblock.py | 8 ++++++- test/framework/options.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 3ac26eb476..944911cf27 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -4656,8 +4656,14 @@ def inject_checksums(ecs, checksum_type): """ def make_list_lines(values, indent_level): """Make lines for list of values.""" + def to_str(s): + if isinstance(s, string_type): + return "'%s'" % s + else: + return str(s) + line_indent = INDENT_4SPACES * indent_level - return [line_indent + "'%s'," % x for x in values] + return [line_indent + to_str(x) + ',' for x in values] def make_checksum_lines(checksums, indent_level): """Make lines for list of checksums.""" diff --git a/test/framework/options.py b/test/framework/options.py index e6be49ab4f..f3173dbb87 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5948,6 +5948,42 @@ def test_inject_checksums(self): ] self.assertEqual(ec['checksums'], expected_checksums) + # Also works for extensions (all 3 patch formats) + write_file(test_ec, textwrap.dedent(""" + exts_list = [ + ("bar", "0.0", { + 'sources': ['bar-0.0-local.tar.gz'], + 'patches': [ + 'bar-0.0_fix-silly-typo-in-printf-statement.patch', # normal patch + ('bar-0.0_fix-very-silly-typo-in-printf-statement.patch', 0), # patch with patch level + ('toy-0.0_fix-silly-typo-in-printf-statement.patch', 'toy_subdir'), + ], + }), + ] + """), append=True) + self._run_mock_eb(args, raise_error=True, strip=True) + ec = EasyConfigParser(test_ec).get_config_dict() + ext = ec['exts_list'][0] + self.assertEqual((ext[0], ext[1]), ("bar", "0.0")) + ext_opts = ext[2] + expected_patches = [ + 'bar-0.0_fix-silly-typo-in-printf-statement.patch', + ('bar-0.0_fix-very-silly-typo-in-printf-statement.patch', 0), + ('toy-0.0_fix-silly-typo-in-printf-statement.patch', 'toy_subdir') + ] + self.assertEqual(ext_opts['patches'], expected_patches) + expected_checksums = [ + {'bar-0.0-local.tar.gz': + 'f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'}, + {'bar-0.0_fix-silly-typo-in-printf-statement.patch': + '84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab'}, + {'bar-0.0_fix-very-silly-typo-in-printf-statement.patch': + 'd0bf102f9c5878445178c5f49b7cd7546e704c33fe2060c7354b7e473cfeb52b'}, + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'} + ] + self.assertEqual(ext_opts['checksums'], expected_checksums) + # passing easyconfig filename as argument to --inject-checksums results in error being reported, # because it's not a valid type of checksum args = ['--inject-checksums', test_ec] From 2816bf8caf273f3e05e69f41f0a0b0788acefb23 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 20 Dec 2023 13:32:11 +0100 Subject: [PATCH 40/45] fix LooseVersion on Python2 --- easybuild/tools/loose_version.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index e5594fc5fb..1855fee74a 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -81,6 +81,9 @@ def _cmp(self, other): def __eq__(self, other): return self._cmp(other) == 0 + def __ne__(self, other): + return self._cmp(other) != 0 + def __lt__(self, other): return self._cmp(other) < 0 From 03d02e6e9b8edfa3aec9f50086a5e7908de4a373 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 20 Dec 2023 13:50:10 +0100 Subject: [PATCH 41/45] Enhance tests for LooseVersion --- test/framework/utilities_test.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 0aa3a9956e..ba4766f302 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -123,6 +123,15 @@ def test_LooseVersion(self): self.assertLess(LooseVersion('1.02'), '2.01') self.assertLessEqual('1.02', LooseVersion('2.01')) self.assertLessEqual(LooseVersion('1.02'), '2.01') + # Negation of all ops, i.e. verify each op can return False + self.assertFalse(LooseVersion('2.02') != '2.02') + self.assertFalse(LooseVersion('2.02') <= '2.01') + self.assertFalse(LooseVersion('2.02') < '2.01') + self.assertFalse(LooseVersion('2.02') < '2.02') + self.assertFalse(LooseVersion('2.02') == '2.03') + self.assertFalse(LooseVersion('2.02') >= '2.03') + self.assertFalse(LooseVersion('2.02') > '2.03') + self.assertFalse(LooseVersion('2.02') > '2.02') # Some comparisons we might do: Full version on left hand side, shorter on right self.assertGreater(LooseVersion('2.1.5'), LooseVersion('2.1')) @@ -135,7 +144,7 @@ def test_LooseVersion(self): self.assertGreater(LooseVersion('1.0'), LooseVersion('1')) self.assertLess(LooseVersion('1'), LooseVersion('1.0')) - # The following test is taken from Python disutils tests + # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 versions = (('1.5.1', '1.5.2b2', -1), ('161', '3.10a', 1), From 5aec5e21be38c515f3b3f944920ff86067f86ecb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 26 Dec 2023 16:44:50 +0100 Subject: [PATCH 42/45] use more recent easyblocks PR in test_github_merge_pr --- test/framework/options.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index f3173dbb87..4b5e1afce8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4822,7 +4822,7 @@ def test_github_merge_pr(self): # --merge-pr also works on easyblocks (& framework) PRs args = [ '--merge-pr', - '2805', + '2995', '--pr-target-repo=easybuild-easyblocks', '-D', '--github-user=%s' % GITHUB_TEST_ACCOUNT, @@ -4830,12 +4830,12 @@ def test_github_merge_pr(self): stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) self.assertEqual(stderr.strip(), '') expected_stdout = '\n'.join([ - "Checking eligibility of easybuilders/easybuild-easyblocks PR #2805 for merging...", + "Checking eligibility of easybuilders/easybuild-easyblocks PR #2995 for merging...", "* targets develop branch: OK", "* test suite passes: OK", "* no pending change requests: OK", - "* approved review: OK (by ocaisa)", - "* milestone is set: OK (4.6.2)", + "* approved review: OK (by boegel)", + "* milestone is set: OK (4.8.1)", "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", From 86971782b2d4329efb2caeb78af72a0bb9a0cc5e Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 29 Dec 2023 15:49:19 +0100 Subject: [PATCH 43/45] prepare release notes for EasyBuild v4.9.0 + bump version to 4.9.0 --- RELEASE_NOTES | 27 +++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index a43ce28535..641a0d70d6 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,33 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.0 (30 December 2023) +------------------------- + +feature release + +- various enhancements, including: + - allow tweaking of easyconfigs from different toolchains (#3669) + - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403) +- various bug fixes, including: + - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361) + - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369) + - Adapt module function check for Environment Modules v4+ (#4371) + - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375) + - Use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377) + - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379) + - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385) + - eliminate duplicate multideps from generated module files (#4386) + - resolve templated values in extension names in `_make_extension_list` (#4392) + - use source toolchain version when passing only `--try-toolchain` (#4395) + - fix writing spider cache for Lmod >= 8.7.12 (#4402) + - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405) + - fix LooseVersion on Python 2.7 (#4408) + - use more recent easyblocks PR in `test_github_merge_pr` (#4414) +- other changes: + - extend test that checks build environment to recent `foss/2023a` toolchain (#4391) + + v4.8.2 (29 October 2023) ------------------------ diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 655f7d0a49..e3d5feb017 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,7 +45,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.0.dev0') +VERSION = LooseVersion('4.9.0') UNKNOWN = 'UNKNOWN' From 4288f5b1274fc4afe362ecbda73e0442520745a7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2023 17:45:03 +0100 Subject: [PATCH 44/45] minor code style fixes in list_software_json --- easybuild/tools/docs.py | 26 +++++++++++++++----------- easybuild/tools/options.py | 4 ++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 43f6038b26..199cd22d78 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -38,8 +38,8 @@ """ import copy import inspect -import os import json +import os from easybuild.tools import LooseVersion from easybuild.base import fancylogger @@ -73,10 +73,10 @@ DETAILED = 'detailed' SIMPLE = 'simple' +FORMAT_JSON = 'json' FORMAT_MD = 'md' FORMAT_RST = 'rst' FORMAT_TXT = 'txt' -FORMAT_JSON = 'json' def generate_doc(name, params): @@ -1065,22 +1065,26 @@ def list_software_json(software, detailed=False): """ lines = ['['] for key in sorted(software, key=lambda x: x.lower()): - for tmp in software[key]: + for entry in software[key]: if detailed: # deep copy here to avoid modifying the original dict - x = copy.deepcopy(tmp) - x['description'] = ' '.join(x['description'].split('\n')).strip() + entry = copy.deepcopy(entry) + entry['description'] = ' '.join(entry['description'].split('\n')).strip() else: - x = {} - x['name'] = key + entry = {} + entry['name'] = key - lines.append(json.dumps(x, indent=4, sort_keys=True, separators=(',', ': ')) + ",") + lines.append(json.dumps(entry, indent=4, sort_keys=True, separators=(',', ': ')) + ",") if not detailed: break - # remove last line last comma + + # remove trailing comma on last line if len(lines) > 1: - lines[-1] = lines[-1][:-1] - return '\n'.join(lines) + '\n]' + lines[-1] = lines[-1].rstrip(',') + + lines.append(']') + + return '\n'.join(lines) def list_toolchains(output_format=FORMAT_TXT): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 64c46b61b5..fec9bb5da6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.configobj import ConfigObj, ConfigObjError -from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON +from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains @@ -470,7 +470,7 @@ def override_options(self): 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, - [FORMAT_MD, FORMAT_RST, FORMAT_TXT, FORMAT_JSON]), + [FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), From 81383e2dc9f1b612f9788e69ed746a6da51801d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2023 18:33:26 +0100 Subject: [PATCH 45/45] minor tweaks to 4.9.0 release notes --- RELEASE_NOTES | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 641a0d70d6..94b66db903 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -11,13 +11,14 @@ feature release - various enhancements, including: - allow tweaking of easyconfigs from different toolchains (#3669) + - add support for `--list-software --output-format=json` (#4152) - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403) - various bug fixes, including: - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361) - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369) - - Adapt module function check for Environment Modules v4+ (#4371) + - adapt module function check for Environment Modules v4+ (#4371) - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375) - - Use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377) + - use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377) - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379) - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385) - eliminate duplicate multideps from generated module files (#4386) @@ -25,7 +26,7 @@ feature release - use source toolchain version when passing only `--try-toolchain` (#4395) - fix writing spider cache for Lmod >= 8.7.12 (#4402) - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405) - - fix LooseVersion on Python 2.7 (#4408) + - fix `LooseVersion` when running with Python 2.7 (#4408) - use more recent easyblocks PR in `test_github_merge_pr` (#4414) - other changes: - extend test that checks build environment to recent `foss/2023a` toolchain (#4391)