Merge pull request #4420 from easybuilders/4.9.x

release EasyBuild v4.9.0

.github/workflows/unit_tests.yml

@@ -103,9 +103,9 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
-        if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
+        if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
             SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
             python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";

RELEASE_NOTES

@@ -4,6 +4,34 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.0 (30 December 2023)
+-------------------------
+
+feature release
+
+- various enhancements, including:
+  - allow tweaking of easyconfigs from different toolchains (#3669)
+  - add support for `--list-software --output-format=json` (#4152)
+  - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403)
+- various bug fixes, including:
+  - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361)
+  - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369)
+  - adapt module function check for Environment Modules v4+ (#4371)
+  - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375)
+  - use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377)
+  - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379)
+  - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385)
+  - eliminate duplicate multideps from generated module files (#4386)
+  - resolve templated values in extension names in `_make_extension_list` (#4392)
+  - use source toolchain version when passing only `--try-toolchain` (#4395)
+  - fix writing spider cache for Lmod >= 8.7.12 (#4402)
+  - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405)
+  - fix `LooseVersion` when running with Python 2.7 (#4408)
+  - use more recent easyblocks PR in `test_github_merge_pr` (#4414)
+- other changes:
+  - extend test that checks build environment to recent `foss/2023a` toolchain (#4391)
+
+
 v4.8.2 (29 October 2023)
 ------------------------
 

easybuild/framework/easyblock.py

@@ -1739,9 +1739,15 @@ def _make_extension_list(self):
 
         Each entry should be a (name, version) tuple or just (name, ) if no version exists
         """
-        # We need only name and version, so don't resolve templates
         # Each extension in exts_list is either a string or a list/tuple with name, version as first entries
-        return [(ext, ) if isinstance(ext, string_type) else ext[:2] for ext in self.cfg.get_ref('exts_list')]
+        # As name can be a templated value we must resolve templates
+        exts_list = []
+        for ext in self.cfg.get_ref('exts_list'):
+            if isinstance(ext, string_type):
+                exts_list.append((resolve_template(ext, self.cfg.template_values), ))
+            else:
+                exts_list.append((resolve_template(ext[0], self.cfg.template_values), ext[1]))
+        return exts_list
 
     def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True):
         """
@@ -4656,8 +4662,14 @@ def inject_checksums(ecs, checksum_type):
     """
     def make_list_lines(values, indent_level):
         """Make lines for list of values."""
+        def to_str(s):
+            if isinstance(s, string_type):
+                return "'%s'" % s
+            else:
+                return str(s)
+
         line_indent = INDENT_4SPACES * indent_level
-        return [line_indent + "'%s'," % x for x in values]
+        return [line_indent + to_str(x) + ',' for x in values]
 
     def make_checksum_lines(checksums, indent_level):
         """Make lines for list of checksums."""

easybuild/framework/easyconfig/tools.py

@@ -472,7 +472,7 @@ def find_related_easyconfigs(path, ec):
     if len(parsed_version) >= 2:
         version_patterns.append(r'%s\.%s\.\w+' % tuple(parsed_version[:2]))  # major/minor version match
     if parsed_version != parsed_version[0]:
-        version_patterns.append(r'%s\.[\d-]+\.\w+' % parsed_version[0])  # major version match
+        version_patterns.append(r'%s\.[\d-]+(\.\w+)*' % parsed_version[0])  # major version match
     version_patterns.append(r'[\w.]+')  # any version
 
     regexes = []

easybuild/framework/easyconfig/tweak.py

@@ -91,15 +91,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
     tweaked_ecs_path, tweaked_ecs_deps_path = None, None
     if targetdirs is not None:
         tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs
-    # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
-    toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
-    if len(toolchains) > 1:
-        raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s",
-                             toolchains)
-    # Toolchain is unique, let's store it
-    source_toolchain = easyconfigs[-1]['ec']['toolchain']
     modifying_toolchains_or_deps = False
-    target_toolchain = {}
     src_to_dst_tc_mapping = {}
     revert_to_regex = False
 
@@ -117,6 +109,16 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
                 revert_to_regex = True
 
         if not revert_to_regex:
+            # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble)
+            toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs])
+            if len(toolchains) > 1:
+                raise EasyBuildError("Multiple toolchains featured in easyconfigs, "
+                                     "--try-X not supported in that case: %s",
+                                     toolchains)
+            # Toolchain is unique, let's store it
+            source_toolchain = easyconfigs[-1]['ec']['toolchain']
+            target_toolchain = {}
+
             # we're doing something that involves the toolchain hierarchy;
             # obtain full dependency graph for specified easyconfigs;
             # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first)

easybuild/toolchains/compiler/intel_compilers.py

@@ -109,8 +109,9 @@ def set_variables(self):
             self.options.options_map['loose'] = ['fp-model fast']
             # fp-model fast=2 gives "warning: overriding '-ffp-model=fast=2' option with '-ffp-model=fast'"
             self.options.options_map['veryloose'] = ['fp-model fast']
-            # recommended in porting guide
-            self.options.options_map['openmp'] = ['fiopenmp']
+            # recommended in porting guide: qopenmp, unlike fiopenmp, works for both classic and oneapi compilers
+            # https://www.intel.com/content/www/us/en/developer/articles/guide/porting-guide-for-ifort-to-ifx.html
+            self.options.options_map['openmp'] = ['qopenmp']
 
             # -xSSE2 is not supported by Intel oneAPI compilers,
             # so use -march=x86-64 -mtune=generic when using optarch=GENERIC

easybuild/toolchains/linalg/flexiblas.py

@@ -70,6 +70,7 @@ class FlexiBLAS(LinAlg):
     """
     BLAS_MODULE_NAME = ['FlexiBLAS']
     BLAS_LIB = ['flexiblas']
+    BLAS_LIB_MT = ['flexiblas']
     BLAS_INCLUDE_DIR = [os.path.join('include', 'flexiblas')]
     BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS
 

easybuild/tools/config.py

@@ -239,6 +239,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'job_polling_interval',
         'job_target_resource',
         'locks_dir',
+        'module_cache_suffix',
         'modules_footer',
         'modules_header',
         'mpi_cmd_template',

easybuild/tools/docs.py

@@ -38,6 +38,7 @@
 """
 import copy
 import inspect
+import json
 import os
 from easybuild.tools import LooseVersion
 
@@ -72,6 +73,7 @@
 DETAILED = 'detailed'
 SIMPLE = 'simple'
 
+FORMAT_JSON = 'json'
 FORMAT_MD = 'md'
 FORMAT_RST = 'rst'
 FORMAT_TXT = 'txt'
@@ -115,6 +117,11 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT):
     return generate_doc('avail_cfgfile_constants_%s' % output_format, [go_cfg_constants])
 
 
+def avail_cfgfile_constants_json(go_cfg_constants):
+    """Generate documentation on constants for configuration files in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_cfgfile_constants_json")
+
+
 def avail_cfgfile_constants_txt(go_cfg_constants):
     """Generate documentation on constants for configuration files in txt format"""
     doc = [
@@ -184,6 +191,11 @@ def avail_easyconfig_constants(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_constants_%s' % output_format, [])
 
 
+def avail_easyconfig_constants_json():
+    """Generate easyconfig constant documentation in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_constants_json")
+
+
 def avail_easyconfig_constants_txt():
     """Generate easyconfig constant documentation in txt format"""
     doc = ["Constants that can be used in easyconfigs"]
@@ -242,6 +254,11 @@ def avail_easyconfig_licenses(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_licenses_%s' % output_format, [])
 
 
+def avail_easyconfig_licenses_json():
+    """Generate easyconfig license documentation in json format"""
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_licenses_json")
+
+
 def avail_easyconfig_licenses_txt():
     """Generate easyconfig license documentation in txt format"""
     doc = ["License constants that can be used in easyconfigs"]
@@ -354,6 +371,13 @@ def avail_easyconfig_params_rst(title, grouped_params):
     return '\n'.join(doc)
 
 
+def avail_easyconfig_params_json():
+    """
+    Compose overview of available easyconfig parameters, in json format.
+    """
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json")
+
+
 def avail_easyconfig_params_txt(title, grouped_params):
     """
     Compose overview of available easyconfig parameters, in plain text format.
@@ -426,6 +450,11 @@ def avail_easyconfig_templates(output_format=FORMAT_TXT):
     return generate_doc('avail_easyconfig_templates_%s' % output_format, [])
 
 
+def avail_easyconfig_templates_json():
+    """ Returns template documentation in json text format """
+    raise NotImplementedError("JSON output format not supported for avail_easyconfig_templates")
+
+
 def avail_easyconfig_templates_txt():
     """ Returns template documentation in plain text format """
     # This has to reflect the methods/steps used in easyconfig _generate_template_values
@@ -640,6 +669,8 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings
 
 
 def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT):
+    if output_format == FORMAT_JSON:
+        raise NotImplementedError("JSON output format not supported for list_easyblocks")
     format_strings = {
         FORMAT_MD: {
             'det_root_templ': "- **%s** (%s%s)",
@@ -1024,6 +1055,38 @@ def list_software_txt(software, detailed=False):
     return '\n'.join(lines)
 
 
+def list_software_json(software, detailed=False):
+    """
+    Return overview of supported software in json
+
+    :param software: software information (strucuted like list_software does)
+    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
+    :return: multi-line string presenting requested info
+    """
+    lines = ['[']
+    for key in sorted(software, key=lambda x: x.lower()):
+        for entry in software[key]:
+            if detailed:
+                # deep copy here to avoid modifying the original dict
+                entry = copy.deepcopy(entry)
+                entry['description'] = ' '.join(entry['description'].split('\n')).strip()
+            else:
+                entry = {}
+            entry['name'] = key
+
+            lines.append(json.dumps(entry, indent=4, sort_keys=True, separators=(',', ': ')) + ",")
+            if not detailed:
+                break
+
+    # remove trailing comma on last line
+    if len(lines) > 1:
+        lines[-1] = lines[-1].rstrip(',')
+
+    lines.append(']')
+
+    return '\n'.join(lines)
+
+
 def list_toolchains(output_format=FORMAT_TXT):
     """Show list of known toolchains."""
     _, all_tcs = search_toolchain('')
@@ -1173,6 +1236,11 @@ def list_toolchains_txt(tcs):
     return '\n'.join(doc)
 
 
+def list_toolchains_json(tcs):
+    """ Returns overview of all toolchains in json format """
+    raise NotImplementedError("JSON output not implemented yet for --list-toolchains")
+
+
 def avail_toolchain_opts(name, output_format=FORMAT_TXT):
     """Show list of known options for given toolchain."""
     tc_class, _ = search_toolchain(name)
@@ -1226,6 +1294,11 @@ def avail_toolchain_opts_rst(name, tc_dict):
     return '\n'.join(doc)
 
 
+def avail_toolchain_opts_json(name, tc_dict):
+    """ Returns overview of toolchain options in jsonformat """
+    raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts")
+
+
 def avail_toolchain_opts_txt(name, tc_dict):
     """ Returns overview of toolchain options in txt format """
     doc = ["Available options for %s toolchain:" % name]
@@ -1252,6 +1325,13 @@ def get_easyblock_classes(package_name):
     return easyblocks
 
 
+def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None):
+    """
+    Compose overview of all easyblocks in the given package in json format
+    """
+    raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview")
+
+
 def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None):
     """
     Compose overview of all easyblocks in the given package in MarkDown format

easybuild/tools/loose_version.py

@@ -81,6 +81,9 @@ def _cmp(self, other):
     def __eq__(self, other):
         return self._cmp(other) == 0
 
+    def __ne__(self, other):
+        return self._cmp(other) != 0
+
     def __lt__(self, other):
         return self._cmp(other) < 0
 

easybuild/tools/module_generator.py

@@ -49,7 +49,7 @@
 from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool
 from easybuild.tools.py2vs3 import string_type
-from easybuild.tools.utilities import get_subclasses, quote_str
+from easybuild.tools.utilities import get_subclasses, nub, quote_str
 
 
 _log = fancylogger.getLogger('module_generator', fname=False)
@@ -667,7 +667,7 @@ def _generate_help_text(self):
         if multi_deps:
             compatible_modules_txt = '\n'.join([
                 "This module is compatible with the following modules, one of each line is required:",
-            ] + ['* %s' % d for d in multi_deps])
+            ] + ['* %s' % d for d in nub(multi_deps)])
             lines.extend(self._generate_section("Compatible modules", compatible_modules_txt))
 
         # Extensions (if any)