Merge pull request #3164 from easybuilders/4.1.x

release EasyBuild v4.1.1

.github/workflows/unit_tests.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         python: [2.7, 3.5, 3.6, 3.7, 3.8]
-        modules_tool: [Lmod-7.8.22, Lmod-8.2.3, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
+        modules_tool: [Lmod-7.8.22, Lmod-8.2.9, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
         module_syntax: [Lua, Tcl]
         # exclude some configuration for non-Lmod modules tool:
         # - don't test with Lua module syntax (only supported in Lmod)
@@ -138,6 +138,9 @@ jobs:
           eb --show-system-info
           # check GitHub configuration
           eb --check-github --github-user=easybuild_test
+          # create file owned by root but writable by anyone (used by test_copy_file)
+          sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+          sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
           # run test suite
           python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
           # try and make sure output of running tests is clean (no printed messages/warnings)
@@ -150,7 +153,7 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20190922.01 778762479d77df641c247aba33c13203c8c5b8275e47f59464bab93917aaaf22"
+          EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
           # test bootstrap script (only compatible with Python 2 for now)

.travis.yml

@@ -85,6 +85,9 @@ script:
     - EB_VERBOSE=1 eb --version
     # check GitHub configuration
     - eb --check-github --github-user=easybuild_test
+    # create file owned by root but writable by anyone (used by test_copy_file)
+    - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
+    - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
     # run test suite
     - python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
     # try and make sure output of running tests is clean (no printed messages/warnings)
@@ -97,7 +100,7 @@ script:
     - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
     - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
     - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-    - EB_BOOTSTRAP_EXPECTED="20190922.01 778762479d77df641c247aba33c13203c8c5b8275e47f59464bab93917aaaf22"
+    - EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53"
     - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
     # test bootstrap script (skip when testing with Python 3 for now, since latest EasyBuild release is not compatible with Python 3 yet)
     - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap; fi

RELEASE_NOTES

@@ -3,6 +3,38 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.1.1 (January 16th 2020)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add check_log_for_errors function (in easybuild.tools.run) to detect and handle multiple errors (#3118)
+  - implement support for 'eb --show-ec' to show contents of specified easyconfig file (#3132)
+  - also update $XDG_DATA_DIR (share/) and $GI_TYPELIB_PATH environment variables (lib*/girepository-*) in generated module files (#3133)
+  - add support for --copy-ec to copy easyconfig file to specified location (#3142)
+  - mention --disable-* option in --help output for boolean options enabled by default (#3151)
+  - add --cuda-compute-capabilities configuration option (#3161)
+- various bug fixes, including:
+  - ignore imports from vsc namespace made from pkgutil.py (#3120)
+  - only actually change permissions using os.chmod in adjust_permissions if the current permissions are not correct already (#3125)
+  - use shutil.copyfile to just copy file contents if target path exists and is owned by someone else (#3127)
+  - fix or avoid warnings that commonly arise in build log (#3129)
+  - disable buffering in asyncprocess.Popen using bufsize=0, to fix run_cmd_qa missing output (#3130)
+  - update pip & install wheel package in generated Singularity container recipes (#3136)
+  - avoid crash in modify_env & unset unset_env_vars when using (older versions) of Python 3.5 & 3.6 by using list(...) (#3140)
+  - take into account that lib64 could be a symlink to lib (or vice versa) in get_software_libdir function (#3141)
+  - only parse docstring if it exists in gen_easyblock_doc_section_rst function (#3144)
+  - only add useful entries for $CPATH, $(LD_)LIBRARY_PATH and $PATH (non-empty directories) (#3145, #3152)
+  - fix --list-software=detailed when using Python 3 by leveraging sort_looseversions function from py2vs3 module (#3146)
+  - ensure subdirectories in software install directory have correct search (exec) permission (#3147)
+  - take into account that a checksum value may be a tuple of valid checksum in EasyBlock.check_checksums (#3153)
+- other changes:
+  - bump to Lmod 8.2.9 in GitHub CI config (#3115)
+  - update copyright statements for 2020 (#3149)
+  - make Hound CI code style checker ignore "Black would make changes" produced by flake8-black (#3162)
+
+
 v4.1.0 (December 4th 2019)
 --------------------------
 

easybuild/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2011-2019 Ghent University
+# Copyright 2011-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/exceptions.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2015-2018 Ghent University
+# Copyright 2015-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/fancylogger.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2018 Ghent University
+# Copyright 2011-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/generaloption.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2018 Ghent University
+# Copyright 2011-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -1088,11 +1088,15 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non
 
             if default is not None:
                 if len(str(default)) == 0:
-                    extra_help.append("def ''")  # empty string
+                    extra_help.append("default: ''")  # empty string
                 elif typ in ExtOption.TYPE_STRLIST:
-                    extra_help.append("def %s" % sep.join(default))
+                    extra_help.append("default: %s" % sep.join(default))
                 else:
-                    extra_help.append("def %s" % default)
+                    extra_help.append("default: %s" % default)
+
+                # for boolean options enabled by default, mention that they can be disabled using --disable-*
+                if default is True:
+                    extra_help.append("disable with --disable-%s" % key)
 
             if len(extra_help) > 0:
                 hlp += " (%s)" % ("; ".join(extra_help))

easybuild/base/testing.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2014-2018 Ghent University
+# Copyright 2014-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2019 Ghent University
+# Copyright 2009-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyblock.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2019 Ghent University
+# Copyright 2009-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -44,7 +44,6 @@
 import inspect
 import os
 import re
-import shutil
 import stat
 import tempfile
 import time
@@ -62,6 +61,7 @@
 from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH
 from easybuild.framework.easyconfig.tools import get_paths_for
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
+from easybuild.framework.extension import resolve_exts_filter_template
 from easybuild.tools import config, filetools
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs
@@ -71,12 +71,12 @@
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
-from easybuild.tools.filetools import adjust_permissions, apply_patch, apply_regex_substitutions, back_up_file
+from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file
 from easybuild.tools.filetools import change_dir, convert_name, compute_checksum, copy_file, derive_alt_pypi_url
 from easybuild.tools.filetools import diff_files, download_file, encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
-from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file
+from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file, dir_contains_files
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
 from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
@@ -1290,6 +1290,9 @@ def make_module_req(self):
                 note += "for paths are skipped for the statements below due to dry run"
                 lines.append(self.module_generator.comment(note))
 
+            # for these environment variables, the corresponding subdirectory must include at least one file
+            keys_requiring_files = ('CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH')
+
             for key in sorted(requirements):
                 if self.dry_run:
                     self.dry_run_msg(" $%s: %s" % (key, ', '.join(requirements[key])))
@@ -1302,6 +1305,16 @@ def make_module_req(self):
                     # only use glob if the string is non-empty
                     if path and not self.dry_run:
                         paths = sorted(glob.glob(path))
+                        if paths and key in keys_requiring_files:
+                            # only retain paths that contain at least one file
+                            retained_paths = [
+                                path for path in paths
+                                if os.path.isdir(os.path.join(self.installdir, path))
+                                and dir_contains_files(os.path.join(self.installdir, path))
+                            ]
+                            self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s",
+                                          key, paths, retained_paths)
+                            paths = retained_paths
                     else:
                         # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME)
                         paths = [path]
@@ -1317,15 +1330,18 @@ def make_module_req_guess(self):
         """
         A dictionary of possible directories to look for.
         """
+        lib_paths = ['lib', 'lib32', 'lib64']
         return {
             'PATH': ['bin', 'sbin'],
-            'LD_LIBRARY_PATH': ['lib', 'lib32', 'lib64'],
-            'LIBRARY_PATH': ['lib', 'lib32', 'lib64'],
+            'LD_LIBRARY_PATH': lib_paths,
+            'LIBRARY_PATH': lib_paths,
             'CPATH': ['include'],
             'MANPATH': ['man', os.path.join('share', 'man')],
-            'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']],
+            'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in lib_paths + ['share']],
             'ACLOCAL_PATH': [os.path.join('share', 'aclocal')],
             'CLASSPATH': ['*.jar'],
+            'XDG_DATA_DIRS': ['share'],
+            'GI_TYPELIB_PATH': [os.path.join(x, 'girepository-*') for x in lib_paths],
         }
 
     def load_module(self, mod_paths=None, purge=True, extra_modules=None):
@@ -1431,43 +1447,18 @@ def skip_extensions(self):
 
         if not exts_filter or len(exts_filter) == 0:
             raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig")
-        elif isinstance(exts_filter, string_type) or len(exts_filter) != 2:
-            raise EasyBuildError('exts_filter should be a list or tuple of ("command","input")')
-        cmdtmpl = exts_filter[0]
-        cmdinputtmpl = exts_filter[1]
 
         res = []
         for ext in self.exts:
-            name = ext['name']
-            if 'options' in ext and 'modulename' in ext['options']:
-                modname = ext['options']['modulename']
-            else:
-                modname = name
-            tmpldict = {
-                'ext_name': modname,
-                'ext_version': ext.get('version'),
-                'src': ext.get('source'),
-            }
-
-            try:
-                cmd = cmdtmpl % tmpldict
-            except KeyError as err:
-                msg = "KeyError occurred on completing extension filter template: %s; "
-                msg += "'name'/'version' keys are no longer supported, should use 'ext_name'/'ext_version' instead"
-                self.log.nosupport(msg % err, '2.0')
-
-            if cmdinputtmpl:
-                stdin = cmdinputtmpl % tmpldict
-                (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False)
-            else:
-                (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False)
+            cmd, stdin = resolve_exts_filter_template(exts_filter, ext)
+            (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False)
             self.log.info("exts_filter result %s %s", cmdstdouterr, ec)
             if ec:
-                self.log.info("Not skipping %s" % name)
+                self.log.info("Not skipping %s" % ext['name'])
                 self.log.debug("exit code: %s, stdout/err: %s" % (ec, cmdstdouterr))
                 res.append(ext)
             else:
-                self.log.info("Skipping %s" % name)
+                self.log.info("Skipping %s" % ext['name'])
         self.exts = res
 
     #
@@ -1585,8 +1576,9 @@ def post_iter_step(self):
 
     def det_iter_cnt(self):
         """Determine iteration count based on configure/build/install options that may be lists."""
-        iter_opt_counts = [len(self.cfg[opt]) for opt in ITERATE_OPTIONS
-                           if opt not in ['builddependencies'] and isinstance(self.cfg[opt], (list, tuple))]
+        # Using get_ref to avoid resolving templates as their required attributes may not be available yet
+        iter_opt_counts = [len(self.cfg.get_ref(opt)) for opt in ITERATE_OPTIONS
+                           if opt not in ['builddependencies'] and isinstance(self.cfg.get_ref(opt), (list, tuple))]
 
         # we need to take into account that builddependencies is always a list
         # we're only iterating over it if it's a list of lists
@@ -1814,7 +1806,7 @@ def checksum_step(self):
 
     def check_checksums_for(self, ent, sub='', source_cnt=None):
         """
-        Utility method: check whether checksums for all sources/patches are available, for given entity
+        Utility method: check whether SHA256 checksums for all sources/patches are available, for given entity
         """
         ec_fn = os.path.basename(self.cfg.path)
         checksum_issues = []
@@ -1841,8 +1833,19 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
             if isinstance(checksum, dict):
                 checksum = checksum.get(fn)
 
-            if not is_sha256_checksum(checksum):
-                msg = "Non-SHA256 checksum found for %s: %s" % (fn, checksum)
+            # take into account that we may encounter a tuple of valid SHA256 checksums
+            # (see https://github.com/easybuilders/easybuild-framework/pull/2958)
+            if isinstance(checksum, tuple):
+                # 1st tuple item may indicate checksum type, must be SHA256 or else it's blatently ignored here
+                if len(checksum) == 2 and checksum[0] == CHECKSUM_TYPE_SHA256:
+                    valid_checksums = (checksum[1],)
+                else:
+                    valid_checksums = checksum
+            else:
+                valid_checksums = (checksum,)
+
+            if not all(is_sha256_checksum(c) for c in valid_checksums):
+                msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums)
                 checksum_issues.append(msg)
 
         return checksum_issues
@@ -2760,19 +2763,31 @@ def permissions_step(self):
 
         # add read permissions for everybody on all files, taking into account group (if any)
         perms = stat.S_IRUSR | stat.S_IRGRP
+        # directory permissions: readable (r) & searchable (x)
+        dir_perms = stat.S_IXUSR | stat.S_IXGRP
         self.log.debug("Ensuring read permissions for user/group on install dir (recursively)")
+
         if self.group is None:
             perms |= stat.S_IROTH
+            dir_perms |= stat.S_IXOTH
             self.log.debug("Also ensuring read permissions for others on install dir (no group specified)")
 
         umask = build_option('umask')
         if umask is not None:
             # umask is specified as a string, so interpret it first as integer in octal, then take complement (~)
             perms &= ~int(umask, 8)
+            dir_perms &= ~int(umask, 8)
             self.log.debug("Taking umask '%s' into account when ensuring read permissions to install dir", umask)
 
+        self.log.debug("Adding file read permissions in %s using '%s'", self.installdir, oct(perms))
         adjust_permissions(self.installdir, perms, add=True, recursive=True, relative=True, ignore_errors=True)
-        self.log.info("Successfully added read permissions '%s' recursively on install dir", oct(perms))
+
+        # also ensure directories have exec permissions (so they can be opened)
+        self.log.debug("Adding directory search permissions in %s using '%s'", self.installdir, oct(dir_perms))
+        adjust_permissions(self.installdir, dir_perms, add=True, recursive=True, relative=True, onlydirs=True,
+                           ignore_errors=True)
+
+        self.log.info("Successfully added read permissions recursively on install dir %s", self.installdir)
 
     def test_cases_step(self):
         """

easybuild/framework/easyconfig/__init__.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2019 Ghent University
+# Copyright 2009-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/constants.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2013-2019 Ghent University
+# Copyright 2013-2020 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),