Merge branch 'master' into issues/5120-wdl-normalize-trouble

.git-blame-ignore-revs

@@ -0,0 +1,8 @@
+# make format with black 24.10.0 on 2024-10-19
+49d99f17799aa4a4235126bbdfb45f85e6499b27
+# make sort_imports
+a307a57651296e576d55f9159869dc9130da2b7c
+# make remove_unused_import
+1c6217e11a2815476ede1cc3ee584d5da3d5242a
+# make pyupgrade (3.9+)
+00f54f8417b3a0bc5d1ea46650aa2e6ce4792159

Makefile

@@ -341,7 +341,7 @@ diff_mypy:
 	diff-cover --fail-under=100 --compare-branch origin/master cobertura.xml
 
 pyupgrade: $(PYSOURCES)
-	pyupgrade --exit-zero-even-if-changed --py37-plus $^
+	pyupgrade --exit-zero-even-if-changed --py39-plus $^
 
 flake8: $(PYSOURCES)
 	flake8 --ignore=E501,W293,W291,E265,E302,E722,E126,E303,E261,E201,E202,W503,W504,W391,E128,E301,E127,E502,E129,E262,E111,E117,E306,E203,E231,E226,E741,E122,E251,E305,E701,E222,E225,E241,E305,E123,E121,E703,E704,E125,E402 $^

contrib/mypy-stubs/configargparse/__init__.pyi

@@ -1,5 +1,5 @@
+from .configargparse import SUPPRESS as SUPPRESS
 from .configargparse import ArgParser as ArgParser
 from .configargparse import ArgumentParser as ArgumentParser
 from .configargparse import Namespace as Namespace
-from .configargparse import SUPPRESS as SUPPRESS
 from .configargparse import YAMLConfigFileParser as YAMLConfigFileParser

contrib/mypy-stubs/configargparse/configargparse.pyi

@@ -23,7 +23,6 @@ class ArgumentParser(argparse.ArgumentParser):
     def _default_config_files(self) -> List[Any]: ...
     @property
     def _ignore_unknown_config_file_keys(self) -> Any: ...
-
     def __init__(self, *args: Any, **kwargs: Any) -> None: ...
     # There may be a better way of type hinting this without a type: ignore, but mypy gets unhappy pretty much no matter what as the signatures for parse_args doesn't match with its superclass in argparse
     def parse_args(self, args: Sequence[str] | None = None, namespace: Namespace | None = None, config_file_contents: str | None = None, env_vars: Any = None) -> Namespace: ...  # type: ignore[override]

contrib/mypy-stubs/dill/_dill.pyi

@@ -1,8 +1,12 @@
 import sys
-from typing import IO, Any, Callable, Iterable, Protocol, Union
+from typing import Any, Callable, Iterable, Protocol, final
 
 from _typeshed import ReadableBuffer
-from typing_extensions import TypeAlias, final
+
+if sys.version_info < (3, 10):
+    from typing_extensions import TypeAlias
+else:
+    from typing import TypeAlias
 
 class _ReadableFileobj(Protocol):
     def read(self, __n: int) -> bytes: ...
@@ -17,6 +21,7 @@ if sys.version_info >= (3, 8):
         def __init__(self, buffer: ReadableBuffer) -> None: ...
         def raw(self) -> memoryview: ...
         def release(self) -> None: ...
+
     _BufferCallback: TypeAlias = Callable[[PickleBuffer], Any] | None
 
     def dump(

requirements-dev.txt

@@ -9,7 +9,7 @@ astroid>=3,<4
 sphinx-autodoc-typehints>=1.24.0,<3
 sphinxcontrib-autoprogram==0.1.9
 cwltest>=2.2.20211116163652
-mypy==1.12.1
+mypy==1.13.0
 types-aws-xray-sdk
 types-boto<2.49.18.20241020
 types-Flask-Cors

setup.py

@@ -21,6 +21,7 @@
 SETUP_DIR = os.path.dirname(__file__)
 README = os.path.join(SETUP_DIR, "README.rst")
 
+
 def get_requirements(extra=None):
     """
     Load the requirements for the given extra.
@@ -32,7 +33,9 @@ def get_requirements(extra=None):
 
     with open(filename) as fp:
         # Parse out as one per line, dropping comments
-        return [l.split('#')[0].strip() for l in fp.readlines() if l.split('#')[0].strip()]
+        return [
+            l.split("#")[0].strip() for l in fp.readlines() if l.split("#")[0].strip()
+        ]
 
 
 def run_setup():
@@ -58,90 +61,101 @@ def run_setup():
         "google",
         "kubernetes",
         "wdl",
-        "server"
+        "server",
     ]
     for extra in non_htcondor_extras:
         extras_require[extra] = get_requirements(extra)
         all_reqs += extras_require[extra]
     # We exclude htcondor from "all" because it can't be on Mac
     extras_require['htcondor:sys_platform!="darwin"'] = get_requirements("htcondor")
-    extras_require['mesos'] = get_requirements("mesos")
+    extras_require["mesos"] = get_requirements("mesos")
     all_reqs += get_requirements("mesos")
     extras_require["all"] = all_reqs
     setup(
-        name='toil',
+        name="toil",
         version=version.distVersion,
         long_description=open(README).read(),
         long_description_content_type="text/x-rst",
-        description='Pipeline management software for clusters.',
-        author='Benedict Paten and the Toil community',
-        author_email='[email protected]',
+        description="Pipeline management software for clusters.",
+        author="Benedict Paten and the Toil community",
+        author_email="[email protected]",
         url="https://github.com/DataBiosphere/toil",
         classifiers=[
-          'Development Status :: 5 - Production/Stable',
-          'Environment :: Console',
-          'Intended Audience :: Developers',
-          'Intended Audience :: Science/Research',
-          'Intended Audience :: Healthcare Industry',
-          'License :: OSI Approved :: Apache Software License',
-          'Natural Language :: English',
-          'Operating System :: MacOS :: MacOS X',
-          'Operating System :: POSIX',
-          'Operating System :: POSIX :: Linux',
-          'Programming Language :: Python :: 3.9',
-          'Programming Language :: Python :: 3.10',
-          'Programming Language :: Python :: 3.11',
-          'Programming Language :: Python :: 3.12',
-          'Topic :: Scientific/Engineering',
-          'Topic :: Scientific/Engineering :: Bio-Informatics',
-          'Topic :: Scientific/Engineering :: Astronomy',
-          'Topic :: Scientific/Engineering :: Atmospheric Science',
-          'Topic :: Scientific/Engineering :: Information Analysis',
-          'Topic :: Scientific/Engineering :: Medical Science Apps.',
-          'Topic :: System :: Distributed Computing',
-          'Topic :: Utilities'],
+            "Development Status :: 5 - Production/Stable",
+            "Environment :: Console",
+            "Intended Audience :: Developers",
+            "Intended Audience :: Science/Research",
+            "Intended Audience :: Healthcare Industry",
+            "License :: OSI Approved :: Apache Software License",
+            "Natural Language :: English",
+            "Operating System :: MacOS :: MacOS X",
+            "Operating System :: POSIX",
+            "Operating System :: POSIX :: Linux",
+            "Programming Language :: Python :: 3.9",
+            "Programming Language :: Python :: 3.10",
+            "Programming Language :: Python :: 3.11",
+            "Programming Language :: Python :: 3.12",
+            "Topic :: Scientific/Engineering",
+            "Topic :: Scientific/Engineering :: Bio-Informatics",
+            "Topic :: Scientific/Engineering :: Astronomy",
+            "Topic :: Scientific/Engineering :: Atmospheric Science",
+            "Topic :: Scientific/Engineering :: Information Analysis",
+            "Topic :: Scientific/Engineering :: Medical Science Apps.",
+            "Topic :: System :: Distributed Computing",
+            "Topic :: Utilities",
+        ],
         license="Apache License v2.0",
         python_requires=">=3.9",
         install_requires=install_requires,
         extras_require=extras_require,
-        package_dir={'': 'src'},
-        packages=find_packages(where='src'),
+        package_dir={"": "src"},
+        packages=find_packages(where="src"),
         package_data={
-            '': ['*.yml', '*.yaml', 'cloud-config', '*.cwl'],
+            "": ["*.yml", "*.yaml", "cloud-config", "*.cwl"],
         },
         # Unfortunately, the names of the entry points are hard-coded elsewhere in the code base so
         # you can't just change them here. Luckily, most of them are pretty unique strings, and thus
         # easy to search for.
         entry_points={
-            'console_scripts': [
-                'toil = toil.utils.toilMain:main',
-                '_toil_worker = toil.worker:main',
-                'cwltoil = toil.cwl.cwltoil:cwltoil_was_removed [cwl]',
-                'toil-cwl-runner = toil.cwl.cwltoil:main [cwl]',
-                'toil-wdl-runner = toil.wdl.wdltoil:main [wdl]',
-                'toil-wes-cwl-runner = toil.server.cli.wes_cwl_runner:main [server]',
-                '_toil_mesos_executor = toil.batchSystems.mesos.executor:main [mesos]',
-                '_toil_contained_executor = toil.batchSystems.contained_executor:executor']})
+            "console_scripts": [
+                "toil = toil.utils.toilMain:main",
+                "_toil_worker = toil.worker:main",
+                "cwltoil = toil.cwl.cwltoil:cwltoil_was_removed [cwl]",
+                "toil-cwl-runner = toil.cwl.cwltoil:main [cwl]",
+                "toil-wdl-runner = toil.wdl.wdltoil:main [wdl]",
+                "toil-wes-cwl-runner = toil.server.cli.wes_cwl_runner:main [server]",
+                "_toil_mesos_executor = toil.batchSystems.mesos.executor:main [mesos]",
+                "_toil_contained_executor = toil.batchSystems.contained_executor:executor",
+            ]
+        },
+    )
 
 
 def import_version():
     """Return the module object for src/toil/version.py, generate from the template if required."""
-    if not os.path.exists('src/toil/version.py'):
+    if not os.path.exists("src/toil/version.py"):
         for req in get_requirements("cwl"):
             # Determine cwltool version from requirements file
             if req.startswith("cwltool=="):
-                cwltool_version = req[len("cwltool=="):]
+                cwltool_version = req[len("cwltool==") :]
                 break
         # Use the template to generate src/toil/version.py
         import version_template
-        with NamedTemporaryFile(mode='w', dir='src/toil', prefix='version.py.', delete=False) as f:
-            f.write(version_template.expand_(others={
-                # expose the dependency versions that we may need to access in Toil
-                'cwltool_version': cwltool_version,
-            }))
-        os.rename(f.name, 'src/toil/version.py')
-
-    loader = SourceFileLoader('toil.version', 'src/toil/version.py')
+
+        with NamedTemporaryFile(
+            mode="w", dir="src/toil", prefix="version.py.", delete=False
+        ) as f:
+            f.write(
+                version_template.expand_(
+                    others={
+                        # expose the dependency versions that we may need to access in Toil
+                        "cwltool_version": cwltool_version,
+                    }
+                )
+            )
+        os.rename(f.name, "src/toil/version.py")
+
+    loader = SourceFileLoader("toil.version", "src/toil/version.py")
     mod = types.ModuleType(loader.name)
     loader.exec_module(mod)
     return mod