Merge remote-tracking branch 'upstream/master' into fix/source_candid…

…ate_https

.appveyor.yml

@@ -80,7 +80,7 @@ environment:
       INSTALL_SYSPKGS: python3-virtualenv
       CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
       # system git-annex is way too old, use better one
-      INSTALL_GITANNEX: git-annex -m snapshot
+      INSTALL_GITANNEX: git-annex -m datalad/packages
     # Windows core tests
     - ID: WinP39core
       # ~35 min
@@ -98,7 +98,7 @@ environment:
       DTS: datalad.core datalad.dataset datalad.runner datalad.support
       APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
       PY: 3.9
-      # does not give a functional installation
+      # does not give a functional installation - needs env/PATH adjustment
       # INSTALL_GITANNEX: git-annex -m snapshot
       #INSTALL_GITANNEX: git-annex=8.20201129
       INSTALL_GITANNEX: git-annex -m datalad/packages
@@ -121,7 +121,7 @@ environment:
       INSTALL_SYSPKGS: python3-virtualenv
       CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
       # system git-annex is way too old, use better one
-      INSTALL_GITANNEX: git-annex -m snapshot
+      INSTALL_GITANNEX: git-annex -m datalad/packages
     - ID: WinP39a1
       # ~40min
       DTS: >
@@ -184,7 +184,7 @@ environment:
       INSTALL_SYSPKGS: python3-virtualenv
       CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
       # system git-annex is way too old, use better one
-      INSTALL_GITANNEX: git-annex -m snapshot
+      INSTALL_GITANNEX: git-annex -m datalad/packages
     - ID: Ubu22P311b
       # ~25min
       PY: 3.11
@@ -200,7 +200,7 @@ environment:
       INSTALL_SYSPKGS: python3-virtualenv
       CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
       # system git-annex is way too old, use better one
-      INSTALL_GITANNEX: git-annex -m snapshot
+      INSTALL_GITANNEX: git-annex -m datalad/packages
 
 matrix:
   allow_failures:
@@ -308,13 +308,16 @@ install:
       )
   - sh: python -m pip install datalad-installer${DATALAD_INSTALLER_VERSION:-}
   # Missing system software
-  - sh: "[ -n \"$INSTALL_SYSPKGS\" ] && ( [ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacOS\" ] && brew install -q ${INSTALL_SYSPKGS} ||  { sudo apt-get update -y && sudo apt-get install --no-install-recommends -y ${INSTALL_SYSPKGS}; } ) || true"
+  - sh: "[ -n \"$INSTALL_SYSPKGS\" ] && ( [ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacos-monterey\" ] && brew install -q ${INSTALL_SYSPKGS} ||  { sudo apt-get update -y && sudo apt-get install --no-install-recommends -y ${INSTALL_SYSPKGS}; } ) || true"
   # Install git-annex on windows, otherwise INSTALL_SYSPKGS can be used
   # deploy git-annex, if desired
   - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX%
   - sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer --sudo ok ${INSTALL_GITANNEX}"
   # TODO remove when datalad-installer can handle this
   - cmd: tools\ci\appveyor_install_git-annex.bat
+  # "package" installation does not place git-annex into the PATH, and for
+  # compatibility etc, we just symlink also git itself
+  - sh: "[ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacos-monterey\" ] && ln -sf /Applications/git-annex.app/Contents/MacOS/git* /usr/local/bin/ || true"
 
 
 #before_build:

.github/workflows/test-label.yml

@@ -35,5 +35,37 @@ jobs:
             || contains(github.event.pull_request.labels.*.name, 'semver-performance')
             || contains(github.event.pull_request.labels.*.name, 'semver-tests')
           }}
+  check-sync:
+    name: Check that "master" contains everything from "maint" if PR against "master" is marked for a release
+    runs-on: ubuntu-latest
+    if: github.repository == 'datalad/datalad'
+    steps:
+      - name: 'Checkout Code'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Fetch all history for all branches to perform the comparison
+
+      - name: 'Check for "release" Label'
+        id: check_label
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const prLabels = context.payload.pull_request.labels.map(label => label.name);
+            core.setOutput('has_release_label', prLabels.includes('release'));
+
+      - name: 'Check Branch Sync'
+        if: steps.check_label.outputs.has_release_label == 'true' && github.event.pull_request.base.ref == 'master'
+        run: |
+          # Fetch the "maint" branch
+          git fetch origin maint:maint
+          git fetch origin master:master
+ 
+          # Check if "maint" is already merged into "master"
+          if git merge-base --is-ancestor maint master; then
+            echo "All commits from 'maint' are already merged into 'master'."
+          else
+            echo "The 'master' branch is missing commits from 'maint'. Please merge the branches before proceeding."
+            exit 1
+          fi
 
 # vim:set sts=2:

.github/workflows/test.yml

@@ -238,7 +238,7 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: github.event_name != 'schedule'
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           directory: __testhome__
           fail_ci_if_error: false

.github/workflows/test_extensions.yml

@@ -111,7 +111,7 @@ jobs:
         python -m coverage xml
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v5
       with:
         directory: __testhome__
         fail_ci_if_error: false

.github/workflows/test_macos.yml

@@ -58,7 +58,7 @@ jobs:
         python -m pytest -c ../tox.ini -s -v --cov=datalad --cov-report=xml --pyargs datalad.core datalad.support
 
     - name: Upload coverage report to Codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v5
       with:
         file: __testhome__/coverage.xml
         token: ${{ secrets.CODECOV_TOKEN }}

.tributors

@@ -99,7 +99,8 @@
         "name": "Debanjum Singh Solanky",
         "blog": "https://github.com/debanjum",
         "orcid": "0000-0002-0774-6564",
-        "affiliation": "Dartmouth College, Hanover, NH, United States"
+        "affiliation": "Dartmouth College, Hanover, NH, United States",
+        "bio": "Building open, personal AI"
     },
     "nellh": {
         "name": "nellh",
@@ -242,7 +243,8 @@
         "name": "Matt McCormick",
         "blog": "https://www.mmmccormick.com",
         "orcid": "0000-0001-9475-3756",
-        "affiliation": "Kitware Inc"
+        "affiliation": "Kitware Inc",
+        "email": "[email protected]"
     },
     "tstoeter": {
         "name": "tstoeter",

CHANGELOG.md

@@ -1,4 +1,15 @@
 
+<a id='changelog-1.1.5'></a>
+# 1.1.5 (2024-12-15)
+
+## 🧪 Tests
+
+- test: xfail test_subsuperdataset_save on newer gits.  [PR #7687](https://github.com/datalad/datalad/pull/7687) (by [@yarikoptic](https://github.com/yarikoptic))
+
+- test: refactor test_parallel.py to be abit more pytest'y.  [PR #7690](https://github.com/datalad/datalad/pull/7690) (by [@yarikoptic](https://github.com/yarikoptic))
+
+- BF: use datalad/packages method of installing git-annex.  [PR #7692](https://github.com/datalad/datalad/pull/7692) (by [@yarikoptic](https://github.com/yarikoptic))
+
 <a id='changelog-1.1.4'></a>
 # 1.1.4 (2024-11-18)
 

CITATION.cff

@@ -79,4 +79,4 @@ keywords:
 title: >-
   DataLad: distributed system for joint management of code, data, and their
   relationship
-version: 1.1.4
+version: 1.1.5

changelog.d/pr-7590.md

@@ -0,0 +1,3 @@
+### 🏠 Internal
+
+- Make github workflow which would ensure that a "release" PR against `master` would have everything from `maint` merged.  [PR #7590](https://github.com/datalad/datalad/pull/7590) (by [@yarikoptic](https://github.com/yarikoptic))

changelog.d/pr-7656.md

@@ -0,0 +1,3 @@
+### 🚀 Enhancements and New Features
+
+- Remove import of GIT_SSH_COMMAND within .consts (import directly from datalad.runner.gitrunner).  [PR #7656](https://github.com/datalad/datalad/pull/7656) (by [@yarikoptic](https://github.com/yarikoptic))

datalad/consts.py

@@ -48,9 +48,6 @@
 # Format to use for time stamps
 TIMESTAMP_FMT = "%Y-%m-%dT%H:%M:%S%z"
 
-# in order to avoid breakage, import runner-related const
-from datalad.runner.gitrunner import GIT_SSH_COMMAND
-
 # magic sha is from `git hash-object -t tree /dev/null`, i.e. from nothing
 PRE_INIT_COMMIT_SHA = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'
 

datalad/core/local/tests/test_save.py

@@ -24,6 +24,7 @@
 from datalad.distribution.dataset import Dataset
 from datalad.support.annexrepo import AnnexRepo
 from datalad.support.exceptions import CommandError
+from datalad.support.external_versions import external_versions
 from datalad.tests.utils_pytest import (
     DEFAULT_BRANCH,
     OBSCURE_FILENAME,
@@ -245,6 +246,8 @@ def test_subdataset_save(path=None):
     assert len(submodules) == 1
 
 
+@pytest.mark.xfail(external_versions['cmd:git'] > '2.45.2',
+                   reason="https://github.com/datalad/datalad/issues/7681")
 @with_tempfile(mkdir=True)
 def test_subsuperdataset_save(path=None):
     # Verify that when invoked without recursion save does not

datalad/support/exceptions.py

@@ -11,6 +11,7 @@
 
 import logging
 import re
+import sys
 import traceback
 from os import linesep
 from pathlib import Path
@@ -112,15 +113,26 @@ def message(self):
         """
         return str(self.tb)
 
-    @property
-    def name(self):
-        """Returns the class name of the original exception
+    if sys.version_info < (3, 13):
+        @property
+        def name(self):
+            """Returns the class name of the original exception
 
-        Returns
-        -------
-        str
-        """
-        return self.tb.exc_type.__qualname__
+            Returns
+            -------
+            str
+            """
+            return self.tb.exc_type.__qualname__
+    else:
+        @property
+        def name(self):
+            """Returns the class name of the original exception
+
+            Returns
+            -------
+            str
+            """
+            return self.tb.exc_type_str
 
     def __str__(self):
         return self.format_short()
@@ -210,7 +222,8 @@ class name of the exception, with the aim to generate a simple and
     messages.
     """
     s = str(e) or \
-        (e.exc_type.__name__ if isinstance(e, traceback.TracebackException)
+        ((e.exc_type.__name__ if sys.version_info < (3, 13) else e.exc_type_str)
+         if isinstance(e, traceback.TracebackException)
          else e.__class__.__name__)
     exc_cause = getattr(e, '__cause__', None)
     if exc_cause:

datalad/support/parallel.py

@@ -100,17 +100,16 @@ class ProducerConsumer:
 
     Examples
     --------
-    A simple and somewhat boring example could be to count lines in '*.py'
-    files in parallel
-
-        from glob import glob
-        from pprint import pprint
-        from datalad.support.parallel import ProducerConsumer
-
-        def count_lines(fname):
-            with open(fname) as f:
-                return fname, len(f.readlines())
-        pprint(dict(ProducerConsumer(glob("*.py"), count_lines)))
+    A simple and somewhat boring example to count lines in '*.py'
+
+    >>> from glob import glob
+    >>> from pprint import pprint
+    >>> from datalad.support.parallel import ProducerConsumer
+    >>> def count_lines(fname):
+    ...     with open(fname) as f:
+    ...         return fname, len(f.readlines())
+    >>> pprint(dict(ProducerConsumer(glob("*.py"), count_lines)))  # doctest: +SKIP
+    {'setup.py': 182, 'versioneer.py': 2136}
 
     More usage examples could be found in `test_parallel.py` and around the
     codebase `addurls.py`, `get.py`, `save.py`, etc.

datalad/support/tests/test_captured_exception.py

@@ -1,3 +1,4 @@
+import sys
 from unittest.mock import patch
 
 from datalad import cfg
@@ -72,15 +73,16 @@ def f2():
     assert_equal(full_display[0], "Traceback (most recent call last):")
     # points in f and f2 for first exception with two lines each
     # (where is the line and what reads the line):
-    assert_true(full_display[1].lstrip().startswith("File"))
-    assert_equal(full_display[2].strip(), "f2()")
-    assert_true(full_display[3].lstrip().startswith("File"))
-    assert_equal(full_display[4].strip(), "raise Exception(\"my bad again\")")
-    assert_equal(full_display[5].strip(), "Exception: my bad again")
-    assert_equal(full_display[7].strip(), "The above exception was the direct cause of the following exception:")
-    assert_equal(full_display[9], "Traceback (most recent call last):")
+    assert full_display[1].lstrip().startswith("File")
+    assert full_display[2].strip() == "f2()"
+    inc = int(sys.version_info >= (3, 13))
+    assert full_display[3 + inc].lstrip().startswith("File")
+    assert full_display[4 + inc].strip() == "raise Exception(\"my bad again\")"
+    assert full_display[5 + inc].strip() == "Exception: my bad again"
+    assert full_display[7 + inc].strip() == "The above exception was the direct cause of the following exception:"
+    assert full_display[9 + inc] == "Traceback (most recent call last):"
     # ...
-    assert_equal(full_display[-1].strip(), "RuntimeError: new message")
+    assert full_display[-1].strip() == "RuntimeError: new message"
 
     # CapturedException.__repr__:
     assert_re_in(r".*test_captured_exception.py:f2:[0-9]+\]$",

datalad/support/tests/test_parallel.py

@@ -33,19 +33,23 @@
     assert_greater_equal,
     assert_raises,
     assert_repo_status,
-    known_failure_osx,
-    on_osx,
-    on_windows,
     rmtree,
-    skip_if,
     slow,
     with_tempfile,
 )
 
 info_log_level = lgr.getEffectiveLevel() >= logging.INFO
 
 
-def check_ProducerConsumer(PC, jobs):
+@pytest.fixture(params=["auto", None, 1, 10])
+def jobs(request):
+    """Fixture to automagically sweep over a sample of "jobs" values
+    """
+    return request.param
+
+
+@pytest.mark.parametrize("PC", [ProducerConsumer, ProducerConsumerProgressLog])
+def test_ProducerConsumer_PC(PC, jobs):
     def slowprod(n, secs=0.001):
         for i in range(n):
             yield i
@@ -74,7 +78,7 @@ def fastcons(i):
             [{"i": i, "status": "ok" if i % 2 else "error"} for i in range(10)])
 
 
-def check_producing_consumer(jobs):
+def test_producing_consumer(jobs):
     def producer():
         yield from range(3)
     def consumer(i):
@@ -84,10 +88,10 @@ def consumer(i):
 
     # we auto-detect generator function producer
     pc = ProducerConsumer(producer, consumer, jobs=jobs)
-    assert_equal(list(pc), [0, 1, 2, "0", "1", "4"])
+    assert_equal(set(pc), {0, 1, 2, "0", "1", "4"})
 
 
-def check_producer_future_key(jobs):
+def test_producer_future_key(jobs):
     def producer():
         for i in range(3):
             yield i, {"k": i**2}  # dict is mutable, will need a key
@@ -100,15 +104,6 @@ def consumer(args):
     assert_equal(list(pc), [0, 1, 2])
 
 
-def test_ProducerConsumer():
-        # Largely a smoke test, which only verifies correct results output
-    for jobs in "auto", None, 1, 10:
-        for PC in ProducerConsumer, ProducerConsumerProgressLog:
-            check_ProducerConsumer(PC, jobs)
-        check_producing_consumer(jobs)
-        check_producer_future_key(jobs)
-
-
 @slow  # 12sec on Yarik's laptop
 @with_tempfile(mkdir=True)
 def test_creatsubdatasets(topds_path=None, n=2):