Merge branch 'main' into fix-c_powi-117999

.github/workflows/build.yml

@@ -8,19 +8,11 @@ on:
   push:
     branches:
     - 'main'
-    - '3.12'
-    - '3.11'
-    - '3.10'
-    - '3.9'
-    - '3.8'
+    - '3.*'
   pull_request:
     branches:
     - 'main'
-    - '3.12'
-    - '3.11'
-    - '3.10'
-    - '3.9'
-    - '3.8'
+    - '3.*'
 
 permissions:
   contents: read
@@ -218,7 +210,7 @@ jobs:
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       free-threading: true
-      # macos-14 is M1
+      # macos-14-large is Intel with 12 cores (most parallelism)
       os-matrix: '["macos-14"]'
 
   build_ubuntu:
@@ -494,6 +486,7 @@ jobs:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       options: ./configure --config-cache --with-thread-sanitizer --with-pydebug
       suppressions_path: Tools/tsan/supressions.txt
+      tsan_logs_artifact_name: tsan-logs-default
 
   build_tsan_free_threading:
     name: 'Thread sanitizer (free-threading)'
@@ -504,6 +497,7 @@ jobs:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug
       suppressions_path: Tools/tsan/suppressions_free_threading.txt
+      tsan_logs_artifact_name: tsan-logs-free-threading
 
   # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
   cifuzz:

.github/workflows/jit.yml

@@ -23,7 +23,7 @@ jobs:
   jit:
     name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
     runs-on: ${{ matrix.runner }}
-    timeout-minutes: 75
+    timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:

.github/workflows/posix-deps-apt.sh

@@ -15,6 +15,7 @@ apt-get -yq install \
     libgdbm-dev \
     libgdbm-compat-dev \
     liblzma-dev \
+    libmpdec-dev \
     libncurses5-dev \
     libreadline6-dev \
     libsqlite3-dev \

.github/workflows/reusable-macos.yml

@@ -50,7 +50,7 @@ jobs:
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix [email protected])"
     - name: Build CPython
-      run: make -j4
+      run: make -j8
     - name: Display build info
       run: make pythoninfo
     - name: Tests

.github/workflows/reusable-tsan.yml

@@ -11,6 +11,10 @@ on:
         description: 'A repo relative path to the suppressions file'
         required: true
         type: string
+      tsan_logs_artifact_name:
+        description: 'Name of the TSAN logs artifact. Must be unique for each job.'
+        required: true
+        type: string
 
 jobs:
   build_tsan_reusable:
@@ -29,12 +33,19 @@ jobs:
     - name: Install Dependencies
       run: |
         sudo ./.github/workflows/posix-deps-apt.sh
-        sudo apt install -y clang
+        # Install clang-18
+        wget https://apt.llvm.org/llvm.sh
+        chmod +x llvm.sh
+        sudo ./llvm.sh 18
+        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100
+        sudo update-alternatives --set clang /usr/bin/clang-18
+        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100
+        sudo update-alternatives --set clang++ /usr/bin/clang++-18
         # Reduce ASLR to avoid TSAN crashing
         sudo sysctl -w vm.mmap_rnd_bits=28
     - name: TSAN Option Setup
       run: |
-        echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }}" >> $GITHUB_ENV
+        echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> $GITHUB_ENV
         echo "CC=clang" >> $GITHUB_ENV
         echo "CXX=clang++" >> $GITHUB_ENV
     - name: Add ccache to PATH
@@ -53,3 +64,13 @@ jobs:
       run: make pythoninfo
     - name: Tests
       run: ./python -m test --tsan -j4
+    - name: Display TSAN logs
+      if: always()
+      run: find ${GITHUB_WORKSPACE} -name 'tsan_log.*' | xargs head -n 1000
+    - name: Archive TSAN logs
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.tsan_logs_artifact_name }}
+        path: tsan_log.*
+        if-no-files-found: ignore

Doc/c-api/init_config.rst

@@ -1250,10 +1250,10 @@ PyConfig
       If non-zero, initialize the perf trampoline. See :ref:`perf_profiling`
       for more information.
 
-      Set by :option:`-X perf <-X>` command line option and by the
-      :envvar:`PYTHONPERFSUPPORT` environment variable for perf support
-      with stack pointers and :option:`-X perfjit <-X>` command line option
-      and by the :envvar:`PYTHONPERFJITSUPPORT` environment variable for perf
+      Set by :option:`-X perf <-X>` command-line option and by the
+      :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support
+      with stack pointers and :option:`-X perf_jit <-X>` command-line option
+      and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf
       support with DWARF JIT information.
 
       Default: ``-1``.

Doc/c-api/module.rst

@@ -634,7 +634,7 @@ state:
 
    .. versionadded:: 3.9
 
-.. c:function:: int PyModule_ExperimentalSetGIL(PyObject *module, void *gil)
+.. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil)
 
    Indicate that *module* does or does not support running without the global
    interpreter lock (GIL), using one of the values from

Doc/c-api/object.rst

@@ -65,7 +65,7 @@ Object Protocol
 
    Properly handle returning :c:data:`Py_NotImplemented` from within a C
    function (that is, create a new :term:`strong reference`
-   to NotImplemented and return it).
+   to :const:`NotImplemented` and return it).
 
 
 .. c:macro:: Py_PRINT_RAW

Doc/conf.py

@@ -435,6 +435,10 @@
 epub_author = 'Python Documentation Authors'
 epub_publisher = 'Python Software Foundation'
 
+# index pages are not valid xhtml
+# https://github.com/sphinx-doc/sphinx/issues/12359
+epub_use_index = False
+
 # Options for the coverage checker
 # --------------------------------
 

Doc/howto/perf_profiling.rst

@@ -162,12 +162,12 @@ the :option:`!-X` option takes precedence over the environment variable.
 
 Example, using the environment variable::
 
-   $ PYTHONPERFSUPPORT=1 python script.py
+   $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python script.py
    $ perf report -g -i perf.data
 
 Example, using the :option:`!-X` option::
 
-   $ python -X perf script.py
+   $ perf record -F 9999 -g -o perf.data python -X perf script.py
    $ perf report -g -i perf.data
 
 Example, using the :mod:`sys` APIs in file :file:`example.py`:
@@ -184,7 +184,7 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`:
 
 ...then::
 
-   $ python ./example.py
+   $ perf record -F 9999 -g -o perf.data python ./example.py
    $ perf report -g -i perf.data
 
 
@@ -210,31 +210,57 @@ of ``perf``.
 How to work without frame pointers
 ----------------------------------
 
-If you are working with a Python interpreter that has been compiled without frame pointers
-you can still use the ``perf`` profiler but the overhead will be a bit higher because Python
-needs to generate unwinding information for every Python function call on the fly. Additionally,
-``perf`` will take more time to process the data because it will need to use the DWARF debugging
-information to unwind the stack and this is a slow process.
+If you are working with a Python interpreter that has been compiled without
+frame pointers, you can still use the ``perf`` profiler, but the overhead will be
+a bit higher because Python needs to generate unwinding information for every
+Python function call on the fly. Additionally, ``perf`` will take more time to
+process the data because it will need to use the DWARF debugging information to
+unwind the stack and this is a slow process.
 
-To enable this mode, you can use the environment variable :envvar:`PYTHONPERFJITSUPPORT` or the
-:option:`-X perfjit <-X>` option, which will enable the JIT mode for the ``perf`` profiler.
+To enable this mode, you can use the environment variable
+:envvar:`PYTHON_PERF_JIT_SUPPORT` or the :option:`-X perf_jit <-X>` option,
+which will enable the JIT mode for the ``perf`` profiler.
 
-When using the perf JIT mode, you need an extra step before you can run ``perf report``. You need to
-call the ``perf inject`` command to inject the JIT information into the ``perf.data`` file.
+.. note::
 
-    $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperfjit my_script.py
-    $ perf inject -i perf.data --jit
-    $ perf report -g -i perf.data
+    Due to a bug in the ``perf`` tool, only ``perf`` versions higher than v6.8
+    will work with the JIT mode.  The fix was also backported to the v6.7.2
+    version of the tool.
 
-or using the environment variable::
+    Note that when checking the version of the ``perf`` tool (which can be done
+    by running ``perf version``) you must take into account that some distros
+    add some custom version numbers including a ``-`` character.  This means
+    that ``perf 6.7-3`` is not necessarily ``perf 6.7.3``.
+
+When using the perf JIT mode, you need an extra step before you can run ``perf
+report``. You need to call the ``perf inject`` command to inject the JIT
+information into the ``perf.data`` file.::
 
-    $ PYTHONPERFJITSUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py
-    $ perf inject -i perf.data --jit
-    $ perf report -g -i perf.data
+    $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py
+    $ perf inject -i perf.data --jit --output perf.jit.data
+    $ perf report -g -i perf.jit.data
+
+or using the environment variable::
 
-Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take snapshots of the stack of
-the process being profiled and save the information in the ``perf.data`` file. By default the size of
-the stack dump is 8192 bytes but the user can change the size by passing the size after comma like
-``--call-graph dwarf,4096``. The size of the stack dump is important because if the size is too small
-``perf`` will not be able to unwind the stack and the output will be incomplete.
+    $ PYTHON_PERF_JIT_SUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py
+    $ perf inject -i perf.data --jit --output perf.jit.data
+    $ perf report -g -i perf.jit.data
+
+``perf inject --jit`` command will read ``perf.data``,
+automatically pick up the perf dump file that Python creates (in
+``/tmp/perf-$PID.dump``), and then create ``perf.jit.data`` which merges all the
+JIT information together. It should also create a lot of ``jitted-XXXX-N.so``
+files in the current directory which are ELF images for all the JIT trampolines
+that were created by Python.
+
+.. warning::
+    Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take
+    snapshots of the stack of the process being profiled and save the
+    information in the ``perf.data`` file. By default the size of the stack dump
+    is 8192 bytes but the user can change the size by passing the size after
+    comma like ``--call-graph dwarf,4096``. The size of the stack dump is
+    important because if the size is too small ``perf`` will not be able to
+    unwind the stack and the output will be incomplete. On the other hand, if
+    the size is too big, then ``perf`` won't be able to sample the process as
+    frequently as it would like as the overhead will be higher.
 

Doc/howto/urllib2.rst

@@ -594,5 +594,5 @@ This document was reviewed and revised by John Lee.
        scripts with a localhost server, I have to prevent urllib from using
        the proxy.
 .. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe
-       <https://code.activestate.com/recipes/456195/>`_.
+       <https://code.activestate.com/recipes/456195-urrlib2-opener-for-ssl-proxy-connect-method/>`_.
 

Doc/library/__future__.rst

@@ -1,5 +1,5 @@
-:mod:`__future__` --- Future statement definitions
-==================================================
+:mod:`!__future__` --- Future statement definitions
+===================================================
 
 .. module:: __future__
    :synopsis: Future statement definitions

Doc/library/__main__.rst

@@ -1,5 +1,5 @@
-:mod:`__main__` --- Top-level code environment
-==============================================
+:mod:`!__main__` --- Top-level code environment
+===============================================
 
 .. module:: __main__
    :synopsis: The environment where top-level code is run. Covers command-line

Doc/library/_thread.rst

@@ -1,5 +1,5 @@
-:mod:`_thread` --- Low-level threading API
-==========================================
+:mod:`!_thread` --- Low-level threading API
+===========================================
 
 .. module:: _thread
    :synopsis: Low-level threading API.
@@ -169,14 +169,14 @@ Lock objects have the following methods:
    time can acquire a lock --- that's their reason for existence).
 
    If the *blocking* argument is present, the action depends on its
-   value: if it is False, the lock is only acquired if it can be acquired
-   immediately without waiting, while if it is True, the lock is acquired
+   value: if it is false, the lock is only acquired if it can be acquired
+   immediately without waiting, while if it is true, the lock is acquired
    unconditionally as above.
 
    If the floating-point *timeout* argument is present and positive, it
    specifies the maximum wait time in seconds before returning.  A negative
    *timeout* argument specifies an unbounded wait.  You cannot specify
-   a *timeout* if *blocking* is False.
+   a *timeout* if *blocking* is false.
 
    The return value is ``True`` if the lock is acquired successfully,
    ``False`` if not.

Doc/library/abc.rst

@@ -1,5 +1,5 @@
-:mod:`abc` --- Abstract Base Classes
-====================================
+:mod:`!abc` --- Abstract Base Classes
+=====================================
 
 .. module:: abc
    :synopsis: Abstract base classes according to :pep:`3119`.

Doc/library/allos.rst

@@ -16,7 +16,6 @@ but they are available on most other systems as well.  Here's an overview:
    io.rst
    time.rst
    argparse.rst
-   getopt.rst
    logging.rst
    logging.config.rst
    logging.handlers.rst

Doc/library/argparse.rst

@@ -1,5 +1,5 @@
-:mod:`argparse` --- Parser for command-line options, arguments and sub-commands
-===============================================================================
+:mod:`!argparse` --- Parser for command-line options, arguments and sub-commands
+================================================================================
 
 .. module:: argparse
    :synopsis: Command-line option and argument parsing library.

Doc/library/array.rst

@@ -1,5 +1,5 @@
-:mod:`array` --- Efficient arrays of numeric values
-===================================================
+:mod:`!array` --- Efficient arrays of numeric values
+====================================================
 
 .. module:: array
    :synopsis: Space efficient arrays of uniformly typed numeric values.