Merge branch 'main' into re_improvements

.editorconfig

@@ -1,6 +1,6 @@
 root = true
 
-[*.{py,c,cpp,h,rst,md,yml}]
+[*.{py,c,cpp,h,js,rst,md,yml}]
 trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = space
@@ -11,5 +11,5 @@ indent_size = 4
 [*.rst]
 indent_size = 3
 
-[*.yml]
+[*.{js,yml}]
 indent_size = 2

.gitattributes

@@ -77,6 +77,7 @@ Include/internal/pycore_opcode.h                    generated
 Include/internal/pycore_opcode_metadata.h           generated
 Include/internal/pycore_*_generated.h               generated
 Include/internal/pycore_uop_ids.h                   generated
+Include/internal/pycore_uop_metadata.h              generated
 Include/opcode.h                                    generated
 Include/opcode_ids.h                                generated
 Include/token.h                                     generated
@@ -94,6 +95,7 @@ Programs/test_frozenmain.h                          generated
 Python/Python-ast.c                                 generated
 Python/executor_cases.c.h                           generated
 Python/generated_cases.c.h                          generated
+Python/tier2_redundancy_eliminator_cases.c.h        generated
 Python/opcode_targets.h                             generated
 Python/stdlib_module_names.h                        generated
 Tools/peg_generator/pegen/grammar_parser.py         generated

.github/CODEOWNERS

@@ -37,6 +37,8 @@ Python/flowgraph.c            @markshannon @iritkatriel
 Python/ast_opt.c              @isidentical
 Python/bytecodes.c            @markshannon @gvanrossum
 Python/optimizer*.c           @markshannon @gvanrossum
+Python/optimizer_analysis.c   @Fidget-Spinner
+Python/tier2_redundancy_eliminator_bytecodes.c  @Fidget-Spinner
 Lib/test/test_patma.py        @brandtbucher
 Lib/test/test_type_*.py       @JelleZijlstra
 Lib/test/test_capi/test_misc.py  @markshannon @gvanrossum

.github/workflows/build.yml

@@ -131,11 +131,13 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
+      - name: Runner image version
+        run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
       - name: Restore config.cache
         uses: actions/cache@v4
         with:
           path: config.cache
-          key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
+          key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Add ccache to PATH
@@ -160,7 +162,7 @@ jobs:
       - name: Build CPython
         run: |
           make -j4 regen-all
-          make regen-stdlib-module-names
+          make regen-stdlib-module-names regen-sbom
       - name: Check for changes
         run: |
           git add -u
@@ -250,19 +252,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1w, 3.0.11, 3.1.3]
+        openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
       OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -304,7 +308,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
     env:
-      OPENSSL_VER: 3.0.11
+      OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -341,11 +345,13 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -415,16 +421,18 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 3.0.11
+      OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies

.github/workflows/jit.yml

@@ -4,10 +4,14 @@ on:
     paths:
       - '**jit**'
       - 'Python/bytecodes.c'
+      - 'Python/optimizer*.c'
+      - 'Python/tier2_redundancy_eliminator_bytecodes.c'
   push:
     paths:
       - '**jit**'
       - 'Python/bytecodes.c'
+      - 'Python/optimizer*.c'
+      - 'Python/tier2_redundancy_eliminator_bytecodes.c'
   workflow_dispatch:
 
 concurrency:

.github/workflows/reusable-macos.yml

@@ -28,11 +28,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ matrix.os }}-${{ inputs.config_hash }}
+        key: ${{ github.job }}-${{ matrix.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
       run: brew install pkg-config [email protected] xz gdbm tcl-tk
     - name: Configure CPython

.github/workflows/reusable-ubuntu.yml

@@ -14,7 +14,7 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-20.04
     env:
-      OPENSSL_VER: 3.0.11
+      OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -52,11 +52,13 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: ${{ inputs.options }}

Doc/c-api/buffer.rst

@@ -29,7 +29,7 @@ without intermediate copying.
 Python provides such a facility at the C level in the form of the :ref:`buffer
 protocol <bufferobjects>`.  This protocol has two sides:
 
-.. index:: single: PyBufferProcs
+.. index:: single: PyBufferProcs (C type)
 
 - on the producer side, a type can export a "buffer interface" which allows
   objects of that type to expose information about their underlying buffer.

Doc/c-api/code.rst

@@ -22,16 +22,21 @@ bound into a function.
 .. c:var:: PyTypeObject PyCode_Type
 
    This is an instance of :c:type:`PyTypeObject` representing the Python
-   :class:`code` type.
+   :ref:`code object <code-objects>`.
 
 
 .. c:function:: int PyCode_Check(PyObject *co)
 
-   Return true if *co* is a :class:`code` object.  This function always succeeds.
+   Return true if *co* is a :ref:`code object <code-objects>`.
+   This function always succeeds.
 
-.. c:function:: int PyCode_GetNumFree(PyCodeObject *co)
+.. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co)
 
-   Return the number of free variables in *co*.
+   Return the number of free variables in a code object.
+
+.. c:function:: int PyCode_GetFirstFree(PyCodeObject *co)
+
+   Return the position of the first free variable in a code object.
 
 .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
 
@@ -48,7 +53,7 @@ bound into a function.
    .. versionchanged:: 3.11
       Added ``qualname`` and ``exceptiontable`` parameters.
 
-   .. index:: single: PyCode_New
+   .. index:: single: PyCode_New (C function)
 
    .. versionchanged:: 3.12
 
@@ -61,7 +66,7 @@ bound into a function.
    Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments.
    The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function.
 
-   .. index:: single: PyCode_NewWithPosOnlyArgs
+   .. index:: single: PyCode_NewWithPosOnlyArgs (C function)
 
    .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs``
 
@@ -220,7 +225,7 @@ may change without deprecation warnings.
    *free* will be called on non-``NULL`` data stored under the new index.
    Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`.
 
-   .. index:: single: _PyEval_RequestCodeExtraIndex
+   .. index:: single: _PyEval_RequestCodeExtraIndex (C function)
 
    .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex``
 
@@ -238,7 +243,7 @@ may change without deprecation warnings.
    If no data was set under the index, set *extra* to ``NULL`` and return
    0 without setting an exception.
 
-   .. index:: single: _PyCode_GetExtra
+   .. index:: single: _PyCode_GetExtra (C function)
 
    .. versionadded:: 3.6 as ``_PyCode_GetExtra``
 
@@ -253,7 +258,7 @@ may change without deprecation warnings.
    Set the extra data stored under the given index to *extra*.
    Return 0 on success. Set an exception and return -1 on failure.
 
-   .. index:: single: _PyCode_SetExtra
+   .. index:: single: _PyCode_SetExtra (C function)
 
    .. versionadded:: 3.6 as ``_PyCode_SetExtra``
 

Doc/c-api/dict.rst

@@ -174,6 +174,26 @@ Dictionary Objects
    .. versionadded:: 3.4
 
 
+.. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result)
+
+   Inserts *default_value* into the dictionary *p* with a key of *key* if the
+   key is not already present in the dictionary. If *result* is not ``NULL``,
+   then *\*result* is set to a :term:`strong reference` to either
+   *default_value*, if the key was not present, or the existing value, if *key*
+   was already present in the dictionary.
+   Returns ``1`` if the key was present and *default_value* was not inserted,
+   or ``0`` if the key was not present and *default_value* was inserted.
+   On failure, returns ``-1``, sets an exception, and sets ``*result``
+   to ``NULL``.
+
+   For clarity: if you have a strong reference to *default_value* before
+   calling this function, then after it returns, you hold a strong reference
+   to both *default_value* and *\*result* (if it's not ``NULL``).
+   These may refer to the same object: in that case you hold two separate
+   references to it.
+   .. versionadded:: 3.13
+
+
 .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result)
 
    Remove *key* from dictionary *p* and optionally return the removed value.