GH-45237: [Python] Raise minimum supported cython to >=3 (#45238)

### Rationale for this change We do not require to support Cython 2 anymore. ### What changes are included in this PR? Bump required version of Cython ### Are these changes tested? Via CI ### Are there any user-facing changes? Yes, pyarrow cannot be compiled with Cython2 anymore. * GitHub Issue: #45237 Authored-by: Raúl Cumplido <[email protected]> Signed-off-by: Raúl Cumplido <[email protected]>
apache · Jan 17, 2025 · 8d367d2 · 8d367d2
1 parent 982c469
commit 8d367d2
Show file tree

Hide file tree

Showing 18 changed files with 7 additions and 87 deletions.
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -124,7 +124,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit
-          pip install "cython>=0.29.31" setuptools pytest requests setuptools-scm
+          pip install "cython>=3" setuptools pytest requests setuptools-scm
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt
@@ -18,7 +18,7 @@
 # don't add pandas here, because it is not a mandatory test dependency
 boto3  # not a direct dependency of s3fs, but needed for our s3fs fixture
 cffi
-cython>=0.29.31
+cython>=3
 cloudpickle
 fsspec
 hypothesis

diff --git a/ci/docker/conda-python-cython2.dockerfile b/ci/docker/conda-python-cython2.dockerfile
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -1116,14 +1116,6 @@ tasks:
         PYTHON: "3.10"
       image: conda-python-substrait
 
-  test-conda-python-3.10-cython2:
-    ci: github
-    template: docker-tests/github.linux.yml
-    params:
-      env:
-        PYTHON: "3.10"
-      image: conda-python-cython2
-
   test-ubuntu-22.04-python-313-freethreading:
     ci: github
     template: docker-tests/github.linux.yml

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -123,7 +123,6 @@ x-hierarchy:
         - conda-python-pandas:
           - conda-python-docs
         - conda-python-cpython-debug
-        - conda-python-cython2
         - conda-python-dask
         - conda-python-emscripten
         - conda-python-hdfs
@@ -1495,30 +1494,6 @@ services:
         /arrow/ci/scripts/python_build.sh /arrow /build &&
         /arrow/ci/scripts/integration_substrait.sh"]
 
-  conda-python-cython2:
-    # Usage:
-    #   docker compose build conda
-    #   docker compose build conda-cpp
-    #   docker compose build conda-python
-    #   docker compose build conda-python-cython2
-    #   docker compose run --rm conda-python-cython2
-    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-cython2
-    build:
-      context: .
-      dockerfile: ci/docker/conda-python-cython2.dockerfile
-      cache_from:
-        - ${REPO}:${ARCH}-conda-python-${PYTHON}-cython2
-      args:
-        repo: ${REPO}
-        arch: ${ARCH}
-        python: ${PYTHON}
-    shm_size: *shm-size
-    environment:
-      <<: [*common, *ccache]
-      PYTEST_ARGS:  # inherit
-    volumes: *conda-volumes
-    command: *python-conda-command
-
   conda-python-cpython-debug:
     # Usage:
     #   docker compose build conda

diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
@@ -200,10 +200,6 @@ if(MSVC)
   # seem harmless, and probably not worth the effort of working around it
   string(APPEND CMAKE_CXX_FLAGS " /wd4800")
 
-  # See https://github.com/cython/cython/issues/2731. Change introduced in
-  # Cython 0.29.1 causes "unsafe use of type 'bool' in operation"
-  string(APPEND CMAKE_CXX_FLAGS " /wd4804")
-
   # See https://github.com/cython/cython/issues/4445.
   #
   # Cython 3 emits "(void)__Pyx_PyObject_CallMethod0;" to suppress a

diff --git a/python/pyarrow/_azurefs.pyx b/python/pyarrow/_azurefs.pyx
@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-from cython cimport binding
-
 
 from pyarrow.lib import frombytes, tobytes
 from pyarrow.includes.libarrow_fs cimport *
@@ -115,7 +113,6 @@ cdef class AzureFileSystem(FileSystem):
         self.azurefs = <CAzureFileSystem*> wrapped.get()
 
     @staticmethod
-    @binding(True)  # Required for cython < 3
     def _reconstruct(kwargs):
         # __reduce__ doesn't allow passing named arguments directly to the
         # reconstructor, hence this wrapper.

diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx
@@ -19,7 +19,6 @@
 
 """Dataset support for Parquet file format."""
 
-from cython cimport binding
 from cython.operator cimport dereference as deref
 
 import os
@@ -890,7 +889,6 @@ cdef class ParquetFragmentScanOptions(FragmentScanOptions):
         return attrs == other_attrs
 
     @staticmethod
-    @binding(True)  # Required for Cython < 3
     def _reconstruct(kwargs):
         # __reduce__ doesn't allow passing named arguments directly to the
         # reconstructor, hence this wrapper.

diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx
@@ -18,7 +18,6 @@
 # cython: language_level = 3
 
 from cpython.datetime cimport datetime, PyDateTime_DateTime
-from cython cimport binding
 
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow_python cimport PyDateTime_to_TimePoint
@@ -422,7 +421,6 @@ cdef class FileSystem(_Weakrefable):
                         "SubTreeFileSystem")
 
     @staticmethod
-    @binding(True)  # Required for cython < 3
     def _from_uri(uri):
         fs, _path = FileSystem.from_uri(uri)
         return fs

diff --git a/python/pyarrow/_gcsfs.pyx b/python/pyarrow/_gcsfs.pyx
@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-from cython cimport binding
-
 from pyarrow.lib cimport (pyarrow_wrap_metadata,
                           pyarrow_unwrap_metadata)
 from pyarrow.lib import frombytes, tobytes, ensure_metadata
@@ -164,7 +162,6 @@ cdef class GcsFileSystem(FileSystem):
         return datetime.fromtimestamp(expiration_ns / 1.0e9, timezone.utc)
 
     @staticmethod
-    @binding(True)  # Required for cython < 3
     def _reconstruct(kwargs):
         # __reduce__ doesn't allow passing named arguments directly to the
         # reconstructor, hence this wrapper.

diff --git a/python/pyarrow/_hdfs.pyx b/python/pyarrow/_hdfs.pyx
@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-from cython cimport binding
-
 from pyarrow.includes.common cimport *
 from pyarrow.includes.libarrow cimport *
 from pyarrow.includes.libarrow_fs cimport *
@@ -137,7 +135,6 @@ replication=1)``
         return self
 
     @staticmethod
-    @binding(True)  # Required for cython < 3
     def _reconstruct(kwargs):
         # __reduce__ doesn't allow passing named arguments directly to the
         # reconstructor, hence this wrapper.

diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx
@@ -17,8 +17,6 @@
 
 # cython: language_level = 3
 
-from cython cimport binding
-
 from pyarrow.lib cimport (check_status, pyarrow_wrap_metadata,
                           pyarrow_unwrap_metadata)
 from pyarrow.lib import frombytes, tobytes, KeyValueMetadata
@@ -419,7 +417,6 @@ cdef class S3FileSystem(FileSystem):
         self.s3fs = <CS3FileSystem*> wrapped.get()
 
     @staticmethod
-    @binding(True)  # Required for cython < 3
     def _reconstruct(kwargs):
         # __reduce__ doesn't allow passing named arguments directly to the
         # reconstructor, hence this wrapper.

diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi
@@ -2370,7 +2370,6 @@ cdef class CacheOptions(_Weakrefable):
             ideal_bandwidth_utilization_frac, max_ideal_request_size_mib))
 
     @staticmethod
-    @binding(True)  # Required for Cython < 3
     def _reconstruct(kwargs):
         # __reduce__ doesn't allow passing named arguments directly to the
         # reconstructor, hence this wrapper.

diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
@@ -16,7 +16,6 @@
 # under the License.
 
 import collections
-from cython cimport binding
 from uuid import UUID
 
 
@@ -895,7 +894,6 @@ cdef class DictionaryScalar(Scalar):
     """
 
     @staticmethod
-    @binding(True)  # Required for cython < 3
     def _reconstruct(type, is_valid, index, dictionary):
         cdef:
             CDictionaryScalarIndexAndDictionary value

diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -17,7 +17,7 @@
 
 [build-system]
 requires = [
-    "cython >= 0.29.31",
+    "cython >= 3",
     # Starting with NumPy 1.25, NumPy is (by default) as far back compatible
     # as oldest-support-numpy was (customizable with a NPY_TARGET_VERSION
     # define).  For older Python versions (where NumPy 1.25 is not yet available)

diff --git a/python/requirements-build.txt b/python/requirements-build.txt
@@ -1,4 +1,4 @@
-cython>=0.29.31
+cython>=3
 oldest-supported-numpy>=0.14; python_version<'3.9'
 numpy>=1.25; python_version>='3.9'
 setuptools_scm>=8

diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt
@@ -1,4 +1,4 @@
-cython>=0.29.31
+cython>=3
 oldest-supported-numpy>=0.14; python_version<'3.9'
 numpy>=2.0.0; python_version>='3.9'
 setuptools_scm

diff --git a/python/setup.py b/python/setup.py
@@ -48,9 +48,9 @@
 )
 
 
-if Cython.__version__ < '0.29.31':
+if Cython.__version__ < '3':
     raise Exception(
-        'Please update your Cython version. Supported Cython >= 0.29.31')
+        'Please update your Cython version. Supported Cython >= 3')
 
 setup_dir = os.path.abspath(os.path.dirname(__file__))