From 8d367d2c9952e3884b0b1c6f32a503de647599b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 17 Jan 2025 12:10:27 +0100 Subject: [PATCH] GH-45237: [Python] Raise minimum supported cython to >=3 (#45238) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change We do not require to support Cython 2 anymore. ### What changes are included in this PR? Bump required version of Cython ### Are these changes tested? Via CI ### Are there any user-facing changes? Yes, pyarrow cannot be compiled with Cython2 anymore. * GitHub Issue: #45237 Authored-by: Raúl Cumplido Signed-off-by: Raúl Cumplido --- .github/workflows/dev.yml | 2 +- ci/conda_env_python.txt | 2 +- ci/docker/conda-python-cython2.dockerfile | 24 ---------------------- dev/tasks/tasks.yml | 8 -------- docker-compose.yml | 25 ----------------------- python/CMakeLists.txt | 4 ---- python/pyarrow/_azurefs.pyx | 3 --- python/pyarrow/_dataset_parquet.pyx | 2 -- python/pyarrow/_fs.pyx | 2 -- python/pyarrow/_gcsfs.pyx | 3 --- python/pyarrow/_hdfs.pyx | 3 --- python/pyarrow/_s3fs.pyx | 3 --- python/pyarrow/io.pxi | 1 - python/pyarrow/scalar.pxi | 2 -- python/pyproject.toml | 2 +- python/requirements-build.txt | 2 +- python/requirements-wheel-build.txt | 2 +- python/setup.py | 4 ++-- 18 files changed, 7 insertions(+), 87 deletions(-) delete mode 100644 ci/docker/conda-python-cython2.dockerfile diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index d59da447612a6..f9718cbf7bb18 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -124,7 +124,7 @@ jobs: shell: bash run: | gem install test-unit - pip install "cython>=0.29.31" setuptools pytest requests setuptools-scm + pip install "cython>=3" setuptools pytest requests setuptools-scm - name: Run Release Test env: ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt index bf915493de302..b467cc1ddd3b4 100644 --- a/ci/conda_env_python.txt +++ b/ci/conda_env_python.txt @@ -18,7 +18,7 @@ # don't add pandas here, because it is not a mandatory test dependency boto3 # not a direct dependency of s3fs, but needed for our s3fs fixture cffi -cython>=0.29.31 +cython>=3 cloudpickle fsspec hypothesis diff --git a/ci/docker/conda-python-cython2.dockerfile b/ci/docker/conda-python-cython2.dockerfile deleted file mode 100644 index 859ad868b0c71..0000000000000 --- a/ci/docker/conda-python-cython2.dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch -ARG python=3.9 -FROM ${repo}:${arch}-conda-python-${python} - -RUN mamba install -q -y "cython<3" && \ - mamba clean --all diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 93450df155c9a..cade2d51a0f90 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1116,14 +1116,6 @@ tasks: PYTHON: "3.10" image: conda-python-substrait - test-conda-python-3.10-cython2: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: "3.10" - image: conda-python-cython2 - test-ubuntu-22.04-python-313-freethreading: ci: github template: docker-tests/github.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index 380276d29f879..87383568aaba4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -123,7 +123,6 @@ x-hierarchy: - conda-python-pandas: - conda-python-docs - conda-python-cpython-debug - - conda-python-cython2 - conda-python-dask - conda-python-emscripten - conda-python-hdfs @@ -1495,30 +1494,6 @@ services: /arrow/ci/scripts/python_build.sh /arrow /build && /arrow/ci/scripts/integration_substrait.sh"] - conda-python-cython2: - # Usage: - # docker compose build conda - # docker compose build conda-cpp - # docker compose build conda-python - # docker compose build conda-python-cython2 - # docker compose run --rm conda-python-cython2 - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-cython2 - build: - context: . - dockerfile: ci/docker/conda-python-cython2.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-cython2 - args: - repo: ${REPO} - arch: ${ARCH} - python: ${PYTHON} - shm_size: *shm-size - environment: - <<: [*common, *ccache] - PYTEST_ARGS: # inherit - volumes: *conda-volumes - command: *python-conda-command - conda-python-cpython-debug: # Usage: # docker compose build conda diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index fa9267c8857c5..0b0275f200d9d 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -200,10 +200,6 @@ if(MSVC) # seem harmless, and probably not worth the effort of working around it string(APPEND CMAKE_CXX_FLAGS " /wd4800") - # See https://github.com/cython/cython/issues/2731. Change introduced in - # Cython 0.29.1 causes "unsafe use of type 'bool' in operation" - string(APPEND CMAKE_CXX_FLAGS " /wd4804") - # See https://github.com/cython/cython/issues/4445. # # Cython 3 emits "(void)__Pyx_PyObject_CallMethod0;" to suppress a diff --git a/python/pyarrow/_azurefs.pyx b/python/pyarrow/_azurefs.pyx index 5cd6300c18c6a..60c326d280e2f 100644 --- a/python/pyarrow/_azurefs.pyx +++ b/python/pyarrow/_azurefs.pyx @@ -17,8 +17,6 @@ # cython: language_level = 3 -from cython cimport binding - from pyarrow.lib import frombytes, tobytes from pyarrow.includes.libarrow_fs cimport * @@ -115,7 +113,6 @@ cdef class AzureFileSystem(FileSystem): self.azurefs = wrapped.get() @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/_dataset_parquet.pyx b/python/pyarrow/_dataset_parquet.pyx index 8fe9f30d33af9..863c928591937 100644 --- a/python/pyarrow/_dataset_parquet.pyx +++ b/python/pyarrow/_dataset_parquet.pyx @@ -19,7 +19,6 @@ """Dataset support for Parquet file format.""" -from cython cimport binding from cython.operator cimport dereference as deref import os @@ -890,7 +889,6 @@ cdef class ParquetFragmentScanOptions(FragmentScanOptions): return attrs == other_attrs @staticmethod - @binding(True) # Required for Cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index e315dd6381f41..2b1f0e7eb3189 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -18,7 +18,6 @@ # cython: language_level = 3 from cpython.datetime cimport datetime, PyDateTime_DateTime -from cython cimport binding from pyarrow.includes.common cimport * from pyarrow.includes.libarrow_python cimport PyDateTime_to_TimePoint @@ -422,7 +421,6 @@ cdef class FileSystem(_Weakrefable): "SubTreeFileSystem") @staticmethod - @binding(True) # Required for cython < 3 def _from_uri(uri): fs, _path = FileSystem.from_uri(uri) return fs diff --git a/python/pyarrow/_gcsfs.pyx b/python/pyarrow/_gcsfs.pyx index 5e69413cea953..a5a366fe9cc67 100644 --- a/python/pyarrow/_gcsfs.pyx +++ b/python/pyarrow/_gcsfs.pyx @@ -17,8 +17,6 @@ # cython: language_level = 3 -from cython cimport binding - from pyarrow.lib cimport (pyarrow_wrap_metadata, pyarrow_unwrap_metadata) from pyarrow.lib import frombytes, tobytes, ensure_metadata @@ -164,7 +162,6 @@ cdef class GcsFileSystem(FileSystem): return datetime.fromtimestamp(expiration_ns / 1.0e9, timezone.utc) @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/_hdfs.pyx b/python/pyarrow/_hdfs.pyx index c426337a12ec1..0a1a5483bda32 100644 --- a/python/pyarrow/_hdfs.pyx +++ b/python/pyarrow/_hdfs.pyx @@ -17,8 +17,6 @@ # cython: language_level = 3 -from cython cimport binding - from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport * from pyarrow.includes.libarrow_fs cimport * @@ -137,7 +135,6 @@ replication=1)`` return self @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx index 038f9109d8152..2a7cf6dfdd143 100644 --- a/python/pyarrow/_s3fs.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -17,8 +17,6 @@ # cython: language_level = 3 -from cython cimport binding - from pyarrow.lib cimport (check_status, pyarrow_wrap_metadata, pyarrow_unwrap_metadata) from pyarrow.lib import frombytes, tobytes, KeyValueMetadata @@ -419,7 +417,6 @@ cdef class S3FileSystem(FileSystem): self.s3fs = wrapped.get() @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index b3de15067fbfa..89015dadba5ae 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -2370,7 +2370,6 @@ cdef class CacheOptions(_Weakrefable): ideal_bandwidth_utilization_frac, max_ideal_request_size_mib)) @staticmethod - @binding(True) # Required for Cython < 3 def _reconstruct(kwargs): # __reduce__ doesn't allow passing named arguments directly to the # reconstructor, hence this wrapper. diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi index 2235cd0b981a6..e877b0965d139 100644 --- a/python/pyarrow/scalar.pxi +++ b/python/pyarrow/scalar.pxi @@ -16,7 +16,6 @@ # under the License. import collections -from cython cimport binding from uuid import UUID @@ -895,7 +894,6 @@ cdef class DictionaryScalar(Scalar): """ @staticmethod - @binding(True) # Required for cython < 3 def _reconstruct(type, is_valid, index, dictionary): cdef: CDictionaryScalarIndexAndDictionary value diff --git a/python/pyproject.toml b/python/pyproject.toml index 76d2fab4316f3..567fc96f78ebf 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -17,7 +17,7 @@ [build-system] requires = [ - "cython >= 0.29.31", + "cython >= 3", # Starting with NumPy 1.25, NumPy is (by default) as far back compatible # as oldest-support-numpy was (customizable with a NPY_TARGET_VERSION # define). For older Python versions (where NumPy 1.25 is not yet available) diff --git a/python/requirements-build.txt b/python/requirements-build.txt index c150c842a0cc6..00b523ee52120 100644 --- a/python/requirements-build.txt +++ b/python/requirements-build.txt @@ -1,4 +1,4 @@ -cython>=0.29.31 +cython>=3 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=1.25; python_version>='3.9' setuptools_scm>=8 diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index faa078d3d7fe7..38e59ce45ab8e 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,4 +1,4 @@ -cython>=0.29.31 +cython>=3 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=2.0.0; python_version>='3.9' setuptools_scm diff --git a/python/setup.py b/python/setup.py index 60b9a696d9785..bd658b91a53a5 100755 --- a/python/setup.py +++ b/python/setup.py @@ -48,9 +48,9 @@ ) -if Cython.__version__ < '0.29.31': +if Cython.__version__ < '3': raise Exception( - 'Please update your Cython version. Supported Cython >= 0.29.31') + 'Please update your Cython version. Supported Cython >= 3') setup_dir = os.path.abspath(os.path.dirname(__file__))