Added support for Python 3.12 via manylinux_x_y (#739)

# Description The current versions of manylinux (manylinux1, manylinux2010, manylinux2014) are outdated and do not support Python 3.11+. [PEP 600](https://www.python.org/dev/peps/pep-0600/) defines a new future-proof versioning scheme tagged manylinux_x_y. This PR adds support for manylinux_x_y. The [weekly builds of containers](https://github.com/SINTEF/dlite/actions/workflows/container_builds_weekly.yml) has failed for a while without being noticed (since the weekly build is not run at push time, for good reasons - it takes about two hour to rebuild the musllinux docker image). The reason for this is that the old manylinux2010 and manylinux2014 does not support Python 3.11+ as well as several packages that has been added to `requirements_full.txt`. These issues are also addressed by this PR. See also https://github.com/mayeut/pep600_compliance ## Type of change - [x] Bug fix & code cleanup - [x] New feature - [ ] Documentation update - [ ] Test update ## Checklist for the reviewer This checklist should be used as a help for the reviewer. - [ ] Is the change limited to one issue? - [ ] Does this PR close the issue? - [ ] Is the code easy to read and understand? - [ ] Do all new feature have an accompanying new test? - [ ] Has the documentation been updated as necessary?
SINTEF · Dec 27, 2023 · 05e82b5 · 05e82b5
2 parents 2e31992 + 7808e83
commit 05e82b5
Show file tree

Hide file tree

Showing 6 changed files with 118 additions and 9 deletions.
diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template
@@ -1,5 +1,6 @@
 # -*- Mode: Dockerfile -*-
-# Dockerfile for building Python pypi package
+# Dockerfile for building Python pypi package.  Going to be superseeded by
+# Dockerfile-manylinux_x_y.template
 #
 # Usage:
 #
@@ -24,7 +25,7 @@
 # Reference: https://github.com/pypa/manylinux#manylinux2014-centos-7-based
 FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="7 8 9 10 11 12"
+ARG PY_MINORS="7 8 9 10"
 
 #COPY requirements.txt /tmp/requirements.txt
 COPY requirements_full.txt requirements_dev.txt /tmp/
@@ -55,8 +56,17 @@ RUN if [ "{{ TYPE }}{{ ARCH }}" != "2014i686" ]; then yum install -y hdf5-devel;
 # exist here.
 RUN  cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
-# Change required version of pydantic to be <2
-  sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt
+# Change required version of pydantic to be <2 and remove challenging packages
+  sed -e 's/^\(pydantic>.*<\).*$/\12/' \
+      -e '/^psycopg/d' \
+      -e '/^matplotlib/d' \
+      -e '/^scikit-image/d' \
+      -e '/^ncempy/d' \
+      -e '/^h5py/d' \
+      -e '/oteapi/d' \
+      -e '/^otelib/d' \
+      -i /tmp/requirements_full.txt
+
 # Install required Python packages
 RUN mkdir -p /ci/pip_cache && \
   if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then export PATH="$PATH:/usr/pgsql-9.1/bin"; fi && \

diff --git a/.github/docker/Dockerfile-manylinux_x_y.template b/.github/docker/Dockerfile-manylinux_x_y.template
@@ -0,0 +1,75 @@
+# -*- Mode: Dockerfile -*-
+# Dockerfile for building Python pypi package with manylinux_x_y based on
+# glibc>=x.y.  See https://github.com/mayeut/pep600_compliance
+#
+# Usage:
+#
+# Copy this template file and replace:
+# - `{{ TYPE }}` major glibc version, ex: 2_28, 2_35, 2_38
+# - `{{ ARCH }}` with a valid arch, ex:  x86_64, i686
+# Remove the `.template` suffix from the copy.
+#
+# Build:
+#
+#     docker build -t dlite-manylinux -f .github/docker/Dockerfile-manylinux_x_y .
+#
+# Run (for debugging):
+#
+#     docker run --rm -it \
+#        --volume $PWD:/io \
+#        --user $(id -u):$(id -g) \
+#        dlite-manylinux \
+#        /bin/bash
+#
+
+# Reference: https://github.com/pypa/manylinux
+FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}
+
+ARG PY_MINORS="7 8 9 10 11 12"
+
+#COPY requirements.txt /tmp/requirements.txt
+COPY requirements_full.txt requirements_dev.txt /tmp/
+
+{{ EXTRA_PRE }}
+
+# Enable rpmfusion for additional packages
+# Rust and cargo are needed by pydantic2
+RUN dnf update -y
+RUN dnf localinstall -y --skip-broken \
+    https://mirrors.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm --eval %{centos_ver}).noarch.rpm \
+    https://mirrors.rpmfusion.org/nonfree/el/rpmfusion-nonfree-release-$(rpm --eval %{centos_ver}).noarch.rpm && \
+  dnf install -y \
+    redland-devel \
+    rasqal-devel \
+    swig \
+    libcurl-devel \
+    libxslt-devel \
+    libxml2-devel \
+    hdf5-devel \
+    rust \
+    cargo
+
+# Unpack static libraries
+# It's necessary to be in /opt/_internal because the internal libraries
+# exist here.
+RUN  cd /opt/_internal && \
+    tar -Jxvf static-libs-for-embedding-only.tar.xz
+
+# Change required version of pydantic to be <2
+#RUN  sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt
+
+# Remove psycopg from requirements
+RUN  sed '/^psycopg/d' -i /tmp/requirements_full.txt
+
+# Install required Python packages
+RUN mkdir -p /ci/pip_cache && \
+    if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then export PATH="$PATH:/usr/pgsql-9.1/bin"; fi
+
+RUN for minor in ${PY_MINORS}; do \
+    python3.${minor} -m pip install -U pip && \
+    python3.${minor} -m pip install -U setuptools wheel && \
+    python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \
+    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt -r /tmp/requirements_dev.txt; \
+  done
+
+{{ EXTRA_POST }}
diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template
@@ -23,7 +23,7 @@
 #
 FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="7 8 9 10 11 12"
+ARG PY_MINORS="7 8 9 10"
 
 #COPY requirements.txt /tmp/requirements.txt
 COPY requirements_full.txt requirements_dev.txt /tmp/
@@ -47,8 +47,17 @@ RUN apk add -u rust cargo || true
 # exist here.
 RUN cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
-# Change required version of pydantic to be <2
-  sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt
+# Change required version of pydantic to be <2 and remove challenging packages
+  sed -e 's/^\(pydantic>.*<\).*$/\12/' \
+      -e '/^psycopg/d' \
+      -e '/^matplotlib/d' \
+      -e '/^scikit-image/d' \
+      -e '/^ncempy/d' \
+      -e '/^h5py/d' \
+      -e '/oteapi/d' \
+      -e '/^otelib/d' \
+      -i /tmp/requirements_full.txt
+
 # Install required Python packages
 RUN mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \

diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml
@@ -47,6 +47,10 @@ jobs:
           #   system_type: ["musllinux", "_1_1"]
           #   arch: x86_64
           #   py_minors: 7,10
+          - os: ubuntu-20.04
+            system_type: ["manylinux", "_2_28"]
+            arch: x86_64
+            py_minors: 7,12
 
           # See issue #220: https://github.com/SINTEF/dlite/issues/220
           # # 32-bit Windows

diff --git a/.github/workflows/container_builds_weekly.yml b/.github/workflows/container_builds_weekly.yml
@@ -24,28 +24,39 @@ jobs:
         include:
         # 64-bit linux
         - system: "manylinux"
+          template: "manylinux"
           type: "2010"
           arch: "x86_64"
           py_minors: "7 8 9"
         - system: "manylinux"
+          template: "manylinux"
           type: "2014"
           arch: "x86_64"
           py_minors: "7 8 9 10 11"
+        - system: "manylinux"
+          template: "manylinux_x_y"
+          type: "_2_28"
+          arch: "x86_64"
+          py_minors: "7 8 9 10 11 12"
         - system: "musllinux"
+          template: "musllinux"
           type: "_1_1"
           arch: "x86_64"
           py_minors: "7 8 9 10 11"
 
         # 32-bit linux
         - system: "manylinux"
+          template: "manylinux"
           type: "2010"
           arch: "i686"
           py_minors: "7 8 9"
         - system: "manylinux"
+          template: "manylinux"
           type: "2014"
           arch: "i686"
           py_minors: "7 8 9"
         - system: "musllinux"
+          template: "musllinux"
           type: "_1_1"
           arch: "i686"
           py_minors: "7 8 9"
@@ -96,7 +107,7 @@ jobs:
             -e "s|{{ TYPE }}|${{ matrix.type }}|" \
             -e "s|{{ EXTRA_PRE }}|${EXTRA_PRE}|" \
             -e "s|{{ EXTRA_POST }}|${EXTRA_POST}|" \
-            .github/docker/Dockerfile-${{ matrix.system }}.template \
+            .github/docker/Dockerfile-${{ matrix.template }}.template \
             > docker_build_wheel/Dockerfile-${{ matrix.system }}${{ matrix.type }}_${{ matrix.arch }}
 
       - name: Login to GitHub Container Registry

diff --git a/requirements_full.txt b/requirements_full.txt
@@ -16,7 +16,7 @@ minio>=6.0,<8
 
 # For TEM demo
 matplotlib>=3,<4
-scikit-image>=0.21,<1
+scikit-image>=0.19,<1
 ncempy>=1.10,<2
 h5py>=3.9,<4
 #oteapi-dlite>=0.1.5,<1