Updated the Dockerfile templates for the weekly build of containers (#…

…930)

Updated and added more packages to the Dockerfile templates for the weekly build of containers.
This should hopefully make the CI_build_wheels workflow should be much faster. Fixed also some bugs.

.github/docker/Dockerfile-manylinux.template

@@ -37,6 +37,8 @@ FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest
 
 ARG PY_MINORS="8 9 10 11 12"
 
+COPY requirements_full.txt requirements_dev.txt /tmp/
+
 {{ EXTRA_PRE }}
 
 # Enable rpmfusion for additional packages
@@ -46,26 +48,47 @@ RUN \
     https://mirrors.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm --eval %{centos_ver}).noarch.rpm \
     https://mirrors.rpmfusion.org/nonfree/el/rpmfusion-nonfree-release-$(rpm --eval %{centos_ver}).noarch.rpm && \
   yum install -y \
+    swig \
     redland-devel \
     rasqal-devel \
-    swig \
     libcurl-devel \
     libxslt-devel \
-    libxml2-devel && \
+    libxml2-devel \
+    libffi-devel && \
   # hdf5-devel is needed for building the hdf5 plugin
   yum install -y hdf5-devel || true && \
   # Unpack static libraries
   # It's necessary to be in /opt/_internal because the internal libraries
   # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+  # Filter out challenging packages from requirements
+  sed \
+    -e '/mongomock/d' \
+    -i  /tmp/requirements_dev.txt && \
+  sed \
+    -e '/pandas/d' \
+    -e '/pyarrow/d' \
+    -e '/tables/d' \
+    -e '/openpyxl/d' \
+    -e '/jinja2/d' \
+    -e '/pydantic/d' \
+    -e '/fortran-language-server/d' \
+    -e '/matplotlib/d' \
+    -e '/scikit-image/d' \
+    -e '/ncempy/d' \
+    -e '/h5py/d' \
+    -i /tmp/requirements_full.txt && \
   # Install required Python packages
   mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
-    python3.${minor} -m pip install -U pip && \
-    python3.${minor} -m pip install -U setuptools wheel && \
-    python3.${minor} -m pip install -U --cache-dir /ci/pip_cache \
-      cmake oldest-supported-numpy; \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
+      -U setuptools wheel && \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
+      oldest-supported-numpy \
+      -r /tmp/requirements_dev.txt \
+      -r /tmp/requirements_full.txt; \
   done && \
   rm -rf /ci/pip_cache
 

.github/docker/Dockerfile-manylinux_x_y.template

@@ -21,7 +21,7 @@
 #
 #     cd dlite  # cd to DLite root directory
 #     docker build -t dlite-manylinux \
-#         -f .github/docker/Dockerfile-manylinux_2_24_x86_64 .
+#         -f .github/docker/dockerfiles/Dockerfile-manylinux_2_24_x86_64 .
 #
 # Run (for debugging):
 #
@@ -39,6 +39,8 @@ ARG PY_MINORS="8 9 10 11 12"
 
 {{ EXTRA_PRE }}
 
+COPY requirements_full.txt requirements_dev.txt /tmp/
+
 # Enable rpmfusion for additional packages
 RUN \
   dnf update -y && \
@@ -58,13 +60,22 @@ RUN \
   # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+  # Filter out challenging packages from requirements_full.txt
+  #sed \
+  #  -e '/pyarrow/d' \
+  #  -e '/tables/d' \
+  #  -e '/scikit-image/d' \
+  #  -i /tmp/requirements_full.txt && \
   # Install required Python packages
   mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
-    python3.${minor} -m pip install -U pip && \
-    python3.${minor} -m pip install -U setuptools wheel && \
-    python3.${minor} -m pip install -U --cache-dir /ci/pip_cache \
-      cmake oldest-supported-numpy; \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
+      -U setuptools wheel && \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
+      oldest-supported-numpy \
+      -r /tmp/requirements_dev.txt \
+      -r /tmp/requirements_full.txt; \
   done && \
   rm -rf /ci/pip_cache
 

.github/docker/Dockerfile-musllinux.template

@@ -34,33 +34,52 @@ FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest
 
 ARG PY_MINORS="8 9 10 11 12"
 
+COPY requirements_full.txt requirements_dev.txt /tmp/
+
+{{ EXTRA_PRE }}
+
 # Do not use distutils distributed with setuptools
 # This is due to base changes in the distutils API, removing msvccompiler,
 # which is necessary for building the numpy wheel.
 ENV SETUPTOOLS_USE_DISTUTILS="stdlib"
 
-RUN apk add -u \
-  redland \
-  rasqal \
-  hdf5-dev \
-  openssl-dev \
-  swig && \
+RUN \
+  apk add -u \
+    redland-dev \
+    rasqal-dev \
+    hdf5-dev \
+    openssl-dev \
+    openblas-dev \
+    swig && \
   # Unpack static libraries
   # It's necessary to be in /opt/_internal because the internal libraries
   # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
-  # Filter minor versions
-  minors=$( for n in $PY_MINORS; do \
-    if [ $(uname -m) == "i686" -o $n -lt 12 ]; then echo $n; fi; \
-  done ) && \
-  echo "*** Minors: $minors" && \
+  # Filter out challenging packages from requirements_full.txt
+  sed \
+    -e '/pandas/d' \
+    -e '/pyarrow/d' \
+    -e '/tables/d' \
+    -e '/jinja2/d' \
+    -e '/pydantic/d' \
+    -e '/matplotlib/d' \
+    -e '/scikit-image/d' \
+    -e '/ncempy/d' \
+    -e '/h5py/d' \
+    -i /tmp/requirements_full.txt && \
+    #-e 's/^pydantic.*/pydantic<2/' \
   # Install required Python packages
   mkdir -p /ci/pip_cache && \
-  for minor in ${minors}; do \
-    python3.${minor} -m pip install -U pip && \
-    python3.${minor} -m pip install -U setuptools wheel; \
-    python3.${minor} -m pip install -U --cache-dir /ci/pip_cache \
-      oldest-supported-numpy; \
+  for minor in ${PY_MINORS}; do \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
+      -U setuptools wheel && \
+    python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
+      oldest-supported-numpy \
+      -r /tmp/requirements_dev.txt \
+      -r /tmp/requirements_full.txt; \
   done && \
   rm -rf /ci/pip_cache
+
+{{ EXTRA_POST }}

.github/docker/gen_dockerfile.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Write Dockerfile for given system to stdout.
 #
 # Usage:

.github/docker/run_cibuildwheel.sh

@@ -9,6 +9,10 @@
 # and SYSTEM_TYPE is the specific linux system type, e.g., 2010 or 2014 for manylinux
 # and _1_1 or _1_2 for musllinux
 # More information: https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip
+#
+# Example:
+#
+#     run_cibuildwheel.sh cp38-musllinux_x86_64 _1_1
 set -eu
 
 if [[ $# -eq 0 ]]; then

.github/workflows/container_builds_weekly.yml

@@ -95,7 +95,7 @@ jobs:
         if: env.BUILD_NEW_IMAGE == 'true'
         run: |
           mkdir -p docker_build_wheel
-          sh .github/docker/gen_dockerfile.sh ${{ matrix.system }} ${{ matrix.type }} ${{ matrix.arch }} \
+          bash .github/docker/gen_dockerfile.sh ${{ matrix.system }} ${{ matrix.type }} ${{ matrix.arch }} \
             > docker_build_wheel/Dockerfile-${{ matrix.system }}${{ matrix.type }}_${{ matrix.arch }}
 
       - name: Login to GitHub Container Registry

requirements_full.txt

@@ -25,10 +25,10 @@ typing_extensions>=4.1,<5
 fortran-language-server>=1.12.0,<1.13
 
 # For TEM demo
-matplotlib>=3,<4
-scikit-image>=0.19,<1
-ncempy>=1.10,<2
-h5py>=3.9,<4
+##matplotlib>=3,<4
+##scikit-image>=0.19,<1
+##ncempy>=1.10,<2
+##h5py>=3.9,<4
 #oteapi-dlite>=0.1.5,<1
 #git+https://github.com/EMMC-ASBL/oteapi-dlite.git@fa1b820383eb54a1c37f32f7b8ac9406b556dace#egg=oteapi_dlite
 #otelib>=0.3.2,<1