[RHOAIENG-10451] - storage-initializer hermetic build

Issues fixed by this commit: - Error: FetchError: exception_name: OSError, details: [Errno 28] No space left on device - error: No available modular metadata for modular package 'python39-3.9.19-1.module+el8.10.0+21815+bb024982.x86_64' - ERROR: No matching distribution found for poetry==1.7.1 - and more missing dependencies, added the build-requirements-build.txt Signed-off-by: Spolti <[email protected]>
red-hat-data-services · Sep 17, 2024 · b6338f0 · b6338f0
1 parent fdb2928
commit b6338f0
Show file tree

Hide file tree

Showing 9 changed files with 698 additions and 439 deletions.
diff --git a/.tekton/kserve-storage-initializer-test-hermetic-poc-pull-request.yaml b/.tekton/kserve-storage-initializer-test-hermetic-poc-pull-request.yaml
@@ -477,7 +477,7 @@ spec:
         - ReadWriteOnce
         resources:
           requests:
-            storage: 1Gi
+            storage: 5Gi
       status: {}
   - name: git-auth
     secret:

diff --git a/.tekton/kserve-storage-initializer-test-hermetic-poc-push.yaml b/.tekton/kserve-storage-initializer-test-hermetic-poc-push.yaml
@@ -35,7 +35,7 @@ spec:
     value: "true"
   - name: prefetch-input
     value: |
-     [{"type": "rpm", "path": "python"}, {"type": "pip", "path": "python/kserve", "requirements_files": ["requirements.txt"], "allow_binary": "true"}]
+     [{"type": "rpm", "path": "python"}, {"type": "pip", "path": "python/kserve", "requirements_files": ["requirements-build.txt", "requirements.txt"], "allow_binary": "true"}]
   pipelineSpec:
     finally:
     - name: show-sbom
@@ -476,7 +476,7 @@ spec:
         - ReadWriteOnce
         resources:
           requests:
-            storage: 1Gi
+            storage: 5Gi
       status: {}
   - name: git-auth
     secret:

diff --git a/python/kserve/requirements-build.txt b/python/kserve/requirements-build.txt
@@ -0,0 +1,44 @@
+poetry==1.7.1
+build==1.2.2
+cachecontrol==0.13.1
+cffi==1.16.0
+cleo==2.1.0
+crashtest==0.4.1
+cryptography==42.0.5
+Cython==3.0.11
+decorator==5.1.1
+docopt==0.6.2
+dulwich==0.21.7
+fastjsonschema==2.20.0
+gssapi==1.8.3
+hdfs==2.6.0
+importlib-metadata==8.5.0
+installer==0.7.0
+jaraco.classes==3.4.0
+jeepney==0.4.2
+keyring==24.3.1
+krb5==0.6.0
+krbcontext==0.10
+more-itertools==10.5.0
+pexpect==4.9.0
+pkginfo==1.11.1
+platformdirs==3.11.0
+poetry-core==1.8.1
+poetry-plugin-export==1.6.0
+ptyprocess==0.7.0
+pycparser==2.22
+pyproject-hooks==1.1.0
+pyspnego==0.11.1
+rapidfuzz==3.9.7
+requests-toolbelt==1.0.0
+requests-kerberos==0.14.0
+shellingham==1.5.4
+SecretStorage==3.3.3
+setuptools==56.0.0 #50.3.2
+starlette==0.36.2
+tomli==2.0.1
+tomlkit==0.13.2
+trove-classifiers==2024.9.12
+wheel==0.44.0
+#xattr==0.10.1
+
diff --git a/python/kserve/requirements.txt b/python/kserve/requirements.txt
diff --git a/python/rpms.in.yaml b/python/rpms.in.yaml
@@ -9,3 +9,5 @@ packages:
   - openssl-devel
   - krb5-workstation
   - krb5-libs
+  - shadow-utils
+
diff --git a/python/rpms.lock.yaml b/python/rpms.lock.yaml
@@ -488,3 +488,17 @@ arches:
     checksum: sha256:d632ef6e133e08c97258fdee1642a3c24106aa03f80e3c190a9d24be8669da7f
     name: zlib
     evr: 1.2.11-25.el8
+  - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/baseos/os/Packages/l/libsemanage-2.9-9.el8_6.x86_64.rpm
+    repoid: ubi-8-baseos-rpms
+    size: 172128
+    checksum: sha256:1f686a73273028ca85b5a6ac858292d7b7d2fcbe379d6912ba12fc0a49ac4cce
+    name: libsemanage
+    evr: 2.9-9.el8_6
+    sourcerpm: libsemanage-2.9-9.el8_6.src.rpm
+  - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/baseos/os/Packages/s/shadow-utils-4.6-22.el8.x86_64.rpm
+    repoid: ubi-8-baseos-rpms
+    size: 1292332
+    checksum: sha256:ea73ee201451bbca0d6d14ca434c93800f01c8fb1b9daef727a5af1a27356d07
+    name: shadow-utils
+    evr: 2:4.6-22.el8
+    sourcerpm: shadow-utils-4.6-22.el8.src.rpm
diff --git a/python/storage-initializer.Dockerfile b/python/storage-initializer.Dockerfile
@@ -4,7 +4,8 @@ ARG VENV_PATH=/prod_venv
 FROM registry.access.redhat.com/ubi8/ubi-minimal:latest as builder
 
 # Install Python and dependencies
-RUN microdnf install -y python39 python39-devel gcc libffi-devel openssl-devel krb5-workstation krb5-libs && microdnf clean all
+RUN microdnf install --setopt=ubi-8-appstream-rpms.module_hotfixes=1 \
+    -y python39 python39-devel gcc libffi-devel openssl-devel krb5-workstation krb5-libs && microdnf clean all
 
 # Install Poetry
 ARG POETRY_HOME=/opt/poetry
@@ -22,10 +23,13 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 # Addressing vulnerability scans by upgrading pip/setuptools
 RUN python3 -m pip install --upgrade pip setuptools
 
-COPY kserve/pyproject.toml kserve/poetry.lock kserve/
-RUN cd kserve && poetry install --no-root --no-interaction --no-cache --extras "storage"
+#COPY kserve/pyproject.toml kserve/poetry.lock kserve/
+#RUN cd kserve && poetry install --offline --no-root --no-interaction --no-cache --extras "storage"
+#COPY kserve kserve
+#RUN cd kserve && poetry install --offline --no-interaction --no-cache --extras "storage"
 COPY kserve kserve
-RUN cd kserve && poetry install --no-interaction --no-cache --extras "storage"
+RUN cd kserve && pip install -r requirements.txt --no-cache-dir
+RUN cd kserve && pip install . --no-cache-dir
 
 RUN pip install --no-cache-dir krbcontext==0.10 hdfs~=2.6.0 requests-kerberos==0.14.0
 # Fixes Quay alert GHSA-2jv5-9r88-3w3p https://github.com/Kludex/python-multipart/security/advisories/GHSA-2jv5-9r88-3w3p
@@ -40,7 +44,7 @@ ARG VENV_PATH
 ENV VIRTUAL_ENV=${VENV_PATH}
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-RUN microdnf install -y shadow-utils python39 python39-devel && \
+RUN microdnf install -y --disablerepo=* --enablerepo=ubi-8-baseos-rpms shadow-utils python39 python39-devel && \
     microdnf clean all
 RUN useradd kserve -m -u 1000 -d /home/kserve
 

diff --git a/rpms.in.yaml b/rpms.in.yaml
diff --git a/rpms.lock.yaml b/rpms.lock.yaml