Chromium und Alpine update (#319)

* Update base image to alpine v3.19

* Update Chromium to 121

* Remove libssl1.1 (no longer available in alpine 3.19)

* Add --break-system-packages to pip install commands

* Print debugging info

* Upgrade actions/checkout to v4

* Fix entrypoints and commands

* Upgrade pyOpenSSL to v24.0.0

* Upgrade tenacity to v8

.github/workflows/test.yaml

@@ -6,7 +6,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build Docker image
         run: make dockerimage
       - name: Create placeholder secrets

Dockerfile

@@ -1,28 +1,28 @@
-FROM alpine:3.16.2
+FROM alpine:3.19
 
-ENV CHROMIUM_VERSION=106.0.5249.119-r1
+ENV CHROMIUM_VERSION=121.0.6167.184-r0
 
 RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories && \
     echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
     apk --update --no-cache add ca-certificates \
           chromium=$CHROMIUM_VERSION \
           chromium-chromedriver=$CHROMIUM_VERSION \
           py3-cryptography python3-dev py3-grpcio py3-wheel py3-pip py3-lxml py3-yaml \
-          build-base git icu-libs libssl1.1 libssl3 libxml2 libxml2-dev libxslt libxslt-dev \
+          build-base git icu-libs libssl3 libxml2 libxml2-dev libxslt libxslt-dev \
           libffi-dev openssl-dev cargo
 
 RUN apk info -v | sort
 
 WORKDIR /workdir
 
 # Execute time consuming compilations in a separate step
-RUN python3 -m pip install libcst==0.4.7 sgmllib3k==1.0.0
+RUN python3 -m pip install libcst==0.4.7 sgmllib3k==1.0.0 --break-system-packages
 
 ADD https://pki.google.com/roots.pem /google_roots.pem
 ENV GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=/google_roots.pem
 
 ADD requirements.txt /workdir/
-RUN pip install -r requirements.txt
+RUN pip install -r requirements.txt --break-system-packages
 
 RUN python3 -m pip freeze
 

Makefile

@@ -28,18 +28,20 @@ spider:
 		-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \
 		--shm-size=2g \
 		$(IMAGE) \
-		--credentials-path /secrets/datastore-writer.json \
-		--loglevel debug \
-		spider --kind $(DB_ENTITY) ${ARGS}
+		python3 cli.py \
+			--credentials-path /secrets/datastore-writer.json \
+			--loglevel debug \
+			spider --kind $(DB_ENTITY) ${ARGS}
 
 export:
 	docker run --rm -ti \
 		-v $(PWD)/secrets:/secrets \
 		-v $(PWD)/volumes/json-export:/json-export \
 		$(IMAGE) \
-		--credentials-path /secrets/datastore-reader.json \
-		--loglevel debug \
-		export --kind $(DB_ENTITY)
+		python3 cli.py \
+			--credentials-path /secrets/datastore-reader.json \
+			--loglevel debug \
+			export --kind $(DB_ENTITY)
 
 # run spider tests
 test:
@@ -48,9 +50,8 @@ test:
       -v $(PWD)/secrets:/secrets \
       -v $(PWD)/screenshots:/screenshots \
 	  -v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \
-		--entrypoint "python3" \
 		$(IMAGE) \
-		-m unittest discover -p '*_test.py' -v
+			python3 -m unittest discover -p '*_test.py' -v
 
 VERSION:
 	@echo $(VERSION) > VERSION

requirements.txt

@@ -25,7 +25,7 @@ protobuf==4.21.8
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycparser==2.21
-pyOpenSSL==22.0.0
+pyOpenSSL==23.0.0
 pytz==2021.3
 redis==4.1.0
 requests==2.26.0
@@ -36,7 +36,7 @@ selenium==3.141.0
 smmap==3.0.4
 smmap2==2.0.5
 soupsieve==2.2.1
-tenacity==5.1.5
+tenacity==8.2.3
 urllib3==1.26.7
 w3lib==1.22.0
 websocket-client==0.59.0