Merge pull request #43 from brefphp/compile-php

.dockerignore

@@ -2,7 +2,6 @@
 aws/
 output/
 tests/
-utils/
 .env
 .env.example
 .gitignore

.env.example

@@ -7,3 +7,8 @@
 # Limit the parallelization of layer publication.
 # Default is 7, we recommend a lower number when publishing from a laptop.
 MAX_PARALLEL_PUBLISH=3
+
+# In the CI and on local machines in the core team we build using https://depot.dev
+# as it is much faster to build cross-platform images.
+# Do not uncomment this line, unless you have a depot.dev account.
+#USE_DEPOT=1

.github/workflows/release.yml

@@ -15,57 +15,64 @@ permissions:
     contents: read  # This is required for actions/checkout
 
 jobs:
-    release-x86:
-        name: Publish x86 layers
-        runs-on: ubuntu-latest
-        steps:
-            -   uses: actions/checkout@v3
-            -   name: Configure AWS credentials
-                uses: aws-actions/configure-aws-credentials@v1
-                with:
-                    role-to-assume: arn:aws:iam::534081306603:role/bref-layer-publisher-github-actions
-                    role-session-name: bref-layer-publisher-github-actions
-                    aws-region: us-east-1
-            -   name: Configure Docker Hub credentials
-                uses: docker/login-action@v2
-                with:
-                    username: ${{ secrets.DOCKER_USERNAME }}
-                    password: ${{ secrets.DOCKER_PASSWORD }}
-            -   run: make -f cpu-x86.Makefile layers
-            -   run: make -f cpu-x86.Makefile test
-            -   run: make -f cpu-x86.Makefile upload-layers
-            -   run: make -f cpu-x86.Makefile upload-to-docker-hub
 
-    release-arm:
-        name: Publish ARM layers
+    release:
+        name: Publish ${{ matrix.cpu }} layers
         runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                cpu:
+                    - x86
+                    - arm
         steps:
             -   uses: actions/checkout@v3
+
+            # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
+            -   name: Set up QEMU to run ARM images (that were built with Depot)
+                uses: docker/setup-qemu-action@v2
+
+            -   uses: depot/setup-action@v1
+
             -   name: Configure AWS credentials
                 uses: aws-actions/configure-aws-credentials@v1
                 with:
                     role-to-assume: arn:aws:iam::534081306603:role/bref-layer-publisher-github-actions
                     role-session-name: bref-layer-publisher-github-actions
                     aws-region: us-east-1
+
             -   name: Configure Docker Hub credentials
                 uses: docker/login-action@v2
                 with:
                     username: ${{ secrets.DOCKER_USERNAME }}
                     password: ${{ secrets.DOCKER_PASSWORD }}
-            # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
-            -   name: Set up QEMU to build ARM images
-                uses: docker/setup-qemu-action@v2
-            -   name: Set up Docker buildx to build ARM images
-                uses: docker/setup-buildx-action@v2
-            -   run: make -f cpu-arm.Makefile layers
-            -   run: make -f cpu-arm.Makefile test
-            -   run: make -f cpu-arm.Makefile upload-layers
-            -   run: make -f cpu-arm.Makefile upload-to-docker-hub
+
+            -   name: Build Docker images
+                run: make docker-images
+                env:
+                    CPU: ${{ matrix.cpu }}
+                    USE_DEPOT: 1
+                    DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
+
+            -   run: make layers
+                env:
+                    CPU: ${{ matrix.cpu }}
+
+            -   run: make test
+                env:
+                    CPU: ${{ matrix.cpu }}
+
+            -   run: make upload-layers
+                env:
+                    CPU: ${{ matrix.cpu }}
+
+            -   run: make upload-to-docker-hub
+                env:
+                    CPU: ${{ matrix.cpu }}
 
     update-layer-versions:
         name: Update layer versions in brefphp/bref
         runs-on: ubuntu-latest
-        needs: [ release-x86, release-arm ]
+        needs: [ release ]
         steps:
             -   name: Trigger layer update in brefphp/bref
                 uses: actions/github-script@v6
@@ -82,7 +89,7 @@ jobs:
     update-layer-js-versions:
         name: Update layer versions in brefphp/layers.js
         runs-on: ubuntu-latest
-        needs: [ release-x86, release-arm ]
+        needs: [ release ]
         steps:
             -   name: Trigger release in brefphp/layers.js
                 uses: actions/github-script@v6

.github/workflows/tests.yml

@@ -7,23 +7,43 @@ on:
         branches: [ '*' ]
 
 jobs:
-    tests-x86:
-        name: Build x86 images, layers, and run tests
+    tests:
+        name: Build and tests PHP ${{ matrix.php_version }}, ${{ matrix.cpu }}
         runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                cpu:
+                    - x86
+                    - arm
+                php_version:
+                    - 80
+                    - 81
+                    - 82
         steps:
             -   uses: actions/checkout@v3
-            -   run: make -f cpu-x86.Makefile layers
-            -   run: make -f cpu-x86.Makefile test
 
-    tests-arm:
-        name: Build ARM images, layers, and run tests
-        runs-on: ubuntu-latest
-        steps:
-            -   uses: actions/checkout@v3
             # See https://stackoverflow.com/questions/70312490/github-actions-runner-environment-doesnt-build-for-arm-images
-            -   name: Set up QEMU to build ARM images
+            -   name: Set up QEMU to run ARM images (that were built with Depot)
                 uses: docker/setup-qemu-action@v2
-            -   name: Set up Docker buildx to build ARM images
-                uses: docker/setup-buildx-action@v2
-            -   run: make -f cpu-arm.Makefile layers
-            -   run: make -f cpu-arm.Makefile test
+
+            -   uses: depot/setup-action@v1
+
+            -   name: Build Docker images
+                run: make docker-images-php-${{ matrix.php_version }}
+                env:
+                    CPU: ${{ matrix.cpu }}
+                    USE_DEPOT: 1
+                    DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
+
+            -   name: Test that layers can be exported
+                run: |
+                    make layer-php-${{ matrix.php_version }}
+                    make layer-php-${{ matrix.php_version }}-fpm
+                env:
+                    CPU: ${{ matrix.cpu }}
+
+            -   name: Run tests
+                run: make test-${{ matrix.php_version }}
+                env:
+                    CPU: ${{ matrix.cpu }}

Makefile

@@ -2,46 +2,102 @@
 -include .env
 export # export all variables defined in .env
 
-# - Build all layers
-# - Publish all Docker images to Docker Hub
-# - Publish all layers to AWS Lambda
+# Define all the environment variables depending on the CPU
+# Set CPU= (empty) to build for x86
+# Set CPU=arm to build for ARM
+ifeq ($(CPU), arm) # if $CPU=="arm"
+  $(info "⚠️  Building for ARM") # Print a message
+  export CPU = arm
+  export CPU_PREFIX = arm-
+  export IMAGE_VERSION_SUFFIX = arm64
+  export DOCKER_PLATFORM = linux/arm64
+else
+  $(info "⚠️  Building for x86") # Print a message
+  export CPU = x86
+  export CPU_PREFIX =
+  export IMAGE_VERSION_SUFFIX = x86_64
+  export DOCKER_PLATFORM = linux/amd64
+endif
+
+# By default, Docker images are built using `docker buildx bake`
+# But we use https://depot.dev in CI (super fast) by setting USE_DEPOT=1
+ifeq ($(USE_DEPOT), 1) # if $USE_DEPOT=="1"
+  $(info "⚠️  Building using depot.dev") # Print a message
+  export BAKE_COMMAND = depot bake
+else
+  export BAKE_COMMAND = docker buildx bake
+endif
+
+
+# Build all Docker images and layers *locally*
+# Use this to test your changes
+default: docker-images layers
+
+
+# Build Docker images *locally*
+docker-images: docker-images-php-80 docker-images-php-81 docker-images-php-82
+docker-images-php-%:
+	PHP_VERSION=$* ${BAKE_COMMAND} --load
+
+
+# Build Lambda layers (zip files) *locally*
+layers: layer-php-80 layer-php-81 layer-php-82 layer-php-80-fpm layer-php-81-fpm layer-php-82-fpm
+	# Build the console layer only once (x86 and single PHP version)
+	@if [ ${CPU} = "x86" ]; then \
+		./utils/docker-zip-dir.sh bref/php-80-console-zip console; \
+	fi
+# This rule matches with a wildcard, for example `layer-php-80`.
+# The `$*` variable will contained the matched part, in this case `php-80`.
+layer-%:
+	./utils/docker-zip-dir.sh bref/${CPU_PREFIX}$* ${CPU_PREFIX}$*
+
+
+# Upload the layers to AWS Lambda
 # Uses the current AWS_PROFILE. Most users will not want to use this option
 # as this will publish all layers to all regions + publish all Docker images.
-everything:
-	$(MAKE) -f cpu-x86.Makefile everything
-	$(MAKE) -f cpu-arm.Makefile everything
+upload-layers: upload-layers-php-80 upload-layers-php-81 upload-layers-php-82
+	# Upload the console layer only once (x86 and single PHP version)
+	@if [ ${CPU} = "x86" ]; then \
+		LAYER_NAME=console $(MAKE) -C ./utils/lambda-publish publish-parallel; \
+	fi
+upload-layers-php-%:
+	# Upload the function layers to AWS
+	LAYER_NAME=${CPU_PREFIX}php-$* $(MAKE) -C ./utils/lambda-publish publish-parallel
+	# Upload the FPM layers to AWS
+	LAYER_NAME=${CPU_PREFIX}php-$*-fpm $(MAKE) -C ./utils/lambda-publish publish-parallel
 
-# Build Docker images *locally*
-docker-images:
-	$(MAKE) -f cpu-x86.Makefile docker-images
-	$(MAKE) -f cpu-arm.Makefile docker-images
-
-# Build Lambda layers (zip files) *locally* (will also build Docker images)
-layers:
-	$(MAKE) -f cpu-x86.Makefile layers
-	$(MAKE) -f cpu-arm.Makefile layers
-
-# Upload the layers to AWS Lambda (will also build Docker images and layers)
-upload-layers:
-	$(MAKE) -f cpu-x86.Makefile upload-layers
-	$(MAKE) -f cpu-arm.Makefile upload-layers
-
-# Build and publish Docker images to Docker Hub.
-# Only publishes the `latest` version.
-# This process is executed when a merge to `main` happens.
-# When a release tag is created, GitHub Actions
-# will download the latest images, tag them with the version number
-# and re-upload them with the right tag.
-upload-to-docker-hub:
-	$(MAKE) -f cpu-x86.Makefile upload-to-docker-hub
-	$(MAKE) -f cpu-arm.Makefile upload-to-docker-hub
-
-test:
-	$(MAKE) -f cpu-x86.Makefile test
-	$(MAKE) -f cpu-arm.Makefile test
-
-clean:
-	$(MAKE) -f cpu-x86.Makefile clean
-	$(MAKE) -f cpu-arm.Makefile clean
-
-.PHONY: layers
+
+# Publish Docker images to Docker Hub.
+upload-to-docker-hub: upload-to-docker-hub-php-80 upload-to-docker-hub-php-81 upload-to-docker-hub-php-82
+upload-to-docker-hub-php-%:
+	# While in beta we tag and push the `:2` version, later we'll push `:latest` as well
+	for image in \
+	  "bref/${CPU_PREFIX}php-$*" "bref/${CPU_PREFIX}php-$*-fpm" "bref/${CPU_PREFIX}php-$*-console" \
+	  "bref/${CPU_PREFIX}build-php-$*" "bref/${CPU_PREFIX}php-$*-fpm-dev"; \
+	do \
+		docker tag $$image $$image:2 ; \
+		docker push $$image:2 ; \
+	done
+	# TODO: when v2 becomes "latest", we should also push "latest" tags
+	# We could actually use `docker push --all-tags` at the end probably?
+
+
+test: test-80 test-81 test-82
+test-%:
+	cd tests && $(MAKE) test-$*
+
+
+clean: clean-80 clean-81 clean-82
+	# Clear the build cache, else all images will be rebuilt using cached layers
+	docker builder prune
+	# Remove zip files
+	rm -f output/${CPU_PREFIX}*.zip
+clean-%:
+	# Clean Docker images to force rebuilding them
+	docker image rm --force bref/${CPU_PREFIX}build-php-$* \
+		bref/${CPU_PREFIX}php-$* \
+		bref/${CPU_PREFIX}php-$*-zip \
+		bref/${CPU_PREFIX}php-$*-fpm \
+		bref/${CPU_PREFIX}php-$*-fpm-zip \
+		bref/${CPU_PREFIX}php-$*-fpm-dev \
+		bref/${CPU_PREFIX}php-$*-console