Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[2.0 backport] Port the release script from master #6083

Merged
merged 1 commit into from
Jul 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 20 additions & 16 deletions release/Dockerfile.in
Original file line number Diff line number Diff line change
@@ -1,29 +1,33 @@
FROM multiarch/debian-debootstrap:%TARGET_TAG%
FROM multiarch/alpine:%TARGET_TAG%
# May need configuration on the host:
# docker run --rm --privileged multiarch/qemu-user-static:register --reset
LABEL Description="opam release builds" Vendor="OCamlPro" Version="1.0"

RUN apt-get update && apt-get install bzip2 g++ make patch wget libltdl-dev --yes && apt-get clean --yes
RUN useradd -U --create-home opam
RUN apk add gcc g++ make coreutils openssl

ADD https://caml.inria.fr/pub/distrib/ocaml-4.04/ocaml-4.04.2.tar.gz /root/
RUN addgroup -S opam && adduser -S opam -G opam -s /bin/sh

ADD https://github.com/ocaml/ocaml/archive/refs/tags/%OCAMLV%.tar.gz /root/

WORKDIR /root
RUN tar xzf ocaml-4.04.2.tar.gz
WORKDIR ocaml-4.04.2
RUN tar xzf %OCAMLV%.tar.gz
WORKDIR ocaml-%OCAMLV%
RUN sed -i 's/gnueabi/*eabi/' configure
RUN sed -i 's/musl/musl*/' configure
RUN sed -i -e 's/NGROUPS_MAX/65536/' otherlibs/unix/getgroups.c
RUN ./configure %CONF% -prefix /usr/local
RUN make world opt.opt
RUN make install
RUN rm -rf /root/ocaml-4.04.2 /root/ocaml-4.04.2.tar.gz
RUN make "-j$(nproc)" && make install && rm -rf /root/ocaml-%OCAMLV% /root/ocaml-%OCAMLV%.tar.gz

RUN apk add patch

ENV PATH /usr/local/bin:/usr/bin:/bin
USER opam
VOLUME /src
WORKDIR /home/opam/
CMD tar xzf /src/opam-full-${VERSION}.tar.gz && \
cd opam-full-${VERSION} && \
echo "(${LINKING})" > src/client/linking.sexp && \
CMD tar xz >&2 && \
cd opam-full-${VERSION} >&2 && \
./configure --with-mccs && \
make lib-ext opam && \
strip opam && \
cp opam /src/opam-${VERSION}-${TARGET}
make lib-ext && \
echo "(${LINKING})" > src/client/linking.sexp && \
make opam >&2 && \
strip opam >&2 && \
cat opam
138 changes: 85 additions & 53 deletions release/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,71 +3,80 @@ VERSION = $(shell git describe $(TAG))
OPAM_VERSION = $(subst -,~,$(VERSION))
GIT_URL = ..

FULL_ARCHIVE_URL = https://github.com/ocaml/opam/releases/download/$(VERSION)/opam-full-$(VERSION).tar.gz

TARGETS = x86_64-linux i686-linux armhf-linux arm64-linux
# todo: x86_64-darwin

OCAMLV = 4.10.2
OCAMLV = 4.14.2
# currently hardcoded in Dockerfile.in
OCAML_URL = https://caml.inria.fr/pub/distrib/ocaml-$(basename $(OCAMLV))/ocaml-$(OCAMLV).tar.gz
OCAML_URL = https://github.com/ocaml/ocaml/archive/refs/tags/$(OCAMLV).tar.gz

HOST_OS = $(shell uname -s | tr A-Z a-z)
HOST_OS = $(shell uname -s | tr A-Z a-z | sed -e 's/_.*//' -e 's/darwin/macos/')
HOST = $(shell uname -m | sed 's/amd64/x86_64/')-$(HOST_OS)
OUTDIR = out/$(TAG)

# The equivalent of "which <cmd>". Taken from the GNU Make documentation
pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(PATH)))))

x86_64-linux: $(OUTDIR)/opam-$(VERSION)-x86_64-linux
i686-linux: $(OUTDIR)/opam-$(VERSION)-i686-linux
armhf-linux: $(OUTDIR)/opam-$(VERSION)-armhf-linux
arm64-linux: $(OUTDIR)/opam-$(VERSION)-arm64-linux
ppc64le-linux: $(OUTDIR)/opam-$(VERSION)-ppc64le-linux
s390x-linux: $(OUTDIR)/opam-$(VERSION)-s390x-linux

all: $(patsubst %,out/opam-$(VERSION)-%,$(TARGETS))

out/opam-full-$(VERSION).tar.gz:
mkdir -p out
cd out && curl -OfL $(FULL_ARCHIVE_URL) || { \
git clone $(GIT_URL) -b $(TAG) --depth 1 opam-full-$(VERSION); \
sed 's/^AC_INIT(opam,.*)/AC_INIT(opam,$(OPAM_VERSION))/' > \
opam-full-$(VERSION)/configure.ac.tmp; \
mv opam-full-$(VERSION)/configure.ac.tmp \
opam-full-$(VERSION)/configure.ac; \
$(MAKE) -C opam-full-$(VERSION) configure download-ext; \
tar cz --exclude-vcs opam-full-$(VERSION) -f $(notdir $@); \
rm -rf opam-full-$(VERSION); \
}
$(OUTDIR)/opam-full-$(VERSION).tar.gz:
mkdir -p "$(OUTDIR)"
git clone $(GIT_URL) -b $(TAG) "$(OUTDIR)/opam-full-$(VERSION)"
$(MAKE) -C "$(OUTDIR)/opam-full-$(VERSION)" OCAML=$(call pathsearch,ocaml) download-ext
cd "$(OUTDIR)" && tar cz -f $(notdir $@) --exclude .git opam-full-$(VERSION)
rm -rf "$(OUTDIR)/opam-full-$(VERSION)"

build/Dockerfile.x86_64-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/amd64-jessie/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/x86_64-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.i686-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/i386-jessie/g' $^ | sed 's/%CONF%/-host i686-linux/g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/x86-v3.13/g' -e 's/%CONF%/-build i586-alpine-linux-musl/g' $^ >$@

# Need to lie about gnueabihf instead of musleabihf, because of a ./configure bug
build/Dockerfile.armhf-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/armhf-jessie/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/armv7-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.arm64-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/arm64-jessie/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/arm64-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.ppc64le-linux: Dockerfile.in
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/ppc64le-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.s390x-linux: Dockerfile.in
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/s390x-v3.13/g' -e 's/%CONF%//g' $^ >$@


build/%.image: build/Dockerfile.%
docker build -t opam-build-$* -f $^ build
touch $@

# Actually, this is for debian 8 jessie, and varies wildly
# Actually, this is for alpine 3.13, and varies
CLINKING_linux = \
-Wl,-Bstatic \
-lunix -lmccs_stubs -lmccs_glpk_stubs \
-lstdc++ \
-Wl,-Bdynamic \
-static-libgcc
-static-libgcc \
-static
# -Wl,-Bdynamic

CLINKING_darwin = \
CLINKING_macos = \
-lunix -lmccs_stubs -lmccs_glpk_stubs \
-lstdc++

CLINKING_openbsd = $(CLINKING_darwin)
CLINKING_openbsd = $(CLINKING_macos)
CLINKING_freebsd = $(CLINKING_macos)

LINKING = (-noautolink $(patsubst %,-cclib %,$(CLINKING_$(1))))

EXPORTS_openbsd = \
CPATH=/usr/local/include: \
LIBRARY_PATH=/usr/local/lib: \

EXPORTS_freebsd = \
CPATH=/usr/local/include: \
LIBRARY_PATH=/usr/local/lib: \

%: opam-$(VERSION)-%

opam-$(VERSION)-%: out/opam-$(VERSION)-%
opam-$(VERSION)-%: $(OUTDIR)/opam-$(VERSION)-%
ln -sf $^ $@

# host: opam-$(VERSION)-$(HOST)
Expand All @@ -76,40 +85,44 @@ opam-$(VERSION)-%: out/opam-$(VERSION)-%
build/$(HOST).env:
mkdir -p build/$(HOST)
cd build/$(HOST) && curl -OL $(OCAML_URL)
cd build/$(HOST) && tar xzf ocaml-$(OCAMLV).tar.gz
cd build/$(HOST) && tar xzf $(OCAMLV).tar.gz
cd build/$(HOST)/ocaml-$(OCAMLV) && \
./configure -prefix $(shell pwd)/build/$(HOST) && \
$(MAKE) world opt.opt && \
./configure --prefix "$(shell pwd)/build/$(HOST)" \
--disable-debug-runtime --disable-debugger --disable-instrumented-runtime \
--disable-ocamldoc --disable-stdlib-manpages --disable-ocamltest && \
$(MAKE) -j$(JOBS) && \
$(MAKE) install
rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/ocaml-$(OCAMLV).tar.gz
rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/$(OCAMLV).tar.gz
touch $@

# Actually builds out/opam-$(VERSION)-$(HOST), but we don't want to override the
# Actually builds $(OUTDIR)/opam-$(VERSION)-$(HOST), but we don't want to override the
# rule that goes through a container
host: out/opam-full-$(VERSION).tar.gz build/$(HOST).env
host: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/$(HOST).env
rm -rf build/opam-full-$(VERSION)
cd build && tar xzf ../$<
( export \
PATH=$(shell pwd)/build/$(HOST)/bin:$$PATH \
MAKE=$(MAKE) \
$(EXPORTS_$(HOST_OS)); \
cd build/opam-full-$(VERSION) && \
./configure && \
./configure --with-mccs && \
$(MAKE) lib-ext && \
echo "$(call LINKING,$(HOST_OS))" >src/client/linking.sexp && \
$(MAKE) lib-ext DUNE_ARGS="--root=`pwd`"; \
$(MAKE) opam DUNE_ARGS="--root=`pwd`"; \
$(MAKE) opam; \
)
cp build/opam-full-$(VERSION)/opam out/opam-$(VERSION)-$(HOST)
strip out/opam-$(VERSION)-$(HOST)
cp build/opam-full-$(VERSION)/opam $(OUTDIR)/opam-$(VERSION)-$(HOST)
strip $(OUTDIR)/opam-$(VERSION)-$(HOST)
$(OUTDIR)/opam-$(VERSION)-$(HOST) --version
rm -rf build/opam-full-$(VERSION)

# Containerised builds
out/opam-$(VERSION)-%-linux: build/%-linux.image out/opam-full-$(VERSION).tar.gz
docker run --rm -v `pwd`/out:/src \
$(OUTDIR)/opam-$(VERSION)-%-linux: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/%-linux.image
docker run --rm -i \
-e "VERSION=$(VERSION)" \
-e "TARGET=$*-linux" \
-e "LINKING=$(call LINKING,$(HOST_OS))" \
opam-build-$*-linux
-e "LINKING=$(call LINKING,linux)" \
opam-build-$*-linux \
<$< >$@

clean:
rm -rf build
Expand All @@ -119,9 +132,28 @@ distclean: clean

REMOTE_DIR = /tmp/opam-release
REMOTE_MAKE = make
remote: out/opam-full-$(VERSION).tar.gz
ssh "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/out"
scp Makefile "$(REMOTE):$(REMOTE_DIR)/"
scp "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
ssh "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && ulimit -s 8192 && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
scp "$(REMOTE):$(REMOTE_DIR)/out/opam-$(VERSION)*" out/
REMOTE_SHELL = /bin/sh
SSH = ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
SCP = scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
remote: $(OUTDIR)/opam-full-$(VERSION).tar.gz
$(SSH) "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/$(OUTDIR)"
$(SCP) Makefile "$(REMOTE):$(REMOTE_DIR)/"
$(SCP) "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
$(SSH) "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && ulimit -s 8192 && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
$(SCP) "$(REMOTE):$(REMOTE_DIR)/$(OUTDIR)/opam-$(VERSION)*" "$(OUTDIR)/"

SSH_USER = root
ULIMIT = ulimit -s 8192 &&
qemu: $(OUTDIR)/opam-full-$(VERSION).tar.gz
$(SSH) -p "$(QEMU_PORT)" $(SSH_USER)@localhost "$(REMOTE_SHELL) -c \"mkdir -p $(REMOTE_DIR)/$(OUTDIR)\""
$(SCP) -P "$(QEMU_PORT)" Makefile "$(SSH_USER)@localhost:$(REMOTE_DIR)/"
$(SCP) -P "$(QEMU_PORT)" "$^" "$(SSH_USER)@localhost:$(REMOTE_DIR)/$^"
$(SSH) -p "$(QEMU_PORT)" $(SSH_USER)@localhost "$(REMOTE_SHELL) -c \"cd $(REMOTE_DIR) && $(ULIMIT) $(REMOTE_MAKE) host JOBS=$(JOBS) TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)\""
$(SCP) -P "$(QEMU_PORT)" "$(SSH_USER)@localhost:$(REMOTE_DIR)/$(OUTDIR)/opam-$(VERSION)*" "$(OUTDIR)/"

macos-local: $(OUTDIR)/opam-full-$(VERSION).tar.gz
LOCAL_RELEASE_DIR=$(shell mktemp -d); \
mkdir -p "$${LOCAL_RELEASE_DIR}/${OUTDIR}" && \
cp Makefile "$^" "$${LOCAL_RELEASE_DIR}" && \
cd "$${LOCAL_RELEASE_DIR}" && ulimit -s 8192 && arch -arch $(MACOS_ARCH) make host JOBS=$(JOBS) TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV) && \
cp $${LOCAL_RELEASE_DIR}/$(OUTDIR)/opam-$(VERSION)* "$(GIT_URL)/release/$(OUTDIR)/"
54 changes: 54 additions & 0 deletions release/readme.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Steps to follow for each release

## Finalise opam code for release
* update version in all the opam files and in configure.ac
* run `make configure` to regenerate `./configure` [checked by github actions]
* update copyright headers
* if you're releasing the first final release of a new branch (e.g. 2.2.0): make sure `root_version` in OpamFile.ml is set to the final release number (e.g. for 2.2.0, root_version should be 2.2). Make sure that opamFormatUpgrade.ml also contains an upgrade function from the previous version (that function will most likely be empty)
* run `make tests`, `opam-rt` [checked by github actions]
* update the CHANGE file: take `master_changes.md` content to fill it

## Github release

[ once bump version & changes PRs merged ]
* tag the release (git tag -am 2.2.0 2.2.0; git push origin 2.2.0)
* /!\ Once the tag pushed, it can be updated [different commit] only in case of severe issue
* create a release (or prerelease if intermediate release) draft on github based on your tag (https://github.com/ocaml/opam/releases/new)
* Make sure the repository is in the correct state: `git switch --detach <tag>`
* launch docker using the Docker GUI macOS app
* generate opam artifacts, using `release/release.sh <tag>` from a macOS/arm64 machine, it requires to have Docker and QEMU installed (see below device requirements)
* generate the signatures using `release/sign.sh <tag>`
* add releases notes (content of `master_changes.md`) in the release draft
* upload everything from `release/out/<tag>`
* finalise the release (publish)

## Publish the release

* add hashes in `install.sh` (and check signatures)
* publish opam packages in opam-repository (and add `flags: avoid-version` and `available: opam-version >= "2.1.0"` to each packages if this is not a stable version)
* update versions (and messages, if necessary) in https://github.com/ocaml/opam-repository/blob/master/repo

## Announce!

* a blog entry in opam.ocaml.org
* a announcement in discuss.ocaml.org
* copy the blog entry from opam.ocaml.org for https://github.com/ocaml/ocaml.org/tree/main/data/changelog/opam
* announce the release on the OCaml Discord server


## After release

* Bump the version with a `~dev` at the end (e.g. `2.2.0~alpha~dev`)
* Check if reftests needs an update
* Bring the changes to the changelog (CHANGES) from the branch of the release to the `master` branch

### On a release candidate
* create a branch to a `x.y` for rc's and the final release

---

## Device requirements
* Mac M1 or above with Rosetta2
* brew dependencies: git, gpg, qemu>=8.1.0, docker>=24.0.0, md5sha1sum
* opam repo with the tag fetched
* Have the secret key available
75 changes: 53 additions & 22 deletions release/release.sh
Original file line number Diff line number Diff line change
@@ -1,32 +1,63 @@
#!/usr/bin/env bash
set -uex

# This script is expected to run on Linux with docker available, and to have
# three remotes "some-osx-x86", "some-osx-arm" and "some-openbsd", with the
# corresponding OSes, ocaml deps installed
unset $(env | cut -d= -f1 | grep -Fvx HOME | grep -Fvx PWD | grep -Fvx USER | grep -Fvx SHELL | grep -Fvx TERM)
export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin:/usr/local/bin

LC_ALL=C
DIR=$(dirname $0)
cd "$DIR"
if [[ $# -eq 0 || "x$1" =~ "x-" ]]; then
echo "Usage: $0 TAG [archive|builds]"
usage() {
echo "Usage: $0 TAG"
exit 1
}

if [[ $# -lt 1 || $# -gt 1 || "x$1" =~ "x-" ]]; then
usage
fi

TAG="$1"; shift
TAG="$1"
shift

if [[ $# -eq 0 || " $* " =~ " archive " ]]; then
make TAG="$TAG" GIT_URL="https://github.com/ocaml/opam.git" "out/opam-full-$TAG.tar.gz"
( cd out && git-upload-release ocaml opam "$TAG" "opam-full-$TAG.tar.gz"; )
if test "$(uname -s)" != Darwin -o "$(uname -m)" != arm64; then
echo "This script is required to be run on macOS/arm64"
exit 1
fi

if [[ $# -eq 0 || " $* " =~ " builds " ]]; then
make TAG="$TAG" all &
make TAG="$TAG" remote REMOTE=some-osx-x86 REMOTE_DIR=opam-release &
make TAG="$TAG" remote REMOTE=some-osx-arm REMOTE_DIR=opam-release &
make TAG="$TAG" remote REMOTE=some-openbsd REMOTE_MAKE=gmake REMOTE_DIR=opam-release &
wait
cd out && for f in opam-$TAG-*; do
git-upload-release ocaml opam "$TAG" $f
done
fi
DIR=$(dirname $0)
cd "$DIR"

LC_ALL=C
CWD=$(pwd)
JOBS=$(sysctl -n hw.ncpu)
SSH="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

OUTDIR="out/$TAG"
mkdir -p "$OUTDIR"

qemu_build() {
local port=$1
local image=$2
local install=$3
local make=$4
local arch=$5

if ! ${SSH} -p "${port}" root@localhost true; then
qemu-img convert -O raw "./qemu-base-images/${image}.qcow2" "./qemu-base-images/${image}.raw"
"qemu-system-${arch}" -drive "file=./qemu-base-images/${image}.raw,format=raw" -nic "user,hostfwd=tcp::${port}-:22" -m 2G -smp "${JOBS}" &
sleep 60
fi
${SSH} -p "${port}" root@localhost "${install}"
make TAG="$TAG" JOBS="${JOBS}" qemu QEMU_PORT="${port}" REMOTE_MAKE="${make}" REMOTE_DIR="opam-release-$TAG"
${SSH} -p "${port}" root@localhost "shutdown -p now"
}

make JOBS="${JOBS}" TAG="$TAG" "${OUTDIR}/opam-full-$TAG.tar.gz"
make JOBS="${JOBS}" TAG="$TAG" x86_64-linux
make JOBS="${JOBS}" TAG="$TAG" i686-linux
make JOBS="${JOBS}" TAG="$TAG" armhf-linux
make JOBS="${JOBS}" TAG="$TAG" arm64-linux
make JOBS="${JOBS}" TAG="$TAG" ppc64le-linux
make JOBS="${JOBS}" TAG="$TAG" s390x-linux
[ -f "${OUTDIR}/opam-$TAG-x86_64-macos" ] || make TAG="$TAG" JOBS="${JOBS}" macos-local MACOS_ARCH=x86_64 REMOTE_DIR=opam-release-$TAG GIT_URL="$CWD/.."
[ -f "${OUTDIR}/opam-$TAG-arm64-macos" ] || make TAG="$TAG" JOBS="${JOBS}" macos-local MACOS_ARCH=arm64 REMOTE_DIR=opam-release-$TAG GIT_URL="$CWD/.."
[ -d ./qemu-base-images ] || git clone https://gitlab.com/kit-ty-kate/qemu-base-images.git
[ -f "${OUTDIR}/opam-$TAG-x86_64-openbsd" ] || qemu_build 9999 OpenBSD-7.4-amd64 "pkg_add gmake curl bzip2" gmake x86_64
[ -f "${OUTDIR}/opam-$TAG-x86_64-freebsd" ] || qemu_build 9998 FreeBSD-13.2-RELEASE-amd64 "pkg install -y gmake curl bzip2" gmake x86_64
Loading
Loading