diff --git a/.builders/build.py b/.builders/build.py index 72093d51b3f8b..838edb70e9000 100644 --- a/.builders/build.py +++ b/.builders/build.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import os import shutil import subprocess import sys @@ -65,7 +66,84 @@ def read_dependencies() -> dict[str, list[str]]: def build_macos(): - sys.exit('macOS is not supported') + parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False) + parser.add_argument('output_dir') + parser.add_argument('--python', default='3') + parser.add_argument('--builder-root', required=True, + help='Path to a folder where things will be installed during builder setup.') + parser.add_argument('--skip-setup', default=False, action='store_true', + help='Skip builder setup, assuming it has already been set up.') + args = parser.parse_args() + + context_path = HERE / 'images' / 'macos' + builder_root = Path(args.builder_root).absolute() + builder_root.mkdir(exist_ok=True) + + with temporary_directory() as temp_dir: + mount_dir = temp_dir / 'mnt' + mount_dir.mkdir() + + build_context_dir = shutil.copytree(context_path, mount_dir / 'build_context', dirs_exist_ok=True) + # Copy utilities shared by multiple images + for entry in context_path.parent.iterdir(): + if entry.is_file(): + shutil.copy2(entry, build_context_dir) + + # Folders required by the build_wheels script + wheels_dir = mount_dir / 'wheels' + wheels_dir.mkdir() + built_wheels_dir = wheels_dir / 'built' + built_wheels_dir.mkdir() + external_wheels_dir = wheels_dir / 'external' + external_wheels_dir.mkdir() + + dependency_file = mount_dir / 'requirements.in' + dependency_file.write_text('\n'.join(chain.from_iterable(read_dependencies().values()))) + shutil.copy(HERE / 'deps' / 'build_dependencies.txt', mount_dir) + shutil.copytree(HERE / 'scripts', mount_dir / 'scripts') + shutil.copytree(HERE / 'patches', mount_dir / 'patches') + + prefix_path = builder_root / 'prefix' + env = { + **os.environ, + 'DD_MOUNT_DIR': mount_dir, + 'DD_ENV_FILE': mount_dir / '.env', + # Paths to pythons + 'DD_PY3_BUILDENV_PATH': builder_root / 'py3' / 'bin' / 'python', + 'DD_PY2_BUILDENV_PATH': builder_root / 'py2' / 'bin' / 'python', + # Path where we'll install libraries that we build + 'DD_PREFIX_PATH': prefix_path, + # Common compilation flags + 'LDFLAGS': f'-L{prefix_path}/lib', + 'CFLAGS': f'-I{prefix_path}/include -O2', + # Build command for extra platform-specific build steps + 'DD_BUILD_COMMAND': f'bash {build_context_dir}/extra_build.sh' + } + + if not args.skip_setup: + check_process( + ['bash', str(HERE / 'images' / 'macos' / 'builder_setup.sh')], + env=env, + cwd=builder_root, + ) + + check_process( + [os.environ['DD_PYTHON3'], str(mount_dir / 'scripts' / 'build_wheels.py'), '--python', args.python], + env=env, + cwd=builder_root, + ) + + output_dir = Path(args.output_dir) + if output_dir.is_dir(): + shutil.rmtree(output_dir) + + # Move wheels to the output directory + wheels_dir = mount_dir / 'wheels' + shutil.move(wheels_dir, output_dir / 'wheels') + + # Move the final requirements file to the output directory + final_requirements = mount_dir / 'frozen.txt' + shutil.move(final_requirements, output_dir) def build_image(): diff --git a/.builders/images/linux-x86_64/install-from-source.sh b/.builders/images/install-from-source.sh similarity index 98% rename from .builders/images/linux-x86_64/install-from-source.sh rename to .builders/images/install-from-source.sh index b510aa0d9d210..4bac1dc1b1a4f 100644 --- a/.builders/images/linux-x86_64/install-from-source.sh +++ b/.builders/images/install-from-source.sh @@ -10,8 +10,7 @@ # Optional: # - CONFIGURE_SCRIPT: Alternative to the default ./configure - -set -exu +set -euxo pipefail url=${DOWNLOAD_URL//'{{version}}'/${VERSION}} relative_path=${RELATIVE_PATH//'{{version}}'/${VERSION}} diff --git a/.builders/images/macos/builder_setup.sh b/.builders/images/macos/builder_setup.sh new file mode 100644 index 0000000000000..c3b90b250a4a8 --- /dev/null +++ b/.builders/images/macos/builder_setup.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip +"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv +"${DD_PYTHON3}" -m virtualenv py3 + +"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip +"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv +"${DD_PYTHON2}" -m virtualenv py2 + +"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "${DD_MOUNT_DIR}/build_context/runner_dependencies.txt" + +# Install always with our own prefix path +cp "${DD_MOUNT_DIR}/build_context/install-from-source.sh" . +install-from-source() { + bash "install-from-source.sh" --prefix="${DD_PREFIX_PATH}" "$@" +} + +# mqi +IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit +curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg +sudo installer -pkg /tmp/mq_client.pkg -target / +rm -rf /tmp/mq_client.pkg + +# openssl +DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \ +VERSION="3.0.12" \ +SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \ +RELATIVE_PATH="openssl-{{version}}" \ +CONFIGURE_SCRIPT="./config" \ + install-from-source \ + -fPIC shared \ + no-module \ + no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost + +# libxml & libxslt for lxml +DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \ +VERSION="2.10.3" \ +SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \ +RELATIVE_PATH="libxml2-{{version}}" \ + install-from-source \ + --without-iconv \ + --without-python \ + --without-icu \ + --without-debug \ + --without-mem-debug \ + --without-run-debug \ + --without-legacy \ + --without-catalog \ + --without-docbook \ + --disable-static + +DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \ +VERSION="1.1.37" \ +SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \ +RELATIVE_PATH="libxslt-{{version}}" \ + install-from-source \ + --with-libxml-prefix="${DD_PREFIX_PATH}" \ + --without-python \ + --without-crypto \ + --without-profiler \ + --without-debugger \ + --disable-static + +# curl +DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \ +VERSION="8.4.0" \ +SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \ +RELATIVE_PATH="curl-{{version}}" \ + install-from-source \ + --disable-manual \ + --disable-debug \ + --enable-optimize \ + --disable-static \ + --disable-ldap \ + --disable-ldaps \ + --disable-rtsp \ + --enable-proxy \ + --disable-dependency-tracking \ + --enable-ipv6 \ + --without-libidn \ + --without-gnutls \ + --without-librtmp \ + --without-libssh2 \ + --with-ssl="${DD_PREFIX_PATH}" +# Remove the binary installed so that we consistenly use the same original `curl` binary +rm "${DD_PREFIX_PATH}/bin/curl" + +# Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support +# Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation +# with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being +# compatible with that in the Agent, the rest shouldn't matter much +DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \ +VERSION="0.9.29" \ +SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \ +RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \ +CONFIGURE_SCRIPT="true" \ + install-from-source +DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \ +VERSION="2.1.28" \ +SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \ +RELATIVE_PATH="cyrus-sasl-{{version}}" \ + install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework diff --git a/.builders/images/macos/extra_build.sh b/.builders/images/macos/extra_build.sh new file mode 100644 index 0000000000000..b37119be5cc8d --- /dev/null +++ b/.builders/images/macos/extra_build.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -exu + +# Packages which must be built from source +always_build=() + +if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then + # confluent-kafka and librdkafka need to be compiled from source to get kerberos support + # The librdkafka version needs to stay in sync with the confluent-kafka version, + # thus we extract the version from the requirements file. + kafka_version=$(grep 'confluent-kafka==' "${DD_MOUNT_DIR}/requirements.in" | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/') + DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \ + VERSION="${kafka_version}" \ + SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \ + RELATIVE_PATH="librdkafka-{{version}}" \ + bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl + + always_build+=("confluent-kafka") +fi + +# Empty arrays are flagged as unset when using the `-u` flag. This is the safest way to work around that +# (see https://stackoverflow.com/a/61551944) +pip_no_binary=${always_build[@]+"${always_build[@]}"} +if [[ "$pip_no_binary" ]]; then + # If there are any packages that must always be built, inform pip + echo "PIP_NO_BINARY=\"${pip_no_binary}\"" >> $DD_ENV_FILE +fi diff --git a/.builders/images/runner_dependencies.txt b/.builders/images/runner_dependencies.txt index e3572efcf1cbe..192632213f618 100644 --- a/.builders/images/runner_dependencies.txt +++ b/.builders/images/runner_dependencies.txt @@ -2,3 +2,4 @@ python-dotenv==1.0.0 urllib3==2.1.0 auditwheel==5.4.0; sys_platform == 'linux' delvewheel==1.5.2; sys_platform == 'win32' +delocate==0.10.7; sys_platform == 'darwin' diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py index e3645b0a03420..5f12950d9ce44 100644 --- a/.builders/scripts/build_wheels.py +++ b/.builders/scripts/build_wheels.py @@ -11,6 +11,7 @@ from dotenv import dotenv_values from utils import extract_metadata, normalize_project_name + if sys.platform == 'win32': PY3_PATH = Path('C:\\py3\\Scripts\\python.exe') PY2_PATH = Path('C:\\py2\\Scripts\\python.exe') @@ -26,10 +27,10 @@ def path_to_uri(path: str) -> str: else: import shlex - PY3_PATH = Path('/py3/bin/python') - PY2_PATH = Path('/py2/bin/python') - MOUNT_DIR = Path('/home') - ENV_FILE = Path('/.env') + PY3_PATH = Path(os.environ.get('DD_PY3_BUILDENV_PATH', '/py3/bin/python')) + PY2_PATH = Path(os.environ.get('DD_PY2_BUILDENV_PATH', '/py2/bin/python')) + MOUNT_DIR = Path(os.environ.get('DD_MOUNT_DIR', '/home')) + ENV_FILE = Path(os.environ.get('DD_ENV_FILE', '/.env')) def join_command_args(args: list[str]) -> str: return shlex.join(args) diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py index 1adb733b7f543..6e2d6e2f5832c 100644 --- a/.builders/scripts/repair_wheels.py +++ b/.builders/scripts/repair_wheels.py @@ -116,9 +116,67 @@ def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None: sys.exit(process.returncode) +def repair_darwin(source_dir: str, built_dir: str, external_dir: str) -> None: + from delocate import delocate_wheel + exclusions = [re.compile(s) for s in [ + # pymqi + r'pymqe\.cpython-\d+-darwin\.so', + # confluent_kafka + # We leave cyrus-sasl out of the wheel because of the complexity involved in bundling it portably. + # This means the confluent-kafka wheel will have a runtime dependency on this library + r'libsasl2.\d\.dylib', + # Whitelisted libraries based on the health check default whitelist that we have on omnibus: + # https://github.com/DataDog/omnibus-ruby/blob/044a81fa1b0f1c50fc7083cb45e7d8f90d96905b/lib/omnibus/health_check.rb#L133-L152 + # We use that instead of the more relaxed policy that delocate_wheel defaults to. + r'libobjc\.A\.dylib', + r'libSystem\.B\.dylib', + # Symlink of the previous one + r'libgcc_s\.1\.dylib', + r'CoreFoundation', + r'CoreServices', + r'Tcl$', + r'Cocoa$', + r'Carbon$', + r'IOKit$', + r'Kerberos', + r'Tk$', + r'libutil\.dylib', + r'libffi\.dylib', + r'libncurses\.5\.4\.dylib', + r'libiconv', + r'libstdc\+\+\.6\.dylib', + r'libc\+\+\.1\.dylib', + r'^/System/Library/', + r'libz\.1\.dylib', + ]] + + def copy_filt_func(libname): + return not any(excl.search(libname) for excl in exclusions) + + for wheel in iter_wheels(source_dir): + print(f'--> {wheel.name}') + if not wheel_was_built(wheel): + print('Using existing wheel') + shutil.move(wheel, external_dir) + continue + + copied_libs = delocate_wheel( + str(wheel), + os.path.join(built_dir, os.path.basename(wheel)), + copy_filt_func=copy_filt_func, + ) + print('Repaired wheel') + if copied_libs: + print('Libraries copied into the wheel:') + print('\n'.join(copied_libs)) + else: + print('No libraries were copied into the wheel.') + + REPAIR_FUNCTIONS = { 'linux': repair_linux, 'win32': repair_windows, + 'darwin': repair_darwin, } diff --git a/.github/workflows/build-deps.yml b/.github/workflows/build-deps.yml index 13a3dd4413d3e..a027f76dd33f9 100644 --- a/.github/workflows/build-deps.yml +++ b/.github/workflows/build-deps.yml @@ -98,3 +98,42 @@ jobs: - name: Publish image if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true' run: docker push ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }} + + build-deps-macos: + name: Target macOS + runs-on: macos-12 + steps: + - name: Set up environment + run: | + # We remove everything that comes pre-installed via brew to avoid depending or shipping stuff that + # comes in the runner through brew to better control what might get shipped in the wheels via `delocate` + brew remove --force --ignore-dependencies $(brew list --formula) + brew install coreutils + + - name: Set up Python + env: + # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel + PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg" + PYTHON2_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg" + run: | + curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg + sudo installer -pkg python3.pkg -target / + + curl "$PYTHON2_DOWNLOAD_URL" -o python2.pkg + sudo installer -pkg python2.pkg -target / + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run the build + env: + DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3" + DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python" + # This sets the minimum mac os version compatible for all built artifacts + MACOSX_DEPLOYMENT_TARGET: "10.12" + run: | + ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt + + mkdir builder_root + ${DD_PYTHON3} .builders/build.py --builder-root builder_root --python 3 out_py3 + ${DD_PYTHON3} .builders/build.py --builder-root builder_root --skip-setup --python 2 out_py2