From a1008dcf6318b90e7336307b106d7eadab061d93 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Fri, 12 Jan 2024 15:37:28 +0100
Subject: [PATCH 01/18] Implement MacOS build for wheels

---
 .builders/build.py                            | 36 ++++++++++++-
 .builders/images/linux-x86_64/Dockerfile      |  3 ++
 .builders/images/macos/build.sh               | 51 +++++++++++++++++++
 .builders/images/macos/extra_build.sh         | 15 ++++++
 .builders/images/macos/install-from-source.sh | 29 +++++++++++
 .builders/scripts/build_wheels.py             |  6 +--
 6 files changed, 136 insertions(+), 4 deletions(-)
 create mode 100644 .builders/images/macos/build.sh
 create mode 100644 .builders/images/macos/extra_build.sh
 create mode 100644 .builders/images/macos/install-from-source.sh

diff --git a/.builders/build.py b/.builders/build.py
index 72093d51b3f8b..4fdd235252958 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import os
 import shutil
 import subprocess
 import sys
@@ -65,7 +66,40 @@ def read_dependencies() -> dict[str, list[str]]:
 
 
 def build_macos():
-    sys.exit('macOS is not supported')
+    parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
+    parser.add_argument('output_dir')
+    parser.add_argument('--python', default='3')
+    parser.add_argument('--cache-dir')
+    args = parser.parse_args()
+
+    context_path = HERE / 'images' / 'macos'
+
+    with temporary_directory() as temp_dir:
+        build_context_dir = shutil.copytree(context_path, temp_dir, dirs_exist_ok=True)
+        # Copy utilities shared by multiple images
+        for entry in context_path.parent.iterdir():
+            if entry.is_file():
+                shutil.copy2(entry, build_context_dir)
+
+        mount_dir = temp_dir / 'mnt'
+        mount_dir.mkdir()
+
+        dependency_file = mount_dir / 'requirements.in'
+        dependency_file.write_text('\n'.join(chain.from_iterable(read_dependencies().values())))
+        shutil.copy(HERE / 'deps' / 'build_dependencies.txt', mount_dir)
+        shutil.copytree(HERE / 'scripts', mount_dir / 'scripts')
+        shutil.copytree(HERE / 'patches', mount_dir / 'patches')
+
+        env = {
+            **os.environ,
+            'PREFIX_CACHE': args.cache_dir,
+            'MOUNT_HOME': mount_dir,
+        }
+        check_process(
+            ['bash', str(HERE / 'images' / 'macos' / 'build.sh'), '--python', args.python],
+            env=env,
+            cwd=build_context_dir,
+        )
 
 
 def build_image():
diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile
index 8e9846c46800e..cb147e9261190 100644
--- a/.builders/images/linux-x86_64/Dockerfile
+++ b/.builders/images/linux-x86_64/Dockerfile
@@ -21,6 +21,7 @@ ENV LD_LIBRARY_PATH="/opt/python/${PYTHON2_VERSION}/lib:${LD_LIBRARY_PATH}"
 # Set up virtual environment for Python 2
 RUN /opt/python/${PYTHON2_VERSION}/bin/python -m pip install --no-warn-script-location virtualenv \
  && /opt/python/${PYTHON2_VERSION}/bin/python -m virtualenv /py2
+ENV PY2_BUILDENV_PATH=/py2/bin/python
 
 # openssl
 RUN yum install -y perl-IPC-Cmd && \
@@ -50,6 +51,7 @@ ENV LD_LIBRARY_PATH="/opt/python/${PYTHON_VERSION}/lib:${LD_LIBRARY_PATH}"
 RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location --upgrade pip \
  && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location virtualenv \
  && /opt/python/${PYTHON_VERSION}/bin/python3 -m virtualenv /py3
+ENV PY3_BUILDENV_PATH=/py3/bin/python
 
 # Rust toolchain
 ENV RUST_VERSION="nightly-2022-05-15"
@@ -165,5 +167,6 @@ RUN python3 -m pip install --no-warn-script-location -r /runner_dependencies.txt
 
 COPY build_script.sh /build_script.sh
 ENV DD_BUILD_COMMAND="bash /build_script.sh"
+ENV MOUNT_DIR="/home"
 
 ENTRYPOINT ["python3", "/home/scripts/build_wheels.py"]
diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
new file mode 100644
index 0000000000000..f26a7f6e06a6f
--- /dev/null
+++ b/.builders/images/macos/build.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+"${PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
+"${PYTHON3}" -m pip install --no-warn-script-location virtualenv
+"${PYTHON3}" -m virtualenv py3
+export PY3_BUILDENV_PATH="$(pwd)/py3/bin/python"
+
+"${PYTHON2}" -m pip install --no-warn-script-location --upgrade pip
+"${PYTHON2}" -m pip install --no-warn-script-location virtualenv
+"${PYTHON2}" -m virtualenv py2
+export PY2_BUILDENV_PATH="$(pwd)/py2/bin/python"
+
+export PREFIX_PATH="$(pwd)/prefix"
+export LDFLAGS="-Wl,-rpath,${PREFIX_PATH}/lib -L${PREFIX_PATH}/lib"
+export CFLAGS="-I${PREFIX_PATH}/include -O2"
+export PATH="${PREFIX_PATH}/bin:${PATH}"
+
+"${PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
+
+# Restore cache if it exists
+if [[ -n ${PREFIX_CACHE:-} && -d ${PREFIX_CACHE:-} ]]; then
+    cp -r "${PREFIX_CACHE}" "${PREFIX_PATH}"
+else
+    # openssl
+    DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
+                VERSION="3.0.12" \
+                SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
+                RELATIVE_PATH="openssl-{{version}}" \
+                CONFIGURE_SCRIPT="./config" \
+                bash install-from-source.sh --prefix="${PREFIX_PATH}" \
+                -fPIC shared \
+                no-module \
+                no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
+
+    # postgresql
+    DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
+                VERSION="16.0" \
+                SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
+                RELATIVE_PATH="postgresql-{{version}}" \
+                bash install-from-source.sh --prefix="${PREFIX_PATH}" \
+                --with-openssl --without-readline  --without-icu
+
+    # Cache everything under prefix
+    cp -r "${PREFIX_PATH}" "${PREFIX_CACHE}"
+fi
+
+export DD_BUILD_COMMAND="bash $(pwd)/extra_build.sh"
+
+"${PYTHON3}" "${MOUNT_HOME}/scripts/build_wheels.py" "$@"
diff --git a/.builders/images/macos/extra_build.sh b/.builders/images/macos/extra_build.sh
new file mode 100644
index 0000000000000..f75015475da15
--- /dev/null
+++ b/.builders/images/macos/extra_build.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -exu
+
+if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
+    # confluent-kafka and librdkafka need to be compiled from source to get kerberos support
+    # The librdkafka version needs to stay in sync with the confluent-kafka version,
+    # thus we extract the version from the requirements file.
+    kafka_version=$(grep 'confluent-kafka==' "${MOUNT_HOME}/requirements.in" | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/')
+    DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
+      VERSION="${kafka_version}" \
+      SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \
+      RELATIVE_PATH="librdkafka-{{version}}" \
+      bash install-from-source.sh --prefix="${PREFIX_PATH}" # --enable-sasl
+fi
diff --git a/.builders/images/macos/install-from-source.sh b/.builders/images/macos/install-from-source.sh
new file mode 100644
index 0000000000000..fd07f505624bb
--- /dev/null
+++ b/.builders/images/macos/install-from-source.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Download, unzip, compile and install a package.
+# Arguments to this script are passed directly to the configure script
+# Required env variables:
+# - DOWNLOAD_URL. Use `{{version}}` as a placeholder to be replaced by the actual version
+# - VERSION
+# - SHA256
+# - RELATIVE_PATH. Set to the relative path in the archive where the source needs to built from.
+#    Can also use the `{{version}}` placeholder for replacemnet.
+# Optional:
+# - CONFIGURE_SCRIPT: Alternative to the default ./configure
+
+set -exu
+
+url=${DOWNLOAD_URL//'{{version}}'/${VERSION}}
+relative_path=${RELATIVE_PATH//'{{version}}'/${VERSION}}
+archive_name="$(basename ${url})"
+workdir="/tmp/build-${archive_name}"
+mkdir -p "${workdir}"
+
+curl "${url}" -Lo "${workdir}/${archive_name}"
+echo "${SHA256}  ${workdir}/${archive_name}" | sha256sum --check
+tar -C "${workdir}" -xf "${workdir}/${archive_name}"
+pushd "${workdir}/${relative_path}"
+${CONFIGURE_SCRIPT:-./configure} "$@"
+make -j $(nproc)
+make install
+popd
+rm -rf "${workdir}"
diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py
index e3645b0a03420..0c2a5583b7e55 100644
--- a/.builders/scripts/build_wheels.py
+++ b/.builders/scripts/build_wheels.py
@@ -26,9 +26,9 @@ def path_to_uri(path: str) -> str:
 else:
     import shlex
 
-    PY3_PATH = Path('/py3/bin/python')
-    PY2_PATH = Path('/py2/bin/python')
-    MOUNT_DIR = Path('/home')
+    PY3_PATH = Path(os.environ.get('PY3_BUILDENV_PATH', '/py3/bin/python'))
+    PY2_PATH = Path(os.environ.get('PY2_BUILDENV_PATH', '/py2/bin/python'))
+    MOUNT_DIR = Path(os.environ.get('MOUNT_HOME', '/home'))
     ENV_FILE = Path('/.env')
 
     def join_command_args(args: list[str]) -> str:

From 5a4f013ecd6134d574743c7fa68dbf1db045ca60 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Fri, 12 Jan 2024 18:14:05 +0100
Subject: [PATCH 02/18] Add repair step for MacOS

---
 .builders/build.py                       |  7 ++++++
 .builders/images/macos/build.sh          |  2 ++
 .builders/images/runner_dependencies.txt |  1 +
 .builders/scripts/repair_wheels.py       | 29 ++++++++++++++++++++++++
 4 files changed, 39 insertions(+)

diff --git a/.builders/build.py b/.builders/build.py
index 4fdd235252958..55113ba37f116 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -83,6 +83,13 @@ def build_macos():
 
         mount_dir = temp_dir / 'mnt'
         mount_dir.mkdir()
+        # Folders required by the build_wheels script
+        wheels_dir = mount_dir / 'wheels'
+        wheels_dir.mkdir()
+        built_wheels_dir = wheels_dir / 'built'
+        built_wheels_dir.mkdir()
+        external_wheels_dir = wheels_dir / 'external'
+        external_wheels_dir.mkdir()
 
         dependency_file = mount_dir / 'requirements.in'
         dependency_file.write_text('\n'.join(chain.from_iterable(read_dependencies().values())))
diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
index f26a7f6e06a6f..c460a66665ce1 100644
--- a/.builders/images/macos/build.sh
+++ b/.builders/images/macos/build.sh
@@ -16,6 +16,8 @@ export PREFIX_PATH="$(pwd)/prefix"
 export LDFLAGS="-Wl,-rpath,${PREFIX_PATH}/lib -L${PREFIX_PATH}/lib"
 export CFLAGS="-I${PREFIX_PATH}/include -O2"
 export PATH="${PREFIX_PATH}/bin:${PATH}"
+# Necessary for `delocate` to pick up the extra libraries we install
+export DYLD_LIBRARY_PATH="${PREFIX_PATH}/lib:${DYLD_LIBRARY_PATH:-}"
 
 "${PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
 
diff --git a/.builders/images/runner_dependencies.txt b/.builders/images/runner_dependencies.txt
index e3572efcf1cbe..192632213f618 100644
--- a/.builders/images/runner_dependencies.txt
+++ b/.builders/images/runner_dependencies.txt
@@ -2,3 +2,4 @@ python-dotenv==1.0.0
 urllib3==2.1.0
 auditwheel==5.4.0; sys_platform == 'linux'
 delvewheel==1.5.2; sys_platform == 'win32'
+delocate==0.10.7; sys_platform == 'darwin'
diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py
index 1adb733b7f543..725e04df414e4 100644
--- a/.builders/scripts/repair_wheels.py
+++ b/.builders/scripts/repair_wheels.py
@@ -116,9 +116,38 @@ def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None:
             sys.exit(process.returncode)
 
 
+def repair_darwin(source_dir: str, built_dir: str, external_dir: str) -> None:
+    from delocate import delocate_wheel
+    from delocate.delocating import filter_system_libs
+
+    exclusions = [
+        # pymqi
+        'libmqic_r.dylib',
+    ]
+
+    def copy_filt_func(libname):
+        filter_system_libs(libname) or any(exclusion in libname for exclusion in exclusions)
+
+    for wheel in iter_wheels(source_dir):
+        print(f'--> {wheel.name}')
+        if not wheel_was_built(wheel):
+            print('Using existing wheel')
+            shutil.move(wheel, external_dir)
+            continue
+
+        delocate_wheel(
+            str(wheel),
+            os.path.join(built_dir, os.path.basename(wheel)),
+            lib_sdir='.libs',
+            copy_filt_func=copy_filt_func,
+        )
+        print('Repaired wheel')
+
+
 REPAIR_FUNCTIONS = {
     'linux': repair_linux,
     'win32': repair_windows,
+    'darwin': repair_darwin,
 }
 
 

From a54c1ac8e1fa123947dba1f3bda5df20f2f20dd2 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Mon, 15 Jan 2024 12:04:51 +0100
Subject: [PATCH 03/18] Standardize environment variable prefixes

---
 .builders/build.py                       |  7 ++--
 .builders/images/linux-x86_64/Dockerfile |  6 +--
 .builders/images/macos/build.sh          | 48 +++++++++++++-----------
 .builders/images/macos/extra_build.sh    |  4 +-
 .builders/scripts/build_wheels.py        |  6 +--
 5 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/.builders/build.py b/.builders/build.py
index 55113ba37f116..e5d507e5ff7eb 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -69,7 +69,8 @@ def build_macos():
     parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
     parser.add_argument('output_dir')
     parser.add_argument('--python', default='3')
-    parser.add_argument('--cache-dir')
+    parser.add_argument('--cache-dir',
+                        help='Path to directory to load or save a build cache.')
     args = parser.parse_args()
 
     context_path = HERE / 'images' / 'macos'
@@ -99,8 +100,8 @@ def build_macos():
 
         env = {
             **os.environ,
-            'PREFIX_CACHE': args.cache_dir,
-            'MOUNT_HOME': mount_dir,
+            'DD_PREFIX_CACHE': args.cache_dir,
+            'DD_MOUNT_DIR': mount_dir,
         }
         check_process(
             ['bash', str(HERE / 'images' / 'macos' / 'build.sh'), '--python', args.python],
diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile
index cb147e9261190..d49e249ba7899 100644
--- a/.builders/images/linux-x86_64/Dockerfile
+++ b/.builders/images/linux-x86_64/Dockerfile
@@ -21,7 +21,7 @@ ENV LD_LIBRARY_PATH="/opt/python/${PYTHON2_VERSION}/lib:${LD_LIBRARY_PATH}"
 # Set up virtual environment for Python 2
 RUN /opt/python/${PYTHON2_VERSION}/bin/python -m pip install --no-warn-script-location virtualenv \
  && /opt/python/${PYTHON2_VERSION}/bin/python -m virtualenv /py2
-ENV PY2_BUILDENV_PATH=/py2/bin/python
+ENV DD_PY2_BUILDENV_PATH=/py2/bin/python
 
 # openssl
 RUN yum install -y perl-IPC-Cmd && \
@@ -51,7 +51,7 @@ ENV LD_LIBRARY_PATH="/opt/python/${PYTHON_VERSION}/lib:${LD_LIBRARY_PATH}"
 RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location --upgrade pip \
  && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location virtualenv \
  && /opt/python/${PYTHON_VERSION}/bin/python3 -m virtualenv /py3
-ENV PY3_BUILDENV_PATH=/py3/bin/python
+ENV DD_PY3_BUILDENV_PATH=/py3/bin/python
 
 # Rust toolchain
 ENV RUST_VERSION="nightly-2022-05-15"
@@ -167,6 +167,6 @@ RUN python3 -m pip install --no-warn-script-location -r /runner_dependencies.txt
 
 COPY build_script.sh /build_script.sh
 ENV DD_BUILD_COMMAND="bash /build_script.sh"
-ENV MOUNT_DIR="/home"
+ENV DD_MOUNT_DIR="/home"
 
 ENTRYPOINT ["python3", "/home/scripts/build_wheels.py"]
diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
index c460a66665ce1..15e5243aaaa6b 100644
--- a/.builders/images/macos/build.sh
+++ b/.builders/images/macos/build.sh
@@ -2,28 +2,30 @@
 
 set -euxo pipefail
 
-"${PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
-"${PYTHON3}" -m pip install --no-warn-script-location virtualenv
-"${PYTHON3}" -m virtualenv py3
-export PY3_BUILDENV_PATH="$(pwd)/py3/bin/python"
-
-"${PYTHON2}" -m pip install --no-warn-script-location --upgrade pip
-"${PYTHON2}" -m pip install --no-warn-script-location virtualenv
-"${PYTHON2}" -m virtualenv py2
-export PY2_BUILDENV_PATH="$(pwd)/py2/bin/python"
-
-export PREFIX_PATH="$(pwd)/prefix"
-export LDFLAGS="-Wl,-rpath,${PREFIX_PATH}/lib -L${PREFIX_PATH}/lib"
-export CFLAGS="-I${PREFIX_PATH}/include -O2"
-export PATH="${PREFIX_PATH}/bin:${PATH}"
+"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
+"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv
+"${DD_PYTHON3}" -m virtualenv py3
+export DD_PY3_BUILDENV_PATH="$(pwd)/py3/bin/python"
+
+"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip
+"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv
+"${DD_PYTHON2}" -m virtualenv py2
+export DD_PY2_BUILDENV_PATH="$(pwd)/py2/bin/python"
+
+# Path where we'll install libraries that we build
+export DD_PREFIX_PATH="$(pwd)/prefix"
+
+export LDFLAGS="-Wl,-rpath,${DD_PREFIX_PATH}/lib -L${DD_PREFIX_PATH}/lib"
+export CFLAGS="-I${DD_PREFIX_PATH}/include -O2"
+export PATH="${DD_PREFIX_PATH}/bin:${PATH}"
 # Necessary for `delocate` to pick up the extra libraries we install
-export DYLD_LIBRARY_PATH="${PREFIX_PATH}/lib:${DYLD_LIBRARY_PATH:-}"
+export DYLD_LIBRARY_PATH="${DD_PREFIX_PATH}/lib:${DYLD_LIBRARY_PATH:-}"
 
-"${PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
+"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
 
 # Restore cache if it exists
-if [[ -n ${PREFIX_CACHE:-} && -d ${PREFIX_CACHE:-} ]]; then
-    cp -r "${PREFIX_CACHE}" "${PREFIX_PATH}"
+if [[ -n ${DD_PREFIX_CACHE:-} && -d ${DD_PREFIX_CACHE:-} ]]; then
+    cp -r "${DD_PREFIX_CACHE}" "${DD_PREFIX_PATH}"
 else
     # openssl
     DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
@@ -31,7 +33,7 @@ else
                 SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
                 RELATIVE_PATH="openssl-{{version}}" \
                 CONFIGURE_SCRIPT="./config" \
-                bash install-from-source.sh --prefix="${PREFIX_PATH}" \
+                bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" \
                 -fPIC shared \
                 no-module \
                 no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
@@ -41,13 +43,15 @@ else
                 VERSION="16.0" \
                 SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
                 RELATIVE_PATH="postgresql-{{version}}" \
-                bash install-from-source.sh --prefix="${PREFIX_PATH}" \
+                bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" \
                 --with-openssl --without-readline  --without-icu
 
     # Cache everything under prefix
-    cp -r "${PREFIX_PATH}" "${PREFIX_CACHE}"
+    if [[ -n ${DD_PREFIX_CACHE:-} ]]; then
+        cp -r "${DD_PREFIX_PATH}" "${DD_PREFIX_CACHE}"
+    fi
 fi
 
 export DD_BUILD_COMMAND="bash $(pwd)/extra_build.sh"
 
-"${PYTHON3}" "${MOUNT_HOME}/scripts/build_wheels.py" "$@"
+"${DD_PYTHON3}" "${DD_MOUNT_DIR}/scripts/build_wheels.py" "$@"
diff --git a/.builders/images/macos/extra_build.sh b/.builders/images/macos/extra_build.sh
index f75015475da15..4af459b8c8293 100644
--- a/.builders/images/macos/extra_build.sh
+++ b/.builders/images/macos/extra_build.sh
@@ -6,10 +6,10 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     # confluent-kafka and librdkafka need to be compiled from source to get kerberos support
     # The librdkafka version needs to stay in sync with the confluent-kafka version,
     # thus we extract the version from the requirements file.
-    kafka_version=$(grep 'confluent-kafka==' "${MOUNT_HOME}/requirements.in" | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/')
+    kafka_version=$(grep 'confluent-kafka==' "${DD_MOUNT_DIR}/requirements.in" | sed -E 's/^.*([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+).*$/\1/')
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
       VERSION="${kafka_version}" \
       SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \
       RELATIVE_PATH="librdkafka-{{version}}" \
-      bash install-from-source.sh --prefix="${PREFIX_PATH}" # --enable-sasl
+      bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" # --enable-sasl
 fi
diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py
index 0c2a5583b7e55..e0d788695f581 100644
--- a/.builders/scripts/build_wheels.py
+++ b/.builders/scripts/build_wheels.py
@@ -26,9 +26,9 @@ def path_to_uri(path: str) -> str:
 else:
     import shlex
 
-    PY3_PATH = Path(os.environ.get('PY3_BUILDENV_PATH', '/py3/bin/python'))
-    PY2_PATH = Path(os.environ.get('PY2_BUILDENV_PATH', '/py2/bin/python'))
-    MOUNT_DIR = Path(os.environ.get('MOUNT_HOME', '/home'))
+    PY3_PATH = Path(os.environ.get('DD_PY3_BUILDENV_PATH', '/py3/bin/python'))
+    PY2_PATH = Path(os.environ.get('DD_PY2_BUILDENV_PATH', '/py2/bin/python'))
+    MOUNT_DIR = Path(os.environ.get('DD_MOUNT_DIR', '/home'))
     ENV_FILE = Path('/.env')
 
     def join_command_args(args: list[str]) -> str:

From c0ce12aee938b9e67992e2b920f197cfd92cac66 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Mon, 15 Jan 2024 12:41:56 +0100
Subject: [PATCH 04/18] Move common helper script to share among linux and
 macos

---
 .../{linux-x86_64 => }/install-from-source.sh |  3 +-
 .builders/images/macos/install-from-source.sh | 29 -------------------
 2 files changed, 1 insertion(+), 31 deletions(-)
 rename .builders/images/{linux-x86_64 => }/install-from-source.sh (98%)
 delete mode 100644 .builders/images/macos/install-from-source.sh

diff --git a/.builders/images/linux-x86_64/install-from-source.sh b/.builders/images/install-from-source.sh
similarity index 98%
rename from .builders/images/linux-x86_64/install-from-source.sh
rename to .builders/images/install-from-source.sh
index b510aa0d9d210..4bac1dc1b1a4f 100644
--- a/.builders/images/linux-x86_64/install-from-source.sh
+++ b/.builders/images/install-from-source.sh
@@ -10,8 +10,7 @@
 # Optional:
 # - CONFIGURE_SCRIPT: Alternative to the default ./configure
 
-
-set -exu
+set -euxo pipefail
 
 url=${DOWNLOAD_URL//'{{version}}'/${VERSION}}
 relative_path=${RELATIVE_PATH//'{{version}}'/${VERSION}}
diff --git a/.builders/images/macos/install-from-source.sh b/.builders/images/macos/install-from-source.sh
deleted file mode 100644
index fd07f505624bb..0000000000000
--- a/.builders/images/macos/install-from-source.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# Download, unzip, compile and install a package.
-# Arguments to this script are passed directly to the configure script
-# Required env variables:
-# - DOWNLOAD_URL. Use `{{version}}` as a placeholder to be replaced by the actual version
-# - VERSION
-# - SHA256
-# - RELATIVE_PATH. Set to the relative path in the archive where the source needs to built from.
-#    Can also use the `{{version}}` placeholder for replacemnet.
-# Optional:
-# - CONFIGURE_SCRIPT: Alternative to the default ./configure
-
-set -exu
-
-url=${DOWNLOAD_URL//'{{version}}'/${VERSION}}
-relative_path=${RELATIVE_PATH//'{{version}}'/${VERSION}}
-archive_name="$(basename ${url})"
-workdir="/tmp/build-${archive_name}"
-mkdir -p "${workdir}"
-
-curl "${url}" -Lo "${workdir}/${archive_name}"
-echo "${SHA256}  ${workdir}/${archive_name}" | sha256sum --check
-tar -C "${workdir}" -xf "${workdir}/${archive_name}"
-pushd "${workdir}/${relative_path}"
-${CONFIGURE_SCRIPT:-./configure} "$@"
-make -j $(nproc)
-make install
-popd
-rm -rf "${workdir}"

From 714e5c065a887dbdff06ef3338ba79ab7c069b9a Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Mon, 15 Jan 2024 15:03:04 +0100
Subject: [PATCH 05/18] Fix wheel repair exclusion

---
 .builders/scripts/repair_wheels.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py
index 725e04df414e4..3523e84f93b02 100644
--- a/.builders/scripts/repair_wheels.py
+++ b/.builders/scripts/repair_wheels.py
@@ -122,11 +122,11 @@ def repair_darwin(source_dir: str, built_dir: str, external_dir: str) -> None:
 
     exclusions = [
         # pymqi
-        'libmqic_r.dylib',
+        'pymqe.cpython-311-darwin.so',
     ]
 
     def copy_filt_func(libname):
-        filter_system_libs(libname) or any(exclusion in libname for exclusion in exclusions)
+        return filter_system_libs(libname) and not any(os.path.basename(libname) == excl for excl in exclusions)
 
     for wheel in iter_wheels(source_dir):
         print(f'--> {wheel.name}')

From 1d85f7c98f43b968f3f29946e3fdb0a029e0d6c6 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Mon, 15 Jan 2024 15:03:50 +0100
Subject: [PATCH 06/18] Copy wheels to output directory

---
 .builders/build.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.builders/build.py b/.builders/build.py
index e5d507e5ff7eb..4d22a9b77e74e 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -109,6 +109,18 @@ def build_macos():
             cwd=build_context_dir,
         )
 
+        output_dir = Path(args.output_dir)
+        if output_dir.is_dir():
+            shutil.rmtree(output_dir)
+
+        # Move wheels to the output directory
+        wheels_dir = mount_dir / 'wheels'
+        shutil.move(wheels_dir, output_dir / 'wheels')
+
+        # Move the final requirements file to the output directory
+        final_requirements = mount_dir / 'frozen.txt'
+        shutil.move(final_requirements, output_dir)
+
 
 def build_image():
     parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)

From 03b11979db0a53a3393df3e73bc89995ace52b56 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Mon, 15 Jan 2024 15:22:16 +0100
Subject: [PATCH 07/18] Build confluent-kafka with kerberos support

While leaving cyrus-sasl out of the wheel like in Linux
---
 .builders/images/macos/build.sh       | 52 ++++++++++++++++++++-------
 .builders/images/macos/extra_build.sh |  2 +-
 .builders/scripts/repair_wheels.py    |  5 ++-
 3 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
index 15e5243aaaa6b..955ed1085ffec 100644
--- a/.builders/images/macos/build.sh
+++ b/.builders/images/macos/build.sh
@@ -23,28 +23,54 @@ export DYLD_LIBRARY_PATH="${DD_PREFIX_PATH}/lib:${DYLD_LIBRARY_PATH:-}"
 
 "${DD_PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
 
+# Install always with our own prefix path
+install-from-source() {
+    bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" "$@"
+}
+
 # Restore cache if it exists
 if [[ -n ${DD_PREFIX_CACHE:-} && -d ${DD_PREFIX_CACHE:-} ]]; then
     cp -r "${DD_PREFIX_CACHE}" "${DD_PREFIX_PATH}"
 else
     # openssl
     DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
-                VERSION="3.0.12" \
-                SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
-                RELATIVE_PATH="openssl-{{version}}" \
-                CONFIGURE_SCRIPT="./config" \
-                bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" \
-                -fPIC shared \
-                no-module \
-                no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
+    VERSION="3.0.12" \
+    SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
+    RELATIVE_PATH="openssl-{{version}}" \
+    CONFIGURE_SCRIPT="./config" \
+      install-from-source \
+        -fPIC shared \
+        no-module \
+        no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
 
     # postgresql
     DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
-                VERSION="16.0" \
-                SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
-                RELATIVE_PATH="postgresql-{{version}}" \
-                bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" \
-                --with-openssl --without-readline  --without-icu
+    VERSION="16.0" \
+    SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
+    RELATIVE_PATH="postgresql-{{version}}" \
+      install-from-source \
+        --with-openssl --without-readline  --without-icu
+
+    # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
+    # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
+    # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
+    # compatible with that in the Agent, the rest shouldn't matter much
+    DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
+    VERSION="0.9.29" \
+    SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
+    RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
+    CONFIGURE_SCRIPT="true" \
+      install-from-source
+    DOWNLOAD_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v{{version}}/e2fsprogs-{{version}}.tar.gz" \
+    VERSION="1.47.0" \
+    SHA256="0b4fe723d779b0927fb83c9ae709bc7b40f66d7df36433bef143e41c54257084" \
+    RELATIVE_PATH="e2fsprogs-{{version}}" \
+      install-from-source
+    DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
+    VERSION="2.1.28" \
+    SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
+    RELATIVE_PATH="cyrus-sasl-{{version}}" \
+      install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework
 
     # Cache everything under prefix
     if [[ -n ${DD_PREFIX_CACHE:-} ]]; then
diff --git a/.builders/images/macos/extra_build.sh b/.builders/images/macos/extra_build.sh
index 4af459b8c8293..4bcb0e706ec53 100644
--- a/.builders/images/macos/extra_build.sh
+++ b/.builders/images/macos/extra_build.sh
@@ -11,5 +11,5 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
       VERSION="${kafka_version}" \
       SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \
       RELATIVE_PATH="librdkafka-{{version}}" \
-      bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" # --enable-sasl
+      bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl
 fi
diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py
index 3523e84f93b02..971d0ab51614f 100644
--- a/.builders/scripts/repair_wheels.py
+++ b/.builders/scripts/repair_wheels.py
@@ -123,6 +123,10 @@ def repair_darwin(source_dir: str, built_dir: str, external_dir: str) -> None:
     exclusions = [
         # pymqi
         'pymqe.cpython-311-darwin.so',
+        # confluent_kafka
+        # We leave cyrus-sasl out of the wheel because of the complexity involved in bundling it portably.
+        # This means the confluent-kafka wheel will have a runtime dependency on this library
+        'libsasl2.3.dylib',
     ]
 
     def copy_filt_func(libname):
@@ -138,7 +142,6 @@ def copy_filt_func(libname):
         delocate_wheel(
             str(wheel),
             os.path.join(built_dir, os.path.basename(wheel)),
-            lib_sdir='.libs',
             copy_filt_func=copy_filt_func,
         )
         print('Repaired wheel')

From 735a7d63bfa3c4becac8aefe94b8eca6d1e6c9c5 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Wed, 17 Jan 2024 09:24:12 +0100
Subject: [PATCH 08/18] Add deployment target

---
 .builders/images/macos/build.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
index 955ed1085ffec..7ec59f2eb6a3d 100644
--- a/.builders/images/macos/build.sh
+++ b/.builders/images/macos/build.sh
@@ -2,6 +2,8 @@
 
 set -euxo pipefail
 
+export MACOSX_DEPLOYMENT_TARGET="10.12"
+
 "${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
 "${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv
 "${DD_PYTHON3}" -m virtualenv py3

From 4dd95ef83bb23d8d0748db7b04beef8e40d4c3cf Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Wed, 17 Jan 2024 10:26:01 +0100
Subject: [PATCH 09/18] Drop postgresql since we no longer need to build
 psycopg

---
 .builders/images/macos/build.sh       |  8 --------
 .builders/images/macos/extra_build.sh | 13 +++++++++++++
 .builders/scripts/build_wheels.py     |  2 +-
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
index 7ec59f2eb6a3d..12f745ad33084 100644
--- a/.builders/images/macos/build.sh
+++ b/.builders/images/macos/build.sh
@@ -45,14 +45,6 @@ else
         no-module \
         no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
 
-    # postgresql
-    DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
-    VERSION="16.0" \
-    SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
-    RELATIVE_PATH="postgresql-{{version}}" \
-      install-from-source \
-        --with-openssl --without-readline  --without-icu
-
     # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
     # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
     # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
diff --git a/.builders/images/macos/extra_build.sh b/.builders/images/macos/extra_build.sh
index 4bcb0e706ec53..b37119be5cc8d 100644
--- a/.builders/images/macos/extra_build.sh
+++ b/.builders/images/macos/extra_build.sh
@@ -2,6 +2,9 @@
 
 set -exu
 
+# Packages which must be built from source
+always_build=()
+
 if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     # confluent-kafka and librdkafka need to be compiled from source to get kerberos support
     # The librdkafka version needs to stay in sync with the confluent-kafka version,
@@ -12,4 +15,14 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
       SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \
       RELATIVE_PATH="librdkafka-{{version}}" \
       bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl
+
+    always_build+=("confluent-kafka")
+fi
+
+# Empty arrays are flagged as unset when using the `-u` flag. This is the safest way to work around that
+# (see https://stackoverflow.com/a/61551944)
+pip_no_binary=${always_build[@]+"${always_build[@]}"}
+if [[ "$pip_no_binary" ]]; then
+    # If there are any packages that must always be built, inform pip
+    echo "PIP_NO_BINARY=\"${pip_no_binary}\"" >> $DD_ENV_FILE
 fi
diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py
index e0d788695f581..cda64693a00cd 100644
--- a/.builders/scripts/build_wheels.py
+++ b/.builders/scripts/build_wheels.py
@@ -29,7 +29,7 @@ def path_to_uri(path: str) -> str:
     PY3_PATH = Path(os.environ.get('DD_PY3_BUILDENV_PATH', '/py3/bin/python'))
     PY2_PATH = Path(os.environ.get('DD_PY2_BUILDENV_PATH', '/py2/bin/python'))
     MOUNT_DIR = Path(os.environ.get('DD_MOUNT_DIR', '/home'))
-    ENV_FILE = Path('/.env')
+    ENV_FILE = MOUNT_DIR / '.env'
 
     def join_command_args(args: list[str]) -> str:
         return shlex.join(args)

From 934f0a70cdb7353135a9a0720c78bbac184dc238 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Wed, 17 Jan 2024 11:34:31 +0100
Subject: [PATCH 10/18] Rely less on system-provided libraries (i.e. make
 whitelist stricter)

---
 .builders/images/macos/build.sh    | 53 ++++++++++++++++++++++++++++++
 .builders/scripts/repair_wheels.py | 35 ++++++++++++++++----
 2 files changed, 81 insertions(+), 7 deletions(-)

diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
index 12f745ad33084..a2b54e81f9a65 100644
--- a/.builders/images/macos/build.sh
+++ b/.builders/images/macos/build.sh
@@ -45,6 +45,59 @@ else
         no-module \
         no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
 
+    # libxml & libxslt for lxml
+    DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \
+    VERSION="2.10.3" \
+    SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \
+    RELATIVE_PATH="libxml2-{{version}}" \
+      install-from-source \
+        --without-iconv \
+        --without-python \
+        --without-icu \
+        --without-debug \
+        --without-mem-debug \
+        --without-run-debug \
+        --without-legacy \
+        --without-catalog \
+        --without-docbook \
+        --disable-static
+
+    DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
+    VERSION="1.1.37" \
+    SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \
+    RELATIVE_PATH="libxslt-{{version}}" \
+      install-from-source \
+        --with-libxml-prefix="${DD_PREFIX_PATH}" \
+        --without-python \
+        --without-crypto \
+        --without-profiler \
+        --without-debugger \
+        --disable-static
+
+    # curl
+    DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
+    VERSION="8.4.0" \
+    SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \
+    RELATIVE_PATH="curl-{{version}}" \
+      install-from-source \
+        --disable-manual \
+        --disable-debug \
+        --enable-optimize \
+        --disable-static \
+        --disable-ldap \
+        --disable-ldaps \
+        --disable-rtsp \
+        --enable-proxy \
+        --disable-dependency-tracking \
+        --enable-ipv6 \
+        --without-libidn \
+        --without-gnutls \
+        --without-librtmp \
+        --without-libssh2 \
+        --with-ssl="${DD_PREFIX_PATH}"
+    # Remove the binary installed so that we consistenly use the same original `curl` binary
+    rm "${DD_PREFIX_PATH}/bin/curl"
+
     # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
     # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
     # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py
index 971d0ab51614f..d1f929f430a37 100644
--- a/.builders/scripts/repair_wheels.py
+++ b/.builders/scripts/repair_wheels.py
@@ -118,19 +118,40 @@ def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None:
 
 def repair_darwin(source_dir: str, built_dir: str, external_dir: str) -> None:
     from delocate import delocate_wheel
-    from delocate.delocating import filter_system_libs
-
-    exclusions = [
+    exclusions = [re.compile(s) for s in [
         # pymqi
-        'pymqe.cpython-311-darwin.so',
+        r'pymqe\.cpython-\d+-darwin\.so',
         # confluent_kafka
         # We leave cyrus-sasl out of the wheel because of the complexity involved in bundling it portably.
         # This means the confluent-kafka wheel will have a runtime dependency on this library
-        'libsasl2.3.dylib',
-    ]
+        r'libsasl2.\d\.dylib',
+        # Whitelisted libraries based on the health check default whitelist that we have on omnibus:
+        # https://github.com/DataDog/omnibus-ruby/blob/044a81fa1b0f1c50fc7083cb45e7d8f90d96905b/lib/omnibus/health_check.rb#L133-L152
+        # We use that instead of the more relaxed policy that delocate_wheel defaults to.
+        r'libobjc\.A\.dylib',
+        r'libSystem\.B\.dylib',
+        # Symlink of the previous one
+        r'libgcc_s\.1\.dylib',
+        r'CoreFoundation',
+        r'CoreServices',
+        r'Tcl$',
+        r'Cocoa$',
+        r'Carbon$',
+        r'IOKit$',
+        r'Kerberos',
+        r'Tk$',
+        r'libutil\.dylib',
+        r'libffi\.dylib',
+        r'libncurses\.5\.4\.dylib',
+        r'libiconv',
+        r'libstdc\+\+\.6\.dylib',
+        r'libc\+\+\.1\.dylib',
+        r'^/System/Library/',
+        r'libz\.1\.dylib',
+    ]]
 
     def copy_filt_func(libname):
-        return filter_system_libs(libname) and not any(os.path.basename(libname) == excl for excl in exclusions)
+        return not any(excl.search(libname) for excl in exclusions)
 
     for wheel in iter_wheels(source_dir):
         print(f'--> {wheel.name}')

From b6513124ac1251ed50cc3bbfb2957f01b5e302ba Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Wed, 17 Jan 2024 14:32:32 +0100
Subject: [PATCH 11/18] Remove unnecessary library

---
 .builders/images/macos/build.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
index a2b54e81f9a65..c1a10359197c3 100644
--- a/.builders/images/macos/build.sh
+++ b/.builders/images/macos/build.sh
@@ -108,11 +108,6 @@ else
     RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
     CONFIGURE_SCRIPT="true" \
       install-from-source
-    DOWNLOAD_URL="https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v{{version}}/e2fsprogs-{{version}}.tar.gz" \
-    VERSION="1.47.0" \
-    SHA256="0b4fe723d779b0927fb83c9ae709bc7b40f66d7df36433bef143e41c54257084" \
-    RELATIVE_PATH="e2fsprogs-{{version}}" \
-      install-from-source
     DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
     VERSION="2.1.28" \
     SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \

From 61b4ec20db2e5482ea9ea8d9c2ef503eca70162b Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Wed, 17 Jan 2024 15:05:27 +0100
Subject: [PATCH 12/18] Add workflow to test build on CI

---
 .builders/build.py                     |  2 +-
 .builders/images/macos/build.sh        |  9 ++++--
 .github/workflows/build-deps-macos.yml | 39 ++++++++++++++++++++++++++
 3 files changed, 47 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/build-deps-macos.yml

diff --git a/.builders/build.py b/.builders/build.py
index 4d22a9b77e74e..132afdcd90674 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -100,7 +100,7 @@ def build_macos():
 
         env = {
             **os.environ,
-            'DD_PREFIX_CACHE': args.cache_dir,
+            'DD_PREFIX_CACHE': args.cache_dir or '',
             'DD_MOUNT_DIR': mount_dir,
         }
         check_process(
diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
index c1a10359197c3..a02b694db653a 100644
--- a/.builders/images/macos/build.sh
+++ b/.builders/images/macos/build.sh
@@ -20,8 +20,6 @@ export DD_PREFIX_PATH="$(pwd)/prefix"
 export LDFLAGS="-Wl,-rpath,${DD_PREFIX_PATH}/lib -L${DD_PREFIX_PATH}/lib"
 export CFLAGS="-I${DD_PREFIX_PATH}/include -O2"
 export PATH="${DD_PREFIX_PATH}/bin:${PATH}"
-# Necessary for `delocate` to pick up the extra libraries we install
-export DYLD_LIBRARY_PATH="${DD_PREFIX_PATH}/lib:${DYLD_LIBRARY_PATH:-}"
 
 "${DD_PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
 
@@ -30,8 +28,15 @@ install-from-source() {
     bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" "$@"
 }
 
+# mqi
+IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit
+curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg
+sudo installer -pkg /tmp/mq_client.pkg -target /
+rm -rf /tmp/mq_client.pkg
+
 # Restore cache if it exists
 if [[ -n ${DD_PREFIX_CACHE:-} && -d ${DD_PREFIX_CACHE:-} ]]; then
+    echo "Using provided cache for built libraries."
     cp -r "${DD_PREFIX_CACHE}" "${DD_PREFIX_PATH}"
 else
     # openssl
diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml
new file mode 100644
index 0000000000000..d8aa014d9f3fa
--- /dev/null
+++ b/.github/workflows/build-deps-macos.yml
@@ -0,0 +1,39 @@
+# Workflow for dev / testing purposes only, to be migrated to a common workflow alongside
+# the rest of the deps build
+
+name: Build dependencies for mac OS
+
+on:
+  pull_request
+
+jobs:
+  build-deps-macos:
+    name: "Build wheels for integration dependencies for mac OS"
+    runs-on: macos-12
+    steps:
+      - name: "Set up environment"
+        run: |
+          # We remove everything that comes pre-installed via brew to avoid depending or shipping stuff that
+          # comes in the runner through brew to better control what might get shipped in the wheels via `delocate`
+          brew remove --force --ignore-dependencies $(brew list --formula)
+          brew install coreutils
+      - name: "Install Python(s)"
+        env:
+          # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
+          PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg"
+          PYTHON2_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg"
+        run: |
+          curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
+          sudo installer -pkg python3.pkg -target /
+
+          curl "$PYTHON2_DOWNLOAD_URL" -o python2.pkg
+          sudo installer -pkg python2.pkg -target /
+
+      - uses: actions/checkout@v4
+      - name: "Run the build"
+        env:
+          DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3"
+          DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python"
+        run: |
+          ${DD_PYTHON3} -m pip install packaging
+          ${DD_PYTHON3} .builders/build.py --python 3 out_py3

From 89ba0af8f4514004597f39a2cf0ee951bcb77f21 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Fri, 19 Jan 2024 10:22:35 +0100
Subject: [PATCH 13/18] Add python2 to CI job

---
 .github/workflows/build-deps-macos.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml
index d8aa014d9f3fa..729753ea68356 100644
--- a/.github/workflows/build-deps-macos.yml
+++ b/.github/workflows/build-deps-macos.yml
@@ -37,3 +37,4 @@ jobs:
         run: |
           ${DD_PYTHON3} -m pip install packaging
           ${DD_PYTHON3} .builders/build.py --python 3 out_py3
+          ${DD_PYTHON3} .builders/build.py --python 2 out_py2

From f959345bac7918d0488e87a54fce6f4d58e24a3b Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Fri, 19 Jan 2024 11:31:06 +0100
Subject: [PATCH 14/18] Split builder setup and actual build and make the
 former optional

---
 .builders/build.py                      |  39 +++++--
 .builders/images/macos/build.sh         | 130 ------------------------
 .builders/images/macos/builder_setup.sh | 105 +++++++++++++++++++
 .github/workflows/build-deps-macos.yml  |   8 +-
 4 files changed, 142 insertions(+), 140 deletions(-)
 delete mode 100644 .builders/images/macos/build.sh
 create mode 100644 .builders/images/macos/builder_setup.sh

diff --git a/.builders/build.py b/.builders/build.py
index 132afdcd90674..7a7b3e96759d4 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -69,21 +69,26 @@ def build_macos():
     parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
     parser.add_argument('output_dir')
     parser.add_argument('--python', default='3')
-    parser.add_argument('--cache-dir',
-                        help='Path to directory to load or save a build cache.')
+    parser.add_argument('--builder-root', required=True,
+                        help='Path to a folder where things will be installed during builder setup.')
+    parser.add_argument('--skip-setup', default=False, action='store_true',
+                        help='Skip builder setup, assuming it has already been set up.')
     args = parser.parse_args()
 
     context_path = HERE / 'images' / 'macos'
+    builder_root = Path(args.builder_root).absolute()
+    builder_root.mkdir(exist_ok=True)
 
     with temporary_directory() as temp_dir:
-        build_context_dir = shutil.copytree(context_path, temp_dir, dirs_exist_ok=True)
+        mount_dir = temp_dir / 'mnt'
+        mount_dir.mkdir()
+
+        build_context_dir = shutil.copytree(context_path, mount_dir / 'build_context', dirs_exist_ok=True)
         # Copy utilities shared by multiple images
         for entry in context_path.parent.iterdir():
             if entry.is_file():
                 shutil.copy2(entry, build_context_dir)
 
-        mount_dir = temp_dir / 'mnt'
-        mount_dir.mkdir()
         # Folders required by the build_wheels script
         wheels_dir = mount_dir / 'wheels'
         wheels_dir.mkdir()
@@ -98,15 +103,33 @@ def build_macos():
         shutil.copytree(HERE / 'scripts', mount_dir / 'scripts')
         shutil.copytree(HERE / 'patches', mount_dir / 'patches')
 
+        prefix_path = builder_root / 'prefix'
         env = {
             **os.environ,
-            'DD_PREFIX_CACHE': args.cache_dir or '',
             'DD_MOUNT_DIR': mount_dir,
+            # Paths to pythons
+            'DD_PY3_BUILDENV_PATH': builder_root / 'py3' / 'bin' / 'python',
+            'DD_PY2_BUILDENV_PATH': builder_root / 'py2' / 'bin' / 'python',
+            # Path where we'll install libraries that we build
+            'DD_PREFIX_PATH': prefix_path,
+            # Common compilation flags
+            'LDFLAGS': f'-Wl,-rpath,{prefix_path}/lib -L{prefix_path}/lib',
+            'CFLAGS': f'-I{prefix_path}/include -O2',
+            # Build command for extra platform-specific build steps
+            'DD_BUILD_COMMAND': f'bash {build_context_dir}/extra_build.sh'
         }
+
+        if not args.skip_setup:
+            check_process(
+                ['bash', str(HERE / 'images' / 'macos' / 'builder_setup.sh')],
+                env=env,
+                cwd=builder_root,
+            )
+
         check_process(
-            ['bash', str(HERE / 'images' / 'macos' / 'build.sh'), '--python', args.python],
+            [os.environ['DD_PYTHON3'], str(mount_dir / 'scripts' / 'build_wheels.py'), '--python', args.python],
             env=env,
-            cwd=build_context_dir,
+            cwd=builder_root,
         )
 
         output_dir = Path(args.output_dir)
diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
deleted file mode 100644
index a02b694db653a..0000000000000
--- a/.builders/images/macos/build.sh
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env bash
-
-set -euxo pipefail
-
-export MACOSX_DEPLOYMENT_TARGET="10.12"
-
-"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
-"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv
-"${DD_PYTHON3}" -m virtualenv py3
-export DD_PY3_BUILDENV_PATH="$(pwd)/py3/bin/python"
-
-"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip
-"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv
-"${DD_PYTHON2}" -m virtualenv py2
-export DD_PY2_BUILDENV_PATH="$(pwd)/py2/bin/python"
-
-# Path where we'll install libraries that we build
-export DD_PREFIX_PATH="$(pwd)/prefix"
-
-export LDFLAGS="-Wl,-rpath,${DD_PREFIX_PATH}/lib -L${DD_PREFIX_PATH}/lib"
-export CFLAGS="-I${DD_PREFIX_PATH}/include -O2"
-export PATH="${DD_PREFIX_PATH}/bin:${PATH}"
-
-"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
-
-# Install always with our own prefix path
-install-from-source() {
-    bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" "$@"
-}
-
-# mqi
-IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit
-curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg
-sudo installer -pkg /tmp/mq_client.pkg -target /
-rm -rf /tmp/mq_client.pkg
-
-# Restore cache if it exists
-if [[ -n ${DD_PREFIX_CACHE:-} && -d ${DD_PREFIX_CACHE:-} ]]; then
-    echo "Using provided cache for built libraries."
-    cp -r "${DD_PREFIX_CACHE}" "${DD_PREFIX_PATH}"
-else
-    # openssl
-    DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
-    VERSION="3.0.12" \
-    SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
-    RELATIVE_PATH="openssl-{{version}}" \
-    CONFIGURE_SCRIPT="./config" \
-      install-from-source \
-        -fPIC shared \
-        no-module \
-        no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
-
-    # libxml & libxslt for lxml
-    DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \
-    VERSION="2.10.3" \
-    SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \
-    RELATIVE_PATH="libxml2-{{version}}" \
-      install-from-source \
-        --without-iconv \
-        --without-python \
-        --without-icu \
-        --without-debug \
-        --without-mem-debug \
-        --without-run-debug \
-        --without-legacy \
-        --without-catalog \
-        --without-docbook \
-        --disable-static
-
-    DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
-    VERSION="1.1.37" \
-    SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \
-    RELATIVE_PATH="libxslt-{{version}}" \
-      install-from-source \
-        --with-libxml-prefix="${DD_PREFIX_PATH}" \
-        --without-python \
-        --without-crypto \
-        --without-profiler \
-        --without-debugger \
-        --disable-static
-
-    # curl
-    DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
-    VERSION="8.4.0" \
-    SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \
-    RELATIVE_PATH="curl-{{version}}" \
-      install-from-source \
-        --disable-manual \
-        --disable-debug \
-        --enable-optimize \
-        --disable-static \
-        --disable-ldap \
-        --disable-ldaps \
-        --disable-rtsp \
-        --enable-proxy \
-        --disable-dependency-tracking \
-        --enable-ipv6 \
-        --without-libidn \
-        --without-gnutls \
-        --without-librtmp \
-        --without-libssh2 \
-        --with-ssl="${DD_PREFIX_PATH}"
-    # Remove the binary installed so that we consistenly use the same original `curl` binary
-    rm "${DD_PREFIX_PATH}/bin/curl"
-
-    # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
-    # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
-    # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
-    # compatible with that in the Agent, the rest shouldn't matter much
-    DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
-    VERSION="0.9.29" \
-    SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
-    RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
-    CONFIGURE_SCRIPT="true" \
-      install-from-source
-    DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
-    VERSION="2.1.28" \
-    SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
-    RELATIVE_PATH="cyrus-sasl-{{version}}" \
-      install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework
-
-    # Cache everything under prefix
-    if [[ -n ${DD_PREFIX_CACHE:-} ]]; then
-        cp -r "${DD_PREFIX_PATH}" "${DD_PREFIX_CACHE}"
-    fi
-fi
-
-export DD_BUILD_COMMAND="bash $(pwd)/extra_build.sh"
-
-"${DD_PYTHON3}" "${DD_MOUNT_DIR}/scripts/build_wheels.py" "$@"
diff --git a/.builders/images/macos/builder_setup.sh b/.builders/images/macos/builder_setup.sh
new file mode 100644
index 0000000000000..c3b90b250a4a8
--- /dev/null
+++ b/.builders/images/macos/builder_setup.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
+"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv
+"${DD_PYTHON3}" -m virtualenv py3
+
+"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip
+"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv
+"${DD_PYTHON2}" -m virtualenv py2
+
+"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "${DD_MOUNT_DIR}/build_context/runner_dependencies.txt"
+
+# Install always with our own prefix path
+cp "${DD_MOUNT_DIR}/build_context/install-from-source.sh" .
+install-from-source() {
+    bash "install-from-source.sh" --prefix="${DD_PREFIX_PATH}" "$@"
+}
+
+# mqi
+IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit
+curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg
+sudo installer -pkg /tmp/mq_client.pkg -target /
+rm -rf /tmp/mq_client.pkg
+
+# openssl
+DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
+VERSION="3.0.12" \
+SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
+RELATIVE_PATH="openssl-{{version}}" \
+CONFIGURE_SCRIPT="./config" \
+  install-from-source \
+    -fPIC shared \
+    no-module \
+    no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
+
+# libxml & libxslt for lxml
+DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \
+VERSION="2.10.3" \
+SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \
+RELATIVE_PATH="libxml2-{{version}}" \
+  install-from-source \
+    --without-iconv \
+    --without-python \
+    --without-icu \
+    --without-debug \
+    --without-mem-debug \
+    --without-run-debug \
+    --without-legacy \
+    --without-catalog \
+    --without-docbook \
+    --disable-static
+
+DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
+VERSION="1.1.37" \
+SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \
+RELATIVE_PATH="libxslt-{{version}}" \
+  install-from-source \
+    --with-libxml-prefix="${DD_PREFIX_PATH}" \
+    --without-python \
+    --without-crypto \
+    --without-profiler \
+    --without-debugger \
+    --disable-static
+
+# curl
+DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
+VERSION="8.4.0" \
+SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \
+RELATIVE_PATH="curl-{{version}}" \
+  install-from-source \
+    --disable-manual \
+    --disable-debug \
+    --enable-optimize \
+    --disable-static \
+    --disable-ldap \
+    --disable-ldaps \
+    --disable-rtsp \
+    --enable-proxy \
+    --disable-dependency-tracking \
+    --enable-ipv6 \
+    --without-libidn \
+    --without-gnutls \
+    --without-librtmp \
+    --without-libssh2 \
+    --with-ssl="${DD_PREFIX_PATH}"
+# Remove the binary installed so that we consistenly use the same original `curl` binary
+rm "${DD_PREFIX_PATH}/bin/curl"
+
+# Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
+# Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
+# with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
+# compatible with that in the Agent, the rest shouldn't matter much
+DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
+VERSION="0.9.29" \
+SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
+RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
+CONFIGURE_SCRIPT="true" \
+  install-from-source
+DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
+VERSION="2.1.28" \
+SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
+RELATIVE_PATH="cyrus-sasl-{{version}}" \
+  install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework
diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml
index 729753ea68356..682c52e357cf5 100644
--- a/.github/workflows/build-deps-macos.yml
+++ b/.github/workflows/build-deps-macos.yml
@@ -34,7 +34,11 @@ jobs:
         env:
           DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3"
           DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python"
+          # This sets the minimum mac os version compatible for all built artifacts
+          MACOSX_DEPLOYMENT_TARGET: "10.12"
         run: |
           ${DD_PYTHON3} -m pip install packaging
-          ${DD_PYTHON3} .builders/build.py --python 3 out_py3
-          ${DD_PYTHON3} .builders/build.py --python 2 out_py2
+
+          mkdir builder_root
+          ${DD_PYTHON3} .builders/build.py --builder-root builder_root --python 3 out_py3
+          ${DD_PYTHON3} .builders/build.py --builder-root builder_root --skip-setup --python 2 out_py2

From 233b268833de10b36c4fc5996df5af238d9b0064 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Mon, 22 Jan 2024 14:21:58 +0100
Subject: [PATCH 15/18] Output information about libraries copied during repair

---
 .builders/scripts/repair_wheels.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.builders/scripts/repair_wheels.py b/.builders/scripts/repair_wheels.py
index d1f929f430a37..6e2d6e2f5832c 100644
--- a/.builders/scripts/repair_wheels.py
+++ b/.builders/scripts/repair_wheels.py
@@ -160,12 +160,17 @@ def copy_filt_func(libname):
             shutil.move(wheel, external_dir)
             continue
 
-        delocate_wheel(
+        copied_libs = delocate_wheel(
             str(wheel),
             os.path.join(built_dir, os.path.basename(wheel)),
             copy_filt_func=copy_filt_func,
         )
         print('Repaired wheel')
+        if copied_libs:
+            print('Libraries copied into the wheel:')
+            print('\n'.join(copied_libs))
+        else:
+            print('No libraries were copied into the wheel.')
 
 
 REPAIR_FUNCTIONS = {

From 3c9de447f72a717c8c8ee859d4ae2122647d76b7 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Thu, 25 Jan 2024 10:36:32 +0100
Subject: [PATCH 16/18] Don't set rpath as it's not needed

---
 .builders/build.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.builders/build.py b/.builders/build.py
index 7a7b3e96759d4..f1cf092c82dff 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -113,7 +113,7 @@ def build_macos():
             # Path where we'll install libraries that we build
             'DD_PREFIX_PATH': prefix_path,
             # Common compilation flags
-            'LDFLAGS': f'-Wl,-rpath,{prefix_path}/lib -L{prefix_path}/lib',
+            'LDFLAGS': f'-L{prefix_path}/lib',
             'CFLAGS': f'-I{prefix_path}/include -O2',
             # Build command for extra platform-specific build steps
             'DD_BUILD_COMMAND': f'bash {build_context_dir}/extra_build.sh'

From f6c9eaaba884d8bbaddab6bc6befe69710c3b689 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Thu, 25 Jan 2024 16:49:43 +0100
Subject: [PATCH 17/18] Move job to common workflow

---
 .github/workflows/build-deps-macos.yml | 44 --------------------------
 .github/workflows/build-deps.yml       | 39 +++++++++++++++++++++++
 2 files changed, 39 insertions(+), 44 deletions(-)
 delete mode 100644 .github/workflows/build-deps-macos.yml

diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml
deleted file mode 100644
index 682c52e357cf5..0000000000000
--- a/.github/workflows/build-deps-macos.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-# Workflow for dev / testing purposes only, to be migrated to a common workflow alongside
-# the rest of the deps build
-
-name: Build dependencies for mac OS
-
-on:
-  pull_request
-
-jobs:
-  build-deps-macos:
-    name: "Build wheels for integration dependencies for mac OS"
-    runs-on: macos-12
-    steps:
-      - name: "Set up environment"
-        run: |
-          # We remove everything that comes pre-installed via brew to avoid depending or shipping stuff that
-          # comes in the runner through brew to better control what might get shipped in the wheels via `delocate`
-          brew remove --force --ignore-dependencies $(brew list --formula)
-          brew install coreutils
-      - name: "Install Python(s)"
-        env:
-          # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
-          PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg"
-          PYTHON2_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg"
-        run: |
-          curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
-          sudo installer -pkg python3.pkg -target /
-
-          curl "$PYTHON2_DOWNLOAD_URL" -o python2.pkg
-          sudo installer -pkg python2.pkg -target /
-
-      - uses: actions/checkout@v4
-      - name: "Run the build"
-        env:
-          DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3"
-          DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python"
-          # This sets the minimum mac os version compatible for all built artifacts
-          MACOSX_DEPLOYMENT_TARGET: "10.12"
-        run: |
-          ${DD_PYTHON3} -m pip install packaging
-
-          mkdir builder_root
-          ${DD_PYTHON3} .builders/build.py --builder-root builder_root --python 3 out_py3
-          ${DD_PYTHON3} .builders/build.py --builder-root builder_root --skip-setup --python 2 out_py2
diff --git a/.github/workflows/build-deps.yml b/.github/workflows/build-deps.yml
index 13a3dd4413d3e..a027f76dd33f9 100644
--- a/.github/workflows/build-deps.yml
+++ b/.github/workflows/build-deps.yml
@@ -98,3 +98,42 @@ jobs:
     - name: Publish image
       if: github.event_name == 'push' && steps.changed-files.outputs.builders_any_changed == 'true'
       run: docker push ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}
+
+  build-deps-macos:
+    name: Target macOS
+    runs-on: macos-12
+    steps:
+      - name: Set up environment
+        run: |
+          # We remove everything that comes pre-installed via brew to avoid depending or shipping stuff that
+          # comes in the runner through brew to better control what might get shipped in the wheels via `delocate`
+          brew remove --force --ignore-dependencies $(brew list --formula)
+          brew install coreutils
+
+      - name: Set up Python
+        env:
+          # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
+          PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.11.5/python-3.11.5-macos11.pkg"
+          PYTHON2_DOWNLOAD_URL: "https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg"
+        run: |
+          curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
+          sudo installer -pkg python3.pkg -target /
+
+          curl "$PYTHON2_DOWNLOAD_URL" -o python2.pkg
+          sudo installer -pkg python2.pkg -target /
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run the build
+        env:
+          DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3"
+          DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python"
+          # This sets the minimum mac os version compatible for all built artifacts
+          MACOSX_DEPLOYMENT_TARGET: "10.12"
+        run: |
+          ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt
+     
+          mkdir builder_root
+          ${DD_PYTHON3} .builders/build.py --builder-root builder_root --python 3 out_py3
+          ${DD_PYTHON3} .builders/build.py --builder-root builder_root --skip-setup --python 2 out_py2

From ebd390959fcd590f7883291ad9e746778cf6b534 Mon Sep 17 00:00:00 2001
From: Alex Lopez <alex.lopez@datadoghq.com>
Date: Fri, 26 Jan 2024 10:23:46 +0100
Subject: [PATCH 18/18] Adjustments to environment variables and paths

---
 .builders/build.py                       | 1 +
 .builders/images/linux-x86_64/Dockerfile | 3 ---
 .builders/scripts/build_wheels.py        | 3 ++-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.builders/build.py b/.builders/build.py
index f1cf092c82dff..838edb70e9000 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -107,6 +107,7 @@ def build_macos():
         env = {
             **os.environ,
             'DD_MOUNT_DIR': mount_dir,
+            'DD_ENV_FILE': mount_dir / '.env',
             # Paths to pythons
             'DD_PY3_BUILDENV_PATH': builder_root / 'py3' / 'bin' / 'python',
             'DD_PY2_BUILDENV_PATH': builder_root / 'py2' / 'bin' / 'python',
diff --git a/.builders/images/linux-x86_64/Dockerfile b/.builders/images/linux-x86_64/Dockerfile
index d49e249ba7899..8e9846c46800e 100644
--- a/.builders/images/linux-x86_64/Dockerfile
+++ b/.builders/images/linux-x86_64/Dockerfile
@@ -21,7 +21,6 @@ ENV LD_LIBRARY_PATH="/opt/python/${PYTHON2_VERSION}/lib:${LD_LIBRARY_PATH}"
 # Set up virtual environment for Python 2
 RUN /opt/python/${PYTHON2_VERSION}/bin/python -m pip install --no-warn-script-location virtualenv \
  && /opt/python/${PYTHON2_VERSION}/bin/python -m virtualenv /py2
-ENV DD_PY2_BUILDENV_PATH=/py2/bin/python
 
 # openssl
 RUN yum install -y perl-IPC-Cmd && \
@@ -51,7 +50,6 @@ ENV LD_LIBRARY_PATH="/opt/python/${PYTHON_VERSION}/lib:${LD_LIBRARY_PATH}"
 RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location --upgrade pip \
  && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location virtualenv \
  && /opt/python/${PYTHON_VERSION}/bin/python3 -m virtualenv /py3
-ENV DD_PY3_BUILDENV_PATH=/py3/bin/python
 
 # Rust toolchain
 ENV RUST_VERSION="nightly-2022-05-15"
@@ -167,6 +165,5 @@ RUN python3 -m pip install --no-warn-script-location -r /runner_dependencies.txt
 
 COPY build_script.sh /build_script.sh
 ENV DD_BUILD_COMMAND="bash /build_script.sh"
-ENV DD_MOUNT_DIR="/home"
 
 ENTRYPOINT ["python3", "/home/scripts/build_wheels.py"]
diff --git a/.builders/scripts/build_wheels.py b/.builders/scripts/build_wheels.py
index cda64693a00cd..5f12950d9ce44 100644
--- a/.builders/scripts/build_wheels.py
+++ b/.builders/scripts/build_wheels.py
@@ -11,6 +11,7 @@
 from dotenv import dotenv_values
 from utils import extract_metadata, normalize_project_name
 
+
 if sys.platform == 'win32':
     PY3_PATH = Path('C:\\py3\\Scripts\\python.exe')
     PY2_PATH = Path('C:\\py2\\Scripts\\python.exe')
@@ -29,7 +30,7 @@ def path_to_uri(path: str) -> str:
     PY3_PATH = Path(os.environ.get('DD_PY3_BUILDENV_PATH', '/py3/bin/python'))
     PY2_PATH = Path(os.environ.get('DD_PY2_BUILDENV_PATH', '/py2/bin/python'))
     MOUNT_DIR = Path(os.environ.get('DD_MOUNT_DIR', '/home'))
-    ENV_FILE = MOUNT_DIR / '.env'
+    ENV_FILE = Path(os.environ.get('DD_ENV_FILE', '/.env'))
 
     def join_command_args(args: list[str]) -> str:
         return shlex.join(args)