fix: Set x-abi-tools-use-exact-versions to true

Signed-off-by: Gordon Smith <[email protected]>
hpcc-systems · Apr 11, 2024 · d72dc8a · d72dc8a
1 parent 5ded246
commit d72dc8a
Show file tree

Hide file tree

Showing 14 changed files with 258 additions and 174 deletions.
diff --git a/.github/workflows/prebuild-docker.yml b/.github/workflows/prebuild-docker.yml
@@ -32,14 +32,32 @@ jobs:
       fail-fast: false
 
     name: "Docker:  ${{ matrix.image }}"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Free additional disk space (remove Android SDK + Tools)
+        continue-on-error: true
         run: |
+          df -h
           sudo rm -rf /usr/local/lib/android
+          sudo rm -rf ${JAVA_HOME_8_X64}
+          sudo rm -rf ${JAVA_HOME_11_X64}
+          sudo rm -rf ${JAVA_HOME_17_X64}
+          sudo rm -rf ${JAVA_HOME_21_X64}
+          sudo rm -rf ${CHROMEWEBDRIVER}
+          sudo rm -rf ${EDGEWEBDRIVER}
+          sudo rm -rf ${GECKOWEBDRIVER}
+          sudo rm -rf ${SELENIUM_JAR_PATH}
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Display disk space (post free)
+        run: |
+          df -h
 
       - name: Check out source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ inputs.branch || github.ref }}
@@ -61,16 +79,21 @@ jobs:
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Pull previous images
+        run: |
+          docker pull hpccsystems/platform-build-base-${{ matrix.image }}:${{ steps.vars.outputs.sha_short }} || true
+          docker pull hpccsystems/platform-build-base-${{ matrix.image }}:${{ steps.vars.outputs.vcpkg_tag }} || true
+
       - name: Build ${{ matrix.os }} image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           builder: ${{ steps.buildx.outputs.name }}
           file: dockerfiles/${{ matrix.image }}.dockerfile

diff --git a/.github/workflows/prebuild-gh_envs.yml b/.github/workflows/prebuild-gh_envs.yml
@@ -120,22 +120,7 @@ jobs:
         if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-12'}}
         shell: "bash"
         run: |
-          brew install pkg-config automake libtool autoconf autoconf-archive
-
-      - name: "Only run on windows"
-        if: ${{ startsWith(matrix.os, 'windows') }}
-        working-directory: ..
-        shell: pwsh
-        run: |
-          Remove-Item -Path "${{ github.workspace }}" -Recurse -Force
-          New-Item -ItemType Directory -Path "C:\workspace"
-          New-Item -ItemType SymbolicLink -Path "${{ github.workspace }}" -Target "C:\workspace"
-  
-      - name: "Remove builtin vcpkg (old)"
-        working-directory: .
-        shell: "bash"
-        run: |
-          ${{ matrix.sudo }} rm -rf "$VCPKG_INSTALLATION_ROOT"
+          brew install bison flex pkg-config automake libtool autoconf autoconf-archive
 
       - name: Check out source code
         uses: actions/checkout@v3
@@ -165,9 +150,12 @@ jobs:
             -source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json"
 
       - name: "Update nuget (overlays)"
+        working-directory: .
         shell: "bash"
         run: |
-          ./vcpkg install --overlay-ports=./overlays --keep-going --triplet ${{ matrix.triplet }}
+          ./vcpkg install \
+            --x-abi-tools-use-exact-versions \
+            --triplet=${{ matrix.triplet }}
 
       - name: Upload error logs
         if: ${{ failure() || cancelled() }}

diff --git a/.gitignore b/.gitignore
@@ -10,8 +10,6 @@
 *.user
 *.userosscache
 *.sln.docstates
-/vcpkg.json
-/vcpkg-configuration.json
 
 # fuzzing
 sync_dir*

diff --git a/dockerfiles/amazonlinux.dockerfile b/dockerfiles/amazonlinux.dockerfile
@@ -18,14 +18,18 @@ RUN yum update -y && yum install -y \
     python3 \
     tar \
     unzip \
-    zip
+    zip && \
+    yum -y clean all && rm -rf /var/cache
 
 RUN yum-config-manager --add-repo http://mirror.centos.org/centos/7/sclo/x86_64/rh/ && \
     LIBGFORTRAN5=libgfortran5-8.3.1-2.1.1.el7.x86_64.rpm && \
     wget http://mirror.centos.org/centos/7/os/x86_64/Packages/${LIBGFORTRAN5} && \
     yum install -y ${LIBGFORTRAN5} && \
     yum install -y devtoolset-11 --nogpgcheck
 
+RUN echo "source /opt/rh/devtoolset-11/enable" > /etc/profile.d/devtoolset-11.sh
+SHELL ["/bin/bash", "--login", "-c"]
+
 RUN yum install -y hostname xz && \
     curl -o pkg-config-0.29.2.tar.gz https://pkg-config.freedesktop.org/releases/pkg-config-0.29.2.tar.gz && \
     tar xvfz pkg-config-0.29.2.tar.gz
@@ -37,9 +41,8 @@ RUN ./configure --prefix=/usr/local/pkg_config/0_29_2 --with-internal-glib && \
     mkdir /usr/local/share/aclocal && \
     ln -s /usr/local/pkg_config/0_29_2/share/aclocal/pkg.m4 /usr/local/share/aclocal/
 
-ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
-ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH
-ENV ACLOCAL_PATH=/usr/local/share/aclocal:$ACLOCAL_PATH
+ENV PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig
+ENV ACLOCAL_PATH=$ACLOCAL_PATH:/usr/local/share/aclocal
 
 RUN curl -o autoconf-2.71.tar.gz http://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz && \
     gunzip autoconf-2.71.tar.gz && \
@@ -92,18 +95,15 @@ RUN ln -s /usr/local/libtool/2_4_6/bin/libtool /usr/local/bin/ && \
 # ldconfig
 # ldconfig -v
 
-RUN echo "source /opt/rh/devtoolset-11/enable" > /etc/profile.d/devtoolset-11.sh
-SHELL ["/bin/bash", "--login", "-c"]
-
 FROM base_build AS vcpkg_build
 
 # Build Tools - Mono  ---
-RUN yum-config-manager --add-repo http://download.mono-project.com/repo/centos/
-RUN yum clean all
-RUN yum makecache
-RUN rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF"
-
-RUN yum install -y mono-complete 
+RUN yum-config-manager --add-repo http://download.mono-project.com/repo/centos/ && \
+    yum clean all && \
+    yum makecache && \
+    rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF" && \
+    yum install -y mono-complete && \
+    yum -y clean all && rm -rf /var/cache
 
 ARG NUGET_MODE=readwrite
 ENV VCPKG_BINARY_SOURCES="clear;nuget,GitHub,${NUGET_MODE}"
@@ -131,10 +131,10 @@ RUN mono `./vcpkg fetch nuget | tail -n 1` \
 # vcpkg  ---
 RUN mkdir /hpcc-dev/build
 RUN ./vcpkg install \
+    --x-abi-tools-use-exact-versions \
     --x-install-root=/hpcc-dev/build/vcpkg_installed \
-    --overlay-ports=./overlays \
-    --triplet=x64-linux-dynamic
-# ./vcpkg install --overlay-ports=./overlays --triplet=x64-linux-dynamic --x-install-root=/hpcc-dev/build/vcpkg_installed
+    --triplet=x64-amazonlinux-dynamic
+# ./vcpkg install --x-abi-tools-use-exact-versions --triplet=x64-amazonlinux-dynamic --x-install-root=/hpcc-dev/build/vcpkg_installed
 
 RUN mkdir -p /hpcc-dev/tools/cmake
 RUN cp -r $(dirname $(dirname `./vcpkg fetch cmake | tail -n 1`))/* /hpcc-dev/tools/cmake
@@ -145,6 +145,16 @@ RUN cp -r $(dirname $(dirname `./vcpkg fetch node | tail -n 1`))/* /hpcc-dev/too
 
 FROM base_build
 
+RUN amazon-linux-extras install java-openjdk11 && yum install -y \
+    java-11-openjdk-devel \
+    python3-devel \
+    epel-release && \
+    yum install -y \
+    ccache \
+    R-core-devel \
+    R-Rcpp-devel \
+    R-RInside-devel
+
 WORKDIR /hpcc-dev
 
 COPY --from=vcpkg_build /hpcc-dev/build/vcpkg_installed /hpcc-dev/vcpkg_installed
@@ -157,3 +167,7 @@ RUN cp -rs /hpcc-dev/tools/cmake/bin /usr/local/ && \
     cp -rs /hpcc-dev/tools/node/include /usr/local/ && \
     cp -rs /hpcc-dev/tools/node/lib /usr/local/ && \
     cp -rs /hpcc-dev/tools/node/share /usr/local/
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]
diff --git a/dockerfiles/build.sh b/dockerfiles/build.sh
@@ -23,24 +23,52 @@ echo "DOCKER_PASSWORD: $DOCKER_PASSWORD"
 docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
 
 function doBuild() {
+    docker pull "hpccsystems/platform-build-vcpkg-$1:$GITHUB_REF" || true
+    docker pull "hpccsystems/platform-build-vcpkg-$1:$GITHUB_BRANCH" || true
+
     docker build --progress plain --rm -f "$SCRIPT_DIR/$1.dockerfile" \
+        --target vcpkg_build \
         --build-arg NUGET_MODE=readwrite \
         --build-arg GITHUB_ACTOR=$GITHUB_ACTOR \
         --build-arg GITHUB_TOKEN=$GITHUB_TOKEN \
-        -t hpccsystems/platform-build-base-$1:$GITHUB_REF \
-        -t hpccsystems/platform-build-base-$1:$GITHUB_BRANCH \
-        --cache-from hpccsystems/platform-build-base-$1:$GITHUB_REF \
-        --cache-from hpccsystems/platform-build-base-$1:$GITHUB_BRANCH \
+        --cache-from hpccsystems/platform-build-vcpkg-$1:$GITHUB_REF \
+        --cache-from hpccsystems/platform-build-vcpkg-$1:$GITHUB_BRANCH \
+        -t hpccsystems/platform-build-vcpkg-$1:$GITHUB_REF \
+        -t hpccsystems/platform-build-vcpkg-$1:$GITHUB_BRANCH \
         "$SCRIPT_DIR/.."
-    docker push hpccsystems/platform-build-base-$1:$GITHUB_REF
-    docker push hpccsystems/platform-build-base-$1:$GITHUB_BRANCH
+
+    # docker push "hpccsystems/platform-build-vcpkg-$1:$GITHUB_REF" &
+    # docker push "hpccsystems/platform-build-vcpkg-$1:$GITHUB_BRANCH" & 
+
+    # mkdir -p build-$1
+    # docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached hpccsystems/platform-build-vcpkg-$1:$GITHUB_REF \
+    #    "rm -rf /hpcc-dev/HPCC-Platform/build-$1/vcpkg_installed || true && \
+    #     cp -r /hpcc-dev/vcpkg_installed /hpcc-dev/HPCC-Platform/build-$1"
+
+    # docker pull "hpccsystems/platform-build-base-$1:$GITHUB_REF" || true
+    # docker pull "hpccsystems/platform-build-base-$1:$GITHUB_BRANCH" || true
+
+    # docker build --progress plain --rm -f "$SCRIPT_DIR/$1.dockerfile" \
+    #     --build-arg NUGET_MODE=readwrite \
+    #     --build-arg GITHUB_ACTOR=$GITHUB_ACTOR \
+    #     --build-arg GITHUB_TOKEN=$GITHUB_TOKEN \
+    #     --cache-from hpccsystems/platform-build-base-$1:$GITHUB_REF \
+    #     --cache-from hpccsystems/platform-build-base-$1:$GITHUB_BRANCH \
+    #     -t hpccsystems/platform-build-base-$1:$GITHUB_REF \
+    #     -t hpccsystems/platform-build-base-$1:$GITHUB_BRANCH \
+    #     "$SCRIPT_DIR/.."
+
+    # docker push hpccsystems/platform-build-base-$1:$GITHUB_REF
+    # docker push hpccsystems/platform-build-base-$1:$GITHUB_BRANCH
 }
 
-doBuild centos-7 &
-doBuild centos-8 &
-doBuild ubuntu-23.04 &
-doBuild ubuntu-22.04 &
-doBuild ubuntu-20.04 &
-doBuild amazonlinux &
+doBuild ubuntu-24.04
+# doBuild amazonlinux 
+# doBuild centos-7 
+# doBuild centos-8 
+# doBuild ubuntu-24.04 &
+# doBuild ubuntu-23.04 &
+# doBuild ubuntu-22.04 &
+# doBuild ubuntu-20.04 &
 
 wait
diff --git a/dockerfiles/centos-7.dockerfile b/dockerfiles/centos-7.dockerfile
@@ -5,8 +5,8 @@ RUN yum update -y && yum install -y \
     centos-release-scl \
     https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm && \
     yum group install -y "Development Tools" && yum install -y \
-    autoconf-archive \
     autoconf \
+    autoconf-archive \
     automake \
     curl \
     git \
@@ -17,7 +17,8 @@ RUN yum update -y && yum install -y \
     tar \
     unzip \
     yum-utils \
-    zip 
+    zip && \
+    yum -y clean all && rm -rf /var/cache
 
 RUN yum install -y devtoolset-11
 
@@ -34,9 +35,8 @@ RUN ./configure --prefix=/usr/local/pkg_config/0_29_2 --with-internal-glib && \
     mkdir /usr/local/share/aclocal && \
     ln -s /usr/local/pkg_config/0_29_2/share/aclocal/pkg.m4 /usr/local/share/aclocal/
 
-ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
-ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH
-ENV ACLOCAL_PATH=/usr/local/share/aclocal:$ACLOCAL_PATH
+ENV PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig
+ENV ACLOCAL_PATH=$ACLOCAL_PATH:/usr/local/share/aclocal
 
 RUN curl -o autoconf-2.71.tar.gz http://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz && \
     gunzip autoconf-2.71.tar.gz && \
@@ -92,12 +92,12 @@ RUN ln -s /usr/local/libtool/2_4_6/bin/libtool /usr/local/bin/ && \
 FROM base_build AS vcpkg_build
 
 # Build Tools - Mono  ---
-RUN yum-config-manager --add-repo http://download.mono-project.com/repo/centos/
-RUN yum clean all
-RUN yum makecache
-RUN rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF"
-
-RUN yum install -y mono-complete 
+RUN yum-config-manager --add-repo http://download.mono-project.com/repo/centos/ && \
+    yum clean all && \
+    yum makecache && \
+    rpm --import "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF" && \
+    yum install -y mono-complete && \
+    yum -y clean all && rm -rf /var/cache
 
 ARG NUGET_MODE=readwrite
 ENV VCPKG_BINARY_SOURCES="clear;nuget,GitHub,${NUGET_MODE}"
@@ -125,10 +125,10 @@ RUN mono `./vcpkg fetch nuget | tail -n 1` \
 # vcpkg  ---
 RUN mkdir /hpcc-dev/build
 RUN ./vcpkg install \
+    --x-abi-tools-use-exact-versions \
     --x-install-root=/hpcc-dev/build/vcpkg_installed \
-    --overlay-ports=./overlays \
-    --triplet=x64-linux-dynamic
-# ./vcpkg install --overlay-ports=./overlays --triplet=x64-linux-dynamic --x-install-root=/hpcc-dev/build/vcpkg_installed
+    --triplet=x64-centos-7-dynamic
+# ./vcpkg install --x-abi-tools-use-exact-versions --x-install-root=/hpcc-dev/build/vcpkg_installed --triplet=x64-centos-7-dynamic
 
 RUN mkdir -p /hpcc-dev/tools/cmake
 RUN cp -r $(dirname $(dirname `./vcpkg fetch cmake | tail -n 1`))/* /hpcc-dev/tools/cmake
@@ -139,6 +139,24 @@ RUN cp -r $(dirname $(dirname `./vcpkg fetch node | tail -n 1`))/* /hpcc-dev/too
 
 FROM base_build
 
+RUN yum makecache && yum install -y \
+    epel-release \
+    java-11-openjdk-devel \
+    python3-devel \
+    wget && \
+    yum update -y && yum install -y \
+    ccache \
+    R-core-devel && \
+    yum -y clean all && rm -rf /var/cache
+
+ENV Rcpp_package=Rcpp_0.12.19.tar.gz
+ENV RInside_package=RInside_0.2.12.tar.gz
+
+RUN wget https://cran.r-project.org/src/contrib/Archive/Rcpp/${Rcpp_package} && \
+    wget https://cran.r-project.org/src/contrib/Archive/RInside/${RInside_package} && \
+    R CMD INSTALL ${Rcpp_package} ${RInside_package} && \
+    rm -f ${Rcpp_package} ${RInside_package}
+
 WORKDIR /hpcc-dev
 
 COPY --from=vcpkg_build /hpcc-dev/build/vcpkg_installed /hpcc-dev/vcpkg_installed
@@ -151,3 +169,7 @@ RUN cp -rs /hpcc-dev/tools/cmake/bin /usr/local/ && \
     cp -rs /hpcc-dev/tools/node/include /usr/local/ && \
     cp -rs /hpcc-dev/tools/node/lib /usr/local/ && \
     cp -rs /hpcc-dev/tools/node/share /usr/local/
+
+ENTRYPOINT ["/bin/bash", "--login", "-c"]
+
+CMD ["/bin/bash"]