From cfc5c8b2287ba2e8c7a92e627da9f090c297f66f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 3 Jul 2024 19:13:15 -0400 Subject: [PATCH 1/7] Add initial macos13 templates for x64 and arm64 Signed-off-by: Peter Zhu --- packer/jenkins-agent-macos13-arm64.json | 106 ++++++++++++++++++ packer/jenkins-agent-macos13-x64.json | 106 ++++++++++++++++++ .../jenkins-agent-macos12-x64.json | 0 3 files changed, 212 insertions(+) create mode 100644 packer/jenkins-agent-macos13-arm64.json create mode 100644 packer/jenkins-agent-macos13-x64.json rename packer/{ => legacy}/jenkins-agent-macos12-x64.json (100%) diff --git a/packer/jenkins-agent-macos13-arm64.json b/packer/jenkins-agent-macos13-arm64.json new file mode 100644 index 00000000..05b6be5f --- /dev/null +++ b/packer/jenkins-agent-macos13-arm64.json @@ -0,0 +1,106 @@ +{ + "variables": { + "ami_name": "Jenkins-Agent-MacOS13-ARM64-Mac2M2ProMetal", + "os_version": "13.6.7", + "os_architecture": "arm64_mac", + "build-region": "us-east-1", + "build-vpc": "vpc-<>", + "build-subnet": "subnet-<>", + "build-secgrp": "sg-<>", + "build-time": "{{isotime \"2006-01-02T03-04-05Z\"}}", + "aws_ami_region": "us-east-1", + "root_volume_size_gb": "300" + }, + "builders": [ + { + "name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "ami_description": "{{user `ami_name`}}", + "type": "amazon-ebs", + "encrypt_boot": "false", + "region": "{{user `build-region`}}", + "ami_regions": "{{user `aws_ami_region`}}", + "instance_type": "mac2-m2pro.metal", + "ami_name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "vpc_id": "{{user `build-vpc`}}", + "subnet_id": "{{user `build-subnet`}}", + "aws_polling": { + "delay_seconds": 60, + "max_attempts": 60 + }, + "security_group_ids": "{{user `build-secgrp`}}", + "ssh_interface": "", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "name": "amzn-ec2-macos-{{user `os_version`}}-*", + "root-device-type": "ebs", + "architecture": "{{user `os_architecture`}}" + }, + "owners": [ + "amazon" + ], + "most_recent": true + }, + "associate_public_ip_address": false, + "ssh_username": "ec2-user", + "ssh_timeout": "3h", + "tenancy": "host", + "ebs_optimized": "true", + "tags": { + "Name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "OS_Version": "{{user `os_version`}}", + "User": "Packer", + "Encrypted_AMI": "False", + "Created": "{{user `build-time`}}" + }, + "launch_block_device_mappings": [ + { + "device_name": "/dev/sda1", + "volume_size": "{{user `root_volume_size_gb`}}", + "delete_on_termination": true, + "volume_type": "gp3" + } + ] + } + ], + "provisioners": [ + { + "type": "shell", + "scripts": [ + "scripts/macos/macos-disk-resize.sh" + ], + "max_retries": 3 + }, + { + "type": "shell", + "inline": [ + "sudo /usr/local/bin/ec2-macos-init clean --all" + ], + "max_retries": 3 + }, + { + "type": "shell", + "inline": [ + "sudo chsh -s /bin/bash $(whoami)" + ], + "max_retries": 3 + }, + { + "type": "file", + "source": "files/macos/bash_profile", + "destination": "/Users/ec2-user/.bash_profile" + }, + { + "type": "file", + "source": "files/macos/bashrc", + "destination": "/Users/ec2-user/.bashrc" + }, + { + "type": "shell", + "scripts": [ + "scripts/macos/macos-agentsetup.sh" + ], + "max_retries": 3 + } + ] +} diff --git a/packer/jenkins-agent-macos13-x64.json b/packer/jenkins-agent-macos13-x64.json new file mode 100644 index 00000000..5ba607b3 --- /dev/null +++ b/packer/jenkins-agent-macos13-x64.json @@ -0,0 +1,106 @@ +{ + "variables": { + "ami_name": "Jenkins-Agent-MacOS13-X64-Mac1Metal-JDK21", + "os_version": "13.6.7", + "os_architecture": "x86_64_mac", + "build-region": "us-east-1", + "build-vpc": "vpc-<>", + "build-subnet": "subnet-<>", + "build-secgrp": "sg-<>", + "build-time": "{{isotime \"2006-01-02T03-04-05Z\"}}", + "aws_ami_region": "us-east-1", + "root_volume_size_gb": "300" + }, + "builders": [ + { + "name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "ami_description": "{{user `ami_name`}}", + "type": "amazon-ebs", + "encrypt_boot": "false", + "region": "{{user `build-region`}}", + "ami_regions": "{{user `aws_ami_region`}}", + "instance_type": "mac1.metal", + "ami_name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "vpc_id": "{{user `build-vpc`}}", + "subnet_id": "{{user `build-subnet`}}", + "aws_polling": { + "delay_seconds": 60, + "max_attempts": 60 + }, + "security_group_ids": "{{user `build-secgrp`}}", + "ssh_interface": "", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "name": "amzn-ec2-macos-{{user `os_version`}}-*", + "root-device-type": "ebs", + "architecture": "{{user `os_architecture`}}" + }, + "owners": [ + "amazon" + ], + "most_recent": true + }, + "associate_public_ip_address": false, + "ssh_username": "ec2-user", + "ssh_timeout": "3h", + "tenancy": "host", + "ebs_optimized": "true", + "tags": { + "Name": "{{user `ami_name`}}-{{user `os_version`}}-{{user `build-time`}}", + "OS_Version": "{{user `os_version`}}", + "User": "Packer", + "Encrypted_AMI": "False", + "Created": "{{user `build-time`}}" + }, + "launch_block_device_mappings": [ + { + "device_name": "/dev/sda1", + "volume_size": "{{user `root_volume_size_gb`}}", + "delete_on_termination": true, + "volume_type": "gp3" + } + ] + } + ], + "provisioners": [ + { + "type": "shell", + "scripts": [ + "scripts/macos/macos-disk-resize.sh" + ], + "max_retries": 3 + }, + { + "type": "shell", + "inline": [ + "sudo /usr/local/bin/ec2-macos-init clean --all" + ], + "max_retries": 3 + }, + { + "type": "shell", + "inline": [ + "sudo chsh -s /bin/bash $(whoami)" + ], + "max_retries": 3 + }, + { + "type": "file", + "source": "files/macos/bash_profile", + "destination": "/Users/ec2-user/.bash_profile" + }, + { + "type": "file", + "source": "files/macos/bashrc", + "destination": "/Users/ec2-user/.bashrc" + }, + { + "type": "shell", + "scripts": [ + "scripts/macos/macos-agentsetup.sh" + ], + "max_retries": 3 + } + ] +} diff --git a/packer/jenkins-agent-macos12-x64.json b/packer/legacy/jenkins-agent-macos12-x64.json similarity index 100% rename from packer/jenkins-agent-macos12-x64.json rename to packer/legacy/jenkins-agent-macos12-x64.json From cc9179da9af698755ab4fdb25a30855f042a0118 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 3 Jul 2024 20:30:57 -0400 Subject: [PATCH 2/7] Tweak macos server scripts with arm64 support Signed-off-by: Peter Zhu --- packer/files/macos/bashrc | 2 +- packer/scripts/macos/macos-agentsetup.sh | 59 ++++++++++++++++-------- packer/scripts/windows/sleep.ps1 | 12 +++++ 3 files changed, 52 insertions(+), 21 deletions(-) create mode 100755 packer/scripts/windows/sleep.ps1 diff --git a/packer/files/macos/bashrc b/packer/files/macos/bashrc index 936c9f24..3a631d02 100644 --- a/packer/files/macos/bashrc +++ b/packer/files/macos/bashrc @@ -1,5 +1,5 @@ export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH -export PATH=/opt/local/bin:$PATH +export PATH=/opt/homebrew/bin:/opt/local/bin:$PATH export PATH=/Users/ec2-user/Library/Python/3.9/bin:/opt/local/Library/Frameworks/Python.framework/Versions/3.9/bin:$PATH export PATH=/usr/local/opt/grep/libexec/gnubin:$PATH export PATH=/usr/local/opt/gnu-sed/libexec/gnubin:$PATH diff --git a/packer/scripts/macos/macos-agentsetup.sh b/packer/scripts/macos/macos-agentsetup.sh index d10841b5..13f27d94 100644 --- a/packer/scripts/macos/macos-agentsetup.sh +++ b/packer/scripts/macos/macos-agentsetup.sh @@ -4,44 +4,63 @@ sudo mkdir -p /var/jenkins/ sudo chown -R ec2-user:staff /var/jenkins -## Setup brew Defaults -/usr/local/bin/brew update --preinstall -/usr/local/bin/brew upgrade -/usr/local/bin/brew install curl -/usr/local/bin/brew install coreutils -/usr/local/bin/brew install gnu-sed -/usr/local/bin/brew install grep -/usr/local/bin/brew install wget -/usr/local/bin/brew install maven -/usr/local/bin/brew install dpkg - ## Array of JDK versions in form of version@URL@priority jdk_versions=( - "8@https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u392-b08/OpenJDK8U-jdk_x64_mac_hotspot_8u392b08.tar.gz@1" - "11@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.21%2B9/OpenJDK11U-jdk_x64_mac_hotspot_11.0.21_9.tar.gz@100" - "17@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.9%2B9/OpenJDK17U-jdk_x64_mac_hotspot_17.0.9_9.tar.gz@1" + "8@https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u412-b08/OpenJDK8U-jdk_x64_mac_hotspot_8u412b08.tar.gz@1" + "11@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_x64_mac_hotspot_11.0.23_9.tar.gz@100" + "17@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_mac_hotspot_17.0.11_9.tar.gz@1" "19@https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_x64_mac_hotspot_19.0.2_7.tar.gz@1" "20@https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.2%2B9/OpenJDK20U-jdk_x64_mac_hotspot_20.0.2_9.tar.gz@1" "21@https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_x64_mac_hotspot_21.0.1_12.tar.gz@1" ) +## Setup brew Defaults + +BREW_PATH=/usr/local/bin +if [ "$ARCH" = "arm64" ]; then + BREW_PATH=/opt/homebrew/bin + sudo softwareupdate --install-rosetta --agree-to-license # Install rosetta in case there is no macOS arm64 specific binaries + jdk_versions=( + # Use Oracle HotSpot version as Adoptium does not have macOS arm64 support on JDK8 + # https://github.com/adoptium/adoptium/issues/96 + # We can also use rosetta but a bit slower so default to Oracle HotSpot for now + "8@https://ci.opensearch.org/ci/dbc/tools/jdk-8u411-macosx-aarch64.tar.gz@1" + "11@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_aarch64_mac_hotspot_11.0.23_9.tar.gz@100" + "17@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_aarch64_mac_hotspot_17.0.11_9.tar.gz@1" + "19@https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_aarch64_mac_hotspot_19.0.2_7.tar.gz@1" + "20@https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.2%2B9/OpenJDK20U-jdk_aarch64_mac_hotspot_20.0.2_9.tar.gz@1" + "21@https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.3%2B9/OpenJDK21U-jdk_aarch64_mac_hotspot_21.0.3_9.tar.gz@1" + ) +fi +$BREW_PATH/brew update --preinstall +$BREW_PATH/brew upgrade +$BREW_PATH/brew install curl +$BREW_PATH/brew install coreutils +$BREW_PATH/brew install gnu-sed +$BREW_PATH/brew install grep +$BREW_PATH/brew install wget +$BREW_PATH/brew install maven +$BREW_PATH/brew install dpkg +$BREW_PATH/brew install ca-certificates + + ## Loop through JDK versions and install them for version_info in "${jdk_versions[@]}"; do version_num=$(echo "$version_info" | cut -d '@' -f 1) version_url=$(echo "$version_info" | cut -d '@' -f 2) version_priority=$(echo "$version_info" | cut -d '@' -f 3) sudo mkdir -p "/opt/java/openjdk-${version_num}/" - /usr/local/bin/wget "$version_url" -O "openjdk-${version_num}.tar.gz" + $BREW_PATH/wget "$version_url" -O "openjdk-${version_num}.tar.gz" sudo tar -xzf "openjdk-${version_num}.tar.gz" -C "/opt/java/openjdk-${version_num}/" --strip-components=1 - /usr/local/bin/update-alternatives --install /usr/local/bin/java java "/opt/java/openjdk-${version_num}/Contents/Home/bin/java" ${version_priority} + $BREW_PATH/update-alternatives --install /usr/local/bin/java java "/opt/java/openjdk-${version_num}/Contents/Home/bin/java" ${version_priority} done ## Set default Java to 21 -/usr/local/bin/update-alternatives --set java "$(/usr/local/bin/update-alternatives --list java | grep openjdk-21)" +$BREW_PATH/update-alternatives --set java `$BREW_PATH/update-alternatives --list java | grep openjdk-21` ## Install MacPorts and python39 rm -rf /opt/local/etc/macports /opt/local/var/macports -/usr/local/bin/wget https://github.com/macports/macports-base/releases/download/v2.9.3/MacPorts-2.9.3.tar.gz +$BREW_PATH/wget https://github.com/macports/macports-base/releases/download/v2.9.3/MacPorts-2.9.3.tar.gz tar -xvf MacPorts-2.9.3.tar.gz cd MacPorts-2.9.3 ./configure && make && sudo make install @@ -53,9 +72,9 @@ sudo port select --set python python39 sudo port select --set python3 python39 ## Install pip and pip packages -/usr/local/bin/wget https://bootstrap.pypa.io/get-pip.py +$BREW_PATH/wget https://bootstrap.pypa.io/get-pip.py python3 get-pip.py export PATH=/Users/ec2-user/Library/Python/3.9/bin:/opt/local/Library/Frameworks/Python.framework/Versions/3.9/bin:$PATH pip install pipenv==2023.6.12 -pip install awscli==1.22.12 +pip install awscli==1.32.17 pip install cmake==3.23.3 diff --git a/packer/scripts/windows/sleep.ps1 b/packer/scripts/windows/sleep.ps1 new file mode 100755 index 00000000..ed9e18e7 --- /dev/null +++ b/packer/scripts/windows/sleep.ps1 @@ -0,0 +1,12 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +$sleepseconds = [int]$args[0] + +echo "Sleep $sleepseconds" + +Start-Sleep -s $sleepseconds From d9d76e29c67ea35f89e5b62245c0e101da1f06fb Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 8 Jul 2024 17:54:25 -0400 Subject: [PATCH 3/7] Add more fixes Signed-off-by: Peter Zhu --- packer/jenkins-agent-macos13-arm64.json | 7 ------- packer/jenkins-agent-macos13-x64.json | 7 ------- packer/scripts/macos/macos-agentsetup.sh | 25 ++++++++++++------------ 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/packer/jenkins-agent-macos13-arm64.json b/packer/jenkins-agent-macos13-arm64.json index 05b6be5f..9cf59d6d 100644 --- a/packer/jenkins-agent-macos13-arm64.json +++ b/packer/jenkins-agent-macos13-arm64.json @@ -64,13 +64,6 @@ } ], "provisioners": [ - { - "type": "shell", - "scripts": [ - "scripts/macos/macos-disk-resize.sh" - ], - "max_retries": 3 - }, { "type": "shell", "inline": [ diff --git a/packer/jenkins-agent-macos13-x64.json b/packer/jenkins-agent-macos13-x64.json index 5ba607b3..9698e988 100644 --- a/packer/jenkins-agent-macos13-x64.json +++ b/packer/jenkins-agent-macos13-x64.json @@ -64,13 +64,6 @@ } ], "provisioners": [ - { - "type": "shell", - "scripts": [ - "scripts/macos/macos-disk-resize.sh" - ], - "max_retries": 3 - }, { "type": "shell", "inline": [ diff --git a/packer/scripts/macos/macos-agentsetup.sh b/packer/scripts/macos/macos-agentsetup.sh index 13f27d94..6f1e729c 100644 --- a/packer/scripts/macos/macos-agentsetup.sh +++ b/packer/scripts/macos/macos-agentsetup.sh @@ -7,11 +7,11 @@ sudo chown -R ec2-user:staff /var/jenkins ## Array of JDK versions in form of version@URL@priority jdk_versions=( "8@https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u412-b08/OpenJDK8U-jdk_x64_mac_hotspot_8u412b08.tar.gz@1" - "11@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_x64_mac_hotspot_11.0.23_9.tar.gz@100" + "11@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_x64_mac_hotspot_11.0.23_9.tar.gz@1" "17@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_x64_mac_hotspot_17.0.11_9.tar.gz@1" "19@https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_x64_mac_hotspot_19.0.2_7.tar.gz@1" "20@https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.2%2B9/OpenJDK20U-jdk_x64_mac_hotspot_20.0.2_9.tar.gz@1" - "21@https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_x64_mac_hotspot_21.0.1_12.tar.gz@1" + "21@https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_x64_mac_hotspot_21.0.1_12.tar.gz@100" ) ## Setup brew Defaults @@ -25,11 +25,11 @@ if [ "$ARCH" = "arm64" ]; then # https://github.com/adoptium/adoptium/issues/96 # We can also use rosetta but a bit slower so default to Oracle HotSpot for now "8@https://ci.opensearch.org/ci/dbc/tools/jdk-8u411-macosx-aarch64.tar.gz@1" - "11@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_aarch64_mac_hotspot_11.0.23_9.tar.gz@100" + "11@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_aarch64_mac_hotspot_11.0.23_9.tar.gz@1" "17@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_aarch64_mac_hotspot_17.0.11_9.tar.gz@1" "19@https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_aarch64_mac_hotspot_19.0.2_7.tar.gz@1" "20@https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.2%2B9/OpenJDK20U-jdk_aarch64_mac_hotspot_20.0.2_9.tar.gz@1" - "21@https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.3%2B9/OpenJDK21U-jdk_aarch64_mac_hotspot_21.0.3_9.tar.gz@1" + "21@https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.3%2B9/OpenJDK21U-jdk_aarch64_mac_hotspot_21.0.3_9.tar.gz@100" ) fi $BREW_PATH/brew update --preinstall @@ -42,6 +42,7 @@ $BREW_PATH/brew install wget $BREW_PATH/brew install maven $BREW_PATH/brew install dpkg $BREW_PATH/brew install ca-certificates +$BREW_PATH/brew cleanup ## Loop through JDK versions and install them @@ -50,18 +51,18 @@ for version_info in "${jdk_versions[@]}"; do version_url=$(echo "$version_info" | cut -d '@' -f 2) version_priority=$(echo "$version_info" | cut -d '@' -f 3) sudo mkdir -p "/opt/java/openjdk-${version_num}/" - $BREW_PATH/wget "$version_url" -O "openjdk-${version_num}.tar.gz" + /usr/local/bin/wget -nv "$version_url" -O "openjdk-${version_num}.tar.gz" sudo tar -xzf "openjdk-${version_num}.tar.gz" -C "/opt/java/openjdk-${version_num}/" --strip-components=1 - $BREW_PATH/update-alternatives --install /usr/local/bin/java java "/opt/java/openjdk-${version_num}/Contents/Home/bin/java" ${version_priority} + /usr/local/bin/update-alternatives --install /usr/local/bin/java java "/opt/java/openjdk-${version_num}/Contents/Home/bin/java" ${version_priority} done ## Set default Java to 21 -$BREW_PATH/update-alternatives --set java `$BREW_PATH/update-alternatives --list java | grep openjdk-21` +/usr/local/bin/update-alternatives --set java `/usr/local/bin/update-alternatives --list java | grep openjdk-21` ## Install MacPorts and python39 -rm -rf /opt/local/etc/macports /opt/local/var/macports -$BREW_PATH/wget https://github.com/macports/macports-base/releases/download/v2.9.3/MacPorts-2.9.3.tar.gz -tar -xvf MacPorts-2.9.3.tar.gz +sudo rm -rf /opt/local/etc/macports /opt/local/var/macports +/usr/local/bin/wget -nv https://github.com/macports/macports-base/releases/download/v2.9.3/MacPorts-2.9.3.tar.gz +tar -xzf MacPorts-2.9.3.tar.gz cd MacPorts-2.9.3 ./configure && make && sudo make install cd .. && rm -rf MacPorts-2.9.3.tar.gz @@ -72,9 +73,9 @@ sudo port select --set python python39 sudo port select --set python3 python39 ## Install pip and pip packages -$BREW_PATH/wget https://bootstrap.pypa.io/get-pip.py +/usr/local/bin/wget -nv https://bootstrap.pypa.io/get-pip.py python3 get-pip.py export PATH=/Users/ec2-user/Library/Python/3.9/bin:/opt/local/Library/Frameworks/Python.framework/Versions/3.9/bin:$PATH pip install pipenv==2023.6.12 pip install awscli==1.32.17 -pip install cmake==3.23.3 +pip install cmake==3.26.4 From 7a98900459666f2a6f49a28e37d1df86806af2a6 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 10 Jul 2024 17:59:16 -0400 Subject: [PATCH 4/7] More changes to enable imdsv2 Signed-off-by: Peter Zhu --- packer/jenkins-agent-macos13-arm64.json | 5 +++++ packer/jenkins-agent-macos13-x64.json | 5 +++++ packer/scripts/macos/macos-agentsetup.sh | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packer/jenkins-agent-macos13-arm64.json b/packer/jenkins-agent-macos13-arm64.json index 9cf59d6d..672256e5 100644 --- a/packer/jenkins-agent-macos13-arm64.json +++ b/packer/jenkins-agent-macos13-arm64.json @@ -42,6 +42,11 @@ "most_recent": true }, "associate_public_ip_address": false, + "metadata_options": { + "http_endpoint": "enabled", + "http_tokens": "required", + "http_put_response_hop_limit": 1 + }, "ssh_username": "ec2-user", "ssh_timeout": "3h", "tenancy": "host", diff --git a/packer/jenkins-agent-macos13-x64.json b/packer/jenkins-agent-macos13-x64.json index 9698e988..30901dc4 100644 --- a/packer/jenkins-agent-macos13-x64.json +++ b/packer/jenkins-agent-macos13-x64.json @@ -42,6 +42,11 @@ "most_recent": true }, "associate_public_ip_address": false, + "metadata_options": { + "http_endpoint": "enabled", + "http_tokens": "required", + "http_put_response_hop_limit": 1 + }, "ssh_username": "ec2-user", "ssh_timeout": "3h", "tenancy": "host", diff --git a/packer/scripts/macos/macos-agentsetup.sh b/packer/scripts/macos/macos-agentsetup.sh index 6f1e729c..1af9b1eb 100644 --- a/packer/scripts/macos/macos-agentsetup.sh +++ b/packer/scripts/macos/macos-agentsetup.sh @@ -15,7 +15,7 @@ jdk_versions=( ) ## Setup brew Defaults - +ARCH=`uname -m` BREW_PATH=/usr/local/bin if [ "$ARCH" = "arm64" ]; then BREW_PATH=/opt/homebrew/bin From 75802cc7d0895e7659bb1be7acc56fd90ba69aa7 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 11 Jul 2024 16:49:55 -0400 Subject: [PATCH 5/7] Add missing brewpath for arm64 macOS Signed-off-by: Peter Zhu --- packer/scripts/macos/macos-agentsetup.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packer/scripts/macos/macos-agentsetup.sh b/packer/scripts/macos/macos-agentsetup.sh index 1af9b1eb..87404dfd 100644 --- a/packer/scripts/macos/macos-agentsetup.sh +++ b/packer/scripts/macos/macos-agentsetup.sh @@ -19,7 +19,7 @@ ARCH=`uname -m` BREW_PATH=/usr/local/bin if [ "$ARCH" = "arm64" ]; then BREW_PATH=/opt/homebrew/bin - sudo softwareupdate --install-rosetta --agree-to-license # Install rosetta in case there is no macOS arm64 specific binaries + sudo zsh -c "softwareupdate --install-rosetta --agree-to-license" jdk_versions=( # Use Oracle HotSpot version as Adoptium does not have macOS arm64 support on JDK8 # https://github.com/adoptium/adoptium/issues/96 @@ -51,17 +51,17 @@ for version_info in "${jdk_versions[@]}"; do version_url=$(echo "$version_info" | cut -d '@' -f 2) version_priority=$(echo "$version_info" | cut -d '@' -f 3) sudo mkdir -p "/opt/java/openjdk-${version_num}/" - /usr/local/bin/wget -nv "$version_url" -O "openjdk-${version_num}.tar.gz" + $BREW_PATH/wget -nv "$version_url" -O "openjdk-${version_num}.tar.gz" sudo tar -xzf "openjdk-${version_num}.tar.gz" -C "/opt/java/openjdk-${version_num}/" --strip-components=1 - /usr/local/bin/update-alternatives --install /usr/local/bin/java java "/opt/java/openjdk-${version_num}/Contents/Home/bin/java" ${version_priority} + $BREW_PATH/update-alternatives --install /usr/local/bin/java java "/opt/java/openjdk-${version_num}/Contents/Home/bin/java" ${version_priority} done ## Set default Java to 21 -/usr/local/bin/update-alternatives --set java `/usr/local/bin/update-alternatives --list java | grep openjdk-21` +$BREW_PATH/update-alternatives --set java `$BREW_PATH/update-alternatives --list java | grep openjdk-21` ## Install MacPorts and python39 sudo rm -rf /opt/local/etc/macports /opt/local/var/macports -/usr/local/bin/wget -nv https://github.com/macports/macports-base/releases/download/v2.9.3/MacPorts-2.9.3.tar.gz +$BREW_PATH/wget -nv https://github.com/macports/macports-base/releases/download/v2.9.3/MacPorts-2.9.3.tar.gz tar -xzf MacPorts-2.9.3.tar.gz cd MacPorts-2.9.3 ./configure && make && sudo make install @@ -73,7 +73,7 @@ sudo port select --set python python39 sudo port select --set python3 python39 ## Install pip and pip packages -/usr/local/bin/wget -nv https://bootstrap.pypa.io/get-pip.py +$BREW_PATH/wget -nv https://bootstrap.pypa.io/get-pip.py python3 get-pip.py export PATH=/Users/ec2-user/Library/Python/3.9/bin:/opt/local/Library/Frameworks/Python.framework/Versions/3.9/bin:$PATH pip install pipenv==2023.6.12 From d165f1b1e7a39fd415f7ea4c5c78edea88364f7e Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 12 Jul 2024 13:07:49 -0400 Subject: [PATCH 6/7] Support macos arm64 java installation now Signed-off-by: Peter Zhu --- packer/scripts/macos/macos-agentsetup.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packer/scripts/macos/macos-agentsetup.sh b/packer/scripts/macos/macos-agentsetup.sh index 87404dfd..e73ddc30 100644 --- a/packer/scripts/macos/macos-agentsetup.sh +++ b/packer/scripts/macos/macos-agentsetup.sh @@ -19,11 +19,13 @@ ARCH=`uname -m` BREW_PATH=/usr/local/bin if [ "$ARCH" = "arm64" ]; then BREW_PATH=/opt/homebrew/bin - sudo zsh -c "softwareupdate --install-rosetta --agree-to-license" + echo "Install rosetta, ignore 'Package Authoring Error: 052-96248'" + sudo /usr/sbin/softwareupdate --install-rosetta --agree-to-license + + # Use Oracle HotSpot version as Adoptium does not have macOS arm64 support on JDK8 + # https://github.com/adoptium/adoptium/issues/96 + # We can also use rosetta but a bit slower so default to Oracle HotSpot for now jdk_versions=( - # Use Oracle HotSpot version as Adoptium does not have macOS arm64 support on JDK8 - # https://github.com/adoptium/adoptium/issues/96 - # We can also use rosetta but a bit slower so default to Oracle HotSpot for now "8@https://ci.opensearch.org/ci/dbc/tools/jdk-8u411-macosx-aarch64.tar.gz@1" "11@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.23%2B9/OpenJDK11U-jdk_aarch64_mac_hotspot_11.0.23_9.tar.gz@1" "17@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.11%2B9/OpenJDK17U-jdk_aarch64_mac_hotspot_17.0.11_9.tar.gz@1" @@ -53,11 +55,13 @@ for version_info in "${jdk_versions[@]}"; do sudo mkdir -p "/opt/java/openjdk-${version_num}/" $BREW_PATH/wget -nv "$version_url" -O "openjdk-${version_num}.tar.gz" sudo tar -xzf "openjdk-${version_num}.tar.gz" -C "/opt/java/openjdk-${version_num}/" --strip-components=1 - $BREW_PATH/update-alternatives --install /usr/local/bin/java java "/opt/java/openjdk-${version_num}/Contents/Home/bin/java" ${version_priority} + JAVA_PATH="/opt/java/openjdk-${version_num}/Contents/Home/bin" + $JAVA_PATH/java -version + sudo $BREW_PATH/update-alternatives --install /usr/local/bin/java java "$JAVA_PATH/java" ${version_priority} done ## Set default Java to 21 -$BREW_PATH/update-alternatives --set java `$BREW_PATH/update-alternatives --list java | grep openjdk-21` +sudo $BREW_PATH/update-alternatives --set java `$BREW_PATH/update-alternatives --list java | grep openjdk-21` ## Install MacPorts and python39 sudo rm -rf /opt/local/etc/macports /opt/local/var/macports From bc896afb789c0b0eccac0871e32a0e5b293c452d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 12 Jul 2024 16:25:19 -0400 Subject: [PATCH 7/7] Remove extra scripts Signed-off-by: Peter Zhu --- packer/scripts/windows/sleep.ps1 | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100755 packer/scripts/windows/sleep.ps1 diff --git a/packer/scripts/windows/sleep.ps1 b/packer/scripts/windows/sleep.ps1 deleted file mode 100755 index ed9e18e7..00000000 --- a/packer/scripts/windows/sleep.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -$sleepseconds = [int]$args[0] - -echo "Sleep $sleepseconds" - -Start-Sleep -s $sleepseconds