From 69eba910db8f9abf73983e79d38b292709b3f36b Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 15 Dec 2023 15:56:54 -0500 Subject: [PATCH 1/8] Remove references to admin:admin Signed-off-by: Derek Ho --- HANDBOOK.md | 2 +- build.gradle | 1 + scripts/integtest.sh | 121 ------------------------------------------- 3 files changed, 2 insertions(+), 122 deletions(-) delete mode 100755 scripts/integtest.sh diff --git a/HANDBOOK.md b/HANDBOOK.md index ebb63090..fe1dfa0d 100644 --- a/HANDBOOK.md +++ b/HANDBOOK.md @@ -66,7 +66,7 @@ plugins.security.nodes_dn_dynamic_config_enabled: true Allow connections from follower cluster on the leader as follows ```bash -curl -k -u admin:admin -XPUT "https://${LEADER}/_plugins/_security/api/nodesdn/follower" \ +curl -k -u admin: -XPUT "https://${LEADER}/_plugins/_security/api/nodesdn/follower" \ -H 'Content-type: application/json' \ -d'{"nodes_dn": ["CN=follower.example.com"]}' ``` diff --git a/build.gradle b/build.gradle index aa4422a7..1473afef 100644 --- a/build.gradle +++ b/build.gradle @@ -682,6 +682,7 @@ clusters.each { name -> if (securityEnabled) { plugin(provider(securityPluginOld)) cliSetup("opensearch-security/install_demo_configuration.sh", "-y") + } // Currently fetching the ARCHIVE distribution fails on mac as it tries to fetch the Mac specific "DARWIN" distribution // for Opensearch which is not publish yet. Changing this to INTEG_TEST to make it work on mac. diff --git a/scripts/integtest.sh b/scripts/integtest.sh deleted file mode 100755 index 1a0498ae..00000000 --- a/scripts/integtest.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash - -set -e - -function usage() { - echo "" - echo "This script is used to run integration tests for plugin installed on a remote OpenSearch/Dashboards cluster." - echo "--------------------------------------------------------------------------" - echo "Usage: $0 [args]" - echo "" - echo "Optional arguments:" - echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not." - echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true." - echo -e "-h Print this message." - echo -e "-v OPENSEARCH_VERSION\t, no defaults" - echo -e "-n SNAPSHOT\t\t, defaults to false" - echo "Required arguments:" - echo "Single cluster test:" - echo -e "-b BIND_ADDRESS\t\t, IP or domain name for the cluster location." - echo -e "-p BIND_PORT\t\t, port for the cluster location." - echo -e "-t TRANSPORT_PORT\t, defaults to 9300, can be changed to any port for the cluster location." - echo "--------------------------------------------------------------------------" - echo "Multi cluster test:" - echo -e "-e Comma seperated endpoint:port, ex: localhost:9200:9300,localhost:9201:9301... ." - echo "--------------------------------------------------------------------------" -} - -while getopts ":h:b:p:t:e:s:c:v:" arg; do - case $arg in - h) - usage - exit 1 - ;; - b) - BIND_ADDRESS=$OPTARG - ;; - p) - BIND_PORT=$OPTARG - ;; - t) - TRANSPORT_PORT=$OPTARG - ;; - e) - ENDPOINT_LIST=$OPTARG - ;; - s) - SECURITY_ENABLED=$OPTARG - ;; - c) - CREDENTIAL=$OPTARG - ;; - v) - # Do nothing as we're not consuming this param. - ;; - :) - echo "-${OPTARG} requires an argument" - usage - exit 1 - ;; - ?) - echo "Invalid option: -${OPTARG}" - exit 1 - ;; - esac -done - -# Common starts -if [ -z "$SECURITY_ENABLED" ] -then - SECURITY_ENABLED="true" -fi - -if [ -z "$CREDENTIAL" ] -then - CREDENTIAL="admin:admin" -fi - -USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` -PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` -# Common ends - - -# Check if test is run on multiple cluster - -if [ -z "$BIND_ADDRESS" ] || [ -z "$BIND_PORT" ] -then - #Proceeding with multi cluster test - if [ -z "$ENDPOINT_LIST" ] - then - echo "requires an argument -e " - usage - exit 1 - fi - - data=$(python3 -c "import json; cluster=$ENDPOINT_LIST ; data_nodes=cluster; print(data_nodes[0][\"data_nodes\"][0][\"endpoint\"],':',data_nodes[0][\"data_nodes\"][0][\"port\"],':',data_nodes[0][\"data_nodes\"][0][\"transport\"],',',data_nodes[1][\"data_nodes\"][0][\"endpoint\"],':',data_nodes[1][\"data_nodes\"][0][\"port\"],':',data_nodes[1][\"data_nodes\"][0][\"transport\"])" | tr -d "[:blank:]") - - - leader=$(echo $data | cut -d ',' -f1 | cut -d ':' -f1,2 ) - follower=$(echo $data | cut -d ',' -f2 | cut -d ':' -f1,2 ) - echo "leader: $leader" - echo "follower: $follower" - - # Get number of nodes, assuming both leader and follower have same number of nodes - numNodes=$((${follower##*:} - ${leader##*:})) - echo "numNodes: $numNodes" - - LTRANSPORT_PORT=$(echo $data | cut -d ',' -f1 | cut -d ':' -f1,3 ) - FTRANSPORT_PORT=$(echo $data | cut -d ',' -f2 | cut -d ':' -f1,3 ) - echo "LTRANSPORT_PORT: $LTRANSPORT_PORT" - echo "FTRANSPORT_PORT: $FTRANSPORT_PORT" - - eval "./gradlew integTestRemote -Dleader.http_host=\"$leader\" -Dfollower.http_host=\"$follower\" -Dfollower.transport_host=\"$FTRANSPORT_PORT\" -Dleader.transport_host=\"$LTRANSPORT_PORT\" -Dsecurity_enabled=\"$SECURITY_ENABLED\" -Duser=\"$USERNAME\" -Dpassword=\"$PASSWORD\" -PnumNodes=$numNodes --console=plain " - -else - # Single cluster - if [ -z "$TRANSPORT_PORT" ] - then - TRANSPORT_PORT="9300" - fi - ./gradlew singleClusterSanityTest -Dfollower.http_host="$BIND_ADDRESS:$BIND_PORT" -Dfollower.transport_host="$BIND_ADDRESS:$TRANSPORT_PORT" -Dsecurity_enabled=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain -fi From 911a23cc523df9fac49ca6bb781f4a572405ef74 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 15 Dec 2023 16:08:39 -0500 Subject: [PATCH 2/8] Try to pass in initial admin password via env variable Signed-off-by: Derek Ho --- build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 1473afef..aec45ce5 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { + System.setProperty("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin") isSnapshot = "true" == System.getProperty("build.snapshot", "true") opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") buildVersionQualifier = System.getProperty("build.version_qualifier", "") @@ -681,8 +682,8 @@ clusters.each { name -> if (securityEnabled) { plugin(provider(securityPluginOld)) - cliSetup("opensearch-security/install_demo_configuration.sh", "-y") - + cliSetup("opensearch-security/install_demo_configuration.sh", "-y", "-t") + } // Currently fetching the ARCHIVE distribution fails on mac as it tries to fetch the Mac specific "DARWIN" distribution // for Opensearch which is not publish yet. Changing this to INTEG_TEST to make it work on mac. From 4c26601c55bf7ff3f5fe274f979197ac78ebdc25 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 15 Dec 2023 16:29:54 -0500 Subject: [PATCH 3/8] Try using the env variable Signed-off-by: Derek Ho --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index aec45ce5..d39a36d0 100644 --- a/build.gradle +++ b/build.gradle @@ -430,7 +430,7 @@ def configureCluster(OpenSearchCluster cluster, Boolean securityEnabled) { } CrossClusterWaitForHttpResource wait = new CrossClusterWaitForHttpResource(protocol, cluster.getFirstNode().getHttpSocketURI(), cluster.nodes.size()) wait.setUsername("admin") - wait.setPassword("admin") + wait.setPassword(System.getProperty("OPENSEARCH_INITIAL_ADMIN_PASSWORD")) return wait.wait(500) } From 859d45f7e2e71d62efe472d5ed40d88851703f33 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 27 Dec 2023 16:25:28 -0500 Subject: [PATCH 4/8] Revert file deletion and add logic for integration tests Signed-off-by: Derek Ho --- build.gradle | 4 +- scripts/integtest.sh | 130 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 scripts/integtest.sh diff --git a/build.gradle b/build.gradle index d39a36d0..9ffc45c7 100644 --- a/build.gradle +++ b/build.gradle @@ -34,7 +34,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - System.setProperty("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "admin") + System.setProperty("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "myStrongPassword123!") isSnapshot = "true" == System.getProperty("build.snapshot", "true") opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") buildVersionQualifier = System.getProperty("build.version_qualifier", "") @@ -682,7 +682,7 @@ clusters.each { name -> if (securityEnabled) { plugin(provider(securityPluginOld)) - cliSetup("opensearch-security/install_demo_configuration.sh", "-y", "-t") + cliSetup("opensearch-security/install_demo_configuration.sh", "-y") } // Currently fetching the ARCHIVE distribution fails on mac as it tries to fetch the Mac specific "DARWIN" distribution diff --git a/scripts/integtest.sh b/scripts/integtest.sh new file mode 100644 index 00000000..436d518f --- /dev/null +++ b/scripts/integtest.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +set -e + +function usage() { + echo "" + echo "This script is used to run integration tests for plugin installed on a remote OpenSearch/Dashboards cluster." + echo "--------------------------------------------------------------------------" + echo "Usage: $0 [args]" + echo "" + echo "Optional arguments:" + echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not." + echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true." + echo -e "-h Print this message." + echo -e "-v OPENSEARCH_VERSION\t, no defaults" + echo -e "-n SNAPSHOT\t\t, defaults to false" + echo "Required arguments:" + echo "Single cluster test:" + echo -e "-b BIND_ADDRESS\t\t, IP or domain name for the cluster location." + echo -e "-p BIND_PORT\t\t, port for the cluster location." + echo -e "-t TRANSPORT_PORT\t, defaults to 9300, can be changed to any port for the cluster location." + echo "--------------------------------------------------------------------------" + echo "Multi cluster test:" + echo -e "-e Comma seperated endpoint:port, ex: localhost:9200:9300,localhost:9201:9301... ." + echo "--------------------------------------------------------------------------" +} + +while getopts ":h:b:p:t:e:s:c:v:" arg; do + case $arg in + h) + usage + exit 1 + ;; + b) + BIND_ADDRESS=$OPTARG + ;; + p) + BIND_PORT=$OPTARG + ;; + t) + TRANSPORT_PORT=$OPTARG + ;; + e) + ENDPOINT_LIST=$OPTARG + ;; + s) + SECURITY_ENABLED=$OPTARG + ;; + c) + CREDENTIAL=$OPTARG + ;; + v) + # Do nothing as we're not consuming this param. + ;; + :) + echo "-${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}" + exit 1 + ;; + esac +done + +# Common starts +if [ -z "$SECURITY_ENABLED" ] +then + SECURITY_ENABLED="true" +fi + +IFS='.' read -ra version_array <<< "$version" + + + +if [ -z "$CREDENTIAL" ] +then + # Starting in 2.12.0, security demo configuration script requires an initial admin password + if (( ${version_array[0]} > 2 || (${version_array[0]} == 2 && ${version_array[1]} >= 12) )); then + CREDENTIAL="admin:myStrongPassword123!" + else + CREDENTIAL="admin:admin" + fi +fi + +USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` +PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` +# Common ends + + +# Check if test is run on multiple cluster + +if [ -z "$BIND_ADDRESS" ] || [ -z "$BIND_PORT" ] +then + #Proceeding with multi cluster test + if [ -z "$ENDPOINT_LIST" ] + then + echo "requires an argument -e " + usage + exit 1 + fi + + data=$(python3 -c "import json; cluster=$ENDPOINT_LIST ; data_nodes=cluster; print(data_nodes[0][\"data_nodes\"][0][\"endpoint\"],':',data_nodes[0][\"data_nodes\"][0][\"port\"],':',data_nodes[0][\"data_nodes\"][0][\"transport\"],',',data_nodes[1][\"data_nodes\"][0][\"endpoint\"],':',data_nodes[1][\"data_nodes\"][0][\"port\"],':',data_nodes[1][\"data_nodes\"][0][\"transport\"])" | tr -d "[:blank:]") + + + leader=$(echo $data | cut -d ',' -f1 | cut -d ':' -f1,2 ) + follower=$(echo $data | cut -d ',' -f2 | cut -d ':' -f1,2 ) + echo "leader: $leader" + echo "follower: $follower" + + # Get number of nodes, assuming both leader and follower have same number of nodes + numNodes=$((${follower##*:} - ${leader##*:})) + echo "numNodes: $numNodes" + + LTRANSPORT_PORT=$(echo $data | cut -d ',' -f1 | cut -d ':' -f1,3 ) + FTRANSPORT_PORT=$(echo $data | cut -d ',' -f2 | cut -d ':' -f1,3 ) + echo "LTRANSPORT_PORT: $LTRANSPORT_PORT" + echo "FTRANSPORT_PORT: $FTRANSPORT_PORT" + + eval "./gradlew integTestRemote -Dleader.http_host=\"$leader\" -Dfollower.http_host=\"$follower\" -Dfollower.transport_host=\"$FTRANSPORT_PORT\" -Dleader.transport_host=\"$LTRANSPORT_PORT\" -Dsecurity_enabled=\"$SECURITY_ENABLED\" -Duser=\"$USERNAME\" -Dpassword=\"$PASSWORD\" -PnumNodes=$numNodes --console=plain " + +else + # Single cluster + if [ -z "$TRANSPORT_PORT" ] + then + TRANSPORT_PORT="9300" + fi + ./gradlew singleClusterSanityTest -Dfollower.http_host="$BIND_ADDRESS:$BIND_PORT" -Dfollower.transport_host="$BIND_ADDRESS:$TRANSPORT_PORT" -Dsecurity_enabled=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain +fi \ No newline at end of file From 7bb465eff25c0fb19e133c6547458273a97bb77f Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Wed, 27 Dec 2023 16:41:03 -0500 Subject: [PATCH 5/8] fix env variable Signed-off-by: Derek Ho --- scripts/integtest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/integtest.sh b/scripts/integtest.sh index 436d518f..ef573e40 100644 --- a/scripts/integtest.sh +++ b/scripts/integtest.sh @@ -70,7 +70,7 @@ then SECURITY_ENABLED="true" fi -IFS='.' read -ra version_array <<< "$version" +IFS='.' read -ra version_array <<< "$VERSION" From b47c3d9be6f1a73b6c9232845b7fb01d37a036b5 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Thu, 28 Dec 2023 15:39:28 -0500 Subject: [PATCH 6/8] Update logic to be the same across all repos Signed-off-by: Derek Ho --- scripts/integtest.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/integtest.sh b/scripts/integtest.sh index ef573e40..21cf8dfa 100644 --- a/scripts/integtest.sh +++ b/scripts/integtest.sh @@ -70,17 +70,13 @@ then SECURITY_ENABLED="true" fi -IFS='.' read -ra version_array <<< "$VERSION" - - - if [ -z "$CREDENTIAL" ] then - # Starting in 2.12.0, security demo configuration script requires an initial admin password - if (( ${version_array[0]} > 2 || (${version_array[0]} == 2 && ${version_array[1]} >= 12) )); then - CREDENTIAL="admin:myStrongPassword123!" + # OpenSearch 2.12 onwards security plugins requires a password to be set to setup admin user + if [ "$(echo "${VERSION} 2.12" | awk '{print ($1 >= $2)}')" -eq 1 ]; then + CREDENTIAL="admin:myStrongPassword123!" else - CREDENTIAL="admin:admin" + CREDENTIAL="admin:admin" fi fi From a1ba4e35c27851a08847f3ea1480dc280051487f Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 29 Dec 2023 11:20:11 -0500 Subject: [PATCH 7/8] Keep old logic Signed-off-by: Derek Ho --- scripts/integtest.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/integtest.sh b/scripts/integtest.sh index 21cf8dfa..b64c0e79 100644 --- a/scripts/integtest.sh +++ b/scripts/integtest.sh @@ -70,10 +70,12 @@ then SECURITY_ENABLED="true" fi +IFS='.' read -ra version_array <<< "$VERSION" + if [ -z "$CREDENTIAL" ] then - # OpenSearch 2.12 onwards security plugins requires a password to be set to setup admin user - if [ "$(echo "${VERSION} 2.12" | awk '{print ($1 >= $2)}')" -eq 1 ]; then + # Starting in 2.12.0, security demo configuration script requires an initial admin password + if (( ${version_array[0]} > 2 || (${version_array[0]} == 2 && ${version_array[1]} >= 12) )); then CREDENTIAL="admin:myStrongPassword123!" else CREDENTIAL="admin:admin" From 2925971c26185c7f21d7dc35dd7d7a84eea04a11 Mon Sep 17 00:00:00 2001 From: Derek Ho Date: Fri, 29 Dec 2023 11:21:53 -0500 Subject: [PATCH 8/8] Change variable name Signed-off-by: Derek Ho --- scripts/integtest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/integtest.sh b/scripts/integtest.sh index b64c0e79..464108c7 100644 --- a/scripts/integtest.sh +++ b/scripts/integtest.sh @@ -70,7 +70,7 @@ then SECURITY_ENABLED="true" fi -IFS='.' read -ra version_array <<< "$VERSION" +IFS='.' read -ra version_array <<< "$OPENSEARCH_VERSION" if [ -z "$CREDENTIAL" ] then