Merge branch 'master' into adam--podmonitor-vpc-cni

Makefile

@@ -279,7 +279,7 @@ docker-metrics-test:     ## Run metrics helper unit test suite in a container.
 		make metrics-unit-test
 
 # Fetch the CNI plugins
-plugins: FETCH_VERSION=1.4.0
+plugins: FETCH_VERSION=1.5.1
 plugins: FETCH_URL=https://github.com/containernetworking/plugins/releases/download/v$(FETCH_VERSION)/cni-plugins-$(GOOS)-$(GOARCH)-v$(FETCH_VERSION).tgz
 plugins: VISIT_URL=https://github.com/containernetworking/plugins/tree/v$(FETCH_VERSION)/plugins/
 plugins:   ## Fetch the CNI plugins

README.md

@@ -68,9 +68,9 @@ For help, please consider the following venues (in order):
 For all Kubernetes releases, *we recommend installing the latest VPC CNI release*. The following table denotes our *oldest* recommended
 VPC CNI version for each actively supported Kubernetes release.
 
-| Kubernetes Release | 1.29     | 1.28     | 1.27     | 1.26     | 1.25     | 1.24    |
-| ------------------ | -------- | -------- | -------- | -------- | -------- | ------- |
-| VPC CNI Version    | v1.14.1+ | v1.13.4+ | v1.12.5+ | v1.12.0+ | v1.11.4+ | v1.9.3+ |
+| Kubernetes Release | 1.31     | 1.30     | 1.29     | 1.28     | 1.27     | 1.26     | 1.25     | 1.24    |
+| ------------------ | -------- | -------- | -------- | -------- | -------- | -------- | -------- | ------- |
+| VPC CNI Version    | v1.16.4+ | v1.16.0+ | v1.14.1+ | v1.13.4+ | v1.12.5+ | v1.12.0+ | v1.11.4+ | v1.9.3+ |
 
 ## Version Upgrade
 
@@ -516,6 +516,7 @@ Valid Values: `strict`, `standard`
 Once `ENABLE_POD_ENI` is set to `true`, this value controls how the traffic of pods with the security group behaves.
 
 * `strict` mode: all inbound/outbound traffic from pod with security group will be enforced by security group rules. This is the **default** mode if POD_SECURITY_GROUP_ENFORCING_MODE is not set.
+  * `strict` mode is supported when kube-proxy configured in `iptables` mode (default with EKS). If kube-proxy is configured in `ipvs` mode, please set `POD_SECURITY_GROUP_ENFORCING_MODE` to `standard`.
 
 * `standard` mode: the traffic of pod with security group behaves same as pods without a security group, except that each pod occupies a dedicated branch ENI.
   * inbound traffic to pod with security group from another host will be enforced by security group rules.

scripts/lib/canary.sh

@@ -4,8 +4,12 @@
 
 SECONDS=0
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$SCRIPT_DIR"/set_kubeconfig.sh
+
+
 echo "Running tests for amazon-vpc-cni-k8s with the following variables
-KUBE_CONFIG_PATH:  $KUBE_CONFIG_PATH
+KUBECONFIG:  $KUBECONFIG
 CLUSTER_NAME: $CLUSTER_NAME
 REGION: $REGION
 ENDPOINT: $ENDPOINT"

scripts/lib/cluster.sh

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$SCRIPT_DIR"/set_kubeconfig.sh
+
+
 function load_cluster_details() {
   echo "loading cluster details $CLUSTER_NAME"
   DESCRIBE_CLUSTER_OP=$(aws eks describe-cluster --name "$CLUSTER_NAME" --region "$REGION" $ENDPOINT_FLAG)
@@ -10,7 +14,7 @@ function load_cluster_details() {
 function load_deveks_cluster_details() {
 
   echo "loading cluster details $CLUSTER_NAME"
-  PROVIDER_ID=$(kubectl get nodes --kubeconfig $KUBE_CONFIG_PATH -ojson | jq -r '.items[0].spec.providerID')
+  PROVIDER_ID=$(kubectl get nodes --kubeconfig $KUBECONFIG -ojson | jq -r '.items[0].spec.providerID')
   INSTANCE_ID=${PROVIDER_ID##*/}
   VPC_ID=$(aws ec2 describe-instances --instance-ids ${INSTANCE_ID} | jq -r '.Reservations[].Instances[].VpcId')
 }

scripts/lib/set_kubeconfig.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# This script sets the KUBECONFIG environment variable based on KUBE_CONFIG_PATH if not already set.
+
+if [ -n "${KUBE_CONFIG_PATH:-}" ] && [ -z "$KUBECONFIG" ]; then
+  export KUBECONFIG=$KUBE_CONFIG_PATH
+fi

scripts/run-canary-test.sh

@@ -11,15 +11,16 @@ GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
 TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazonaws.com"}
 ADC_REGIONS="us-iso-east-1 us-isob-east-1 us-iso-west-1"
 
+source "$SCRIPT_DIR"/lib/set_kubeconfig.sh
 source "$SCRIPT_DIR"/lib/add-on.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
 source "$SCRIPT_DIR"/lib/canary.sh
 
 function run_ginkgo_test() {
   local focus=$1
   echo "Running ginkgo tests with focus: $focus"
-  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/cni.test -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux" --test-image-registry=$TEST_IMAGE_REGISTRY)
-  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/ipamd.test -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux" --test-image-registry=$TEST_IMAGE_REGISTRY)
+  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/cni.test -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux" --test-image-registry=$TEST_IMAGE_REGISTRY)
+  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/ipamd.test -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux" --test-image-registry=$TEST_IMAGE_REGISTRY)
 }
 
 load_cluster_details

scripts/run-cni-release-tests.sh

@@ -6,7 +6,7 @@
 # CLUSTER_NAME: name of the cluster to run the test
 # VPC_ID: cluster VPC ID
 # REGION: default us-west-2
-# KUBE_CONFIG_PATH: path to the kubeconfig file, default ~/.kube/config
+# KUBECONFIG: path to the kubeconfig file, default ~/.kube/config
 # NG_LABEL_KEY: nodegroup label key, default "kubernetes.io/os"
 # NG_LABEL_VAL: nodegroup label val, default "linux"
 # RUN_DEVEKS_TEST: Set this variable for tests to run on a deveks cluster
@@ -18,6 +18,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 INTEGRATION_TEST_DIR="$SCRIPT_DIR/../test/integration"
 TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazonaws.com"}
 
+source "$SCRIPT_DIR"/lib/set_kubeconfig.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
 source "$SCRIPT_DIR"/lib/integration.sh
 
@@ -28,12 +29,12 @@ function run_integration_test() {
 
   echo "Running ipamd integration tests"
   START=$SECONDS
-  cd $INTEGRATION_TEST_DIR/ipamd && CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --skip-file=ipamd_event_test.go -v -timeout 90m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="$NG_LABEL_KEY" --ng-name-label-val="$NG_LABEL_VAL"  --test-image-registry=$TEST_IMAGE_REGISTRY || TEST_RESULT=fail
+  cd $INTEGRATION_TEST_DIR/ipamd && CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --skip-file=ipamd_event_test.go -v -timeout 90m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="$NG_LABEL_KEY" --ng-name-label-val="$NG_LABEL_VAL"  --test-image-registry=$TEST_IMAGE_REGISTRY || TEST_RESULT=fail
   echo "ipamd test took $((SECONDS - START)) seconds."
 
   echo "Running cni integration tests"
   START=$SECONDS
-  cd $INTEGRATION_TEST_DIR/cni && CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --skip-file=soak_test.go -v -timeout 60m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="$NG_LABEL_KEY" --ng-name-label-val="$NG_LABEL_VAL" --test-image-registry=$TEST_IMAGE_REGISTRY || TEST_RESULT=fail
+  cd $INTEGRATION_TEST_DIR/cni && CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --skip-file=soak_test.go -v -timeout 60m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="$NG_LABEL_KEY" --ng-name-label-val="$NG_LABEL_VAL" --test-image-registry=$TEST_IMAGE_REGISTRY || TEST_RESULT=fail
   echo "cni test took $((SECONDS - START)) seconds."
 
   if [[ ! -z $PROD_IMAGE_REGISTRY ]]; then
@@ -46,7 +47,7 @@ function run_integration_test() {
   TAG=$(echo $CNI_METRICS_HELPER | cut -d ":" -f 2)
   echo "Running cni-metrics-helper image($CNI_METRICS_HELPER) tests"
   START=$SECONDS
-  cd $INTEGRATION_TEST_DIR/metrics-helper && CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS -v -timeout 15m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="$NG_LABEL_KEY" --ng-name-label-val="$NG_LABEL_VAL" --cni-metrics-helper-image-repo=$REPO_NAME --cni-metrics-helper-image-tag=$TAG --test-image-registry=$TEST_IMAGE_REGISTRY || TEST_RESULT=fail
+  cd $INTEGRATION_TEST_DIR/metrics-helper && CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS -v -timeout 15m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="$NG_LABEL_KEY" --ng-name-label-val="$NG_LABEL_VAL" --cni-metrics-helper-image-repo=$REPO_NAME --cni-metrics-helper-image-tag=$TAG --test-image-registry=$TEST_IMAGE_REGISTRY || TEST_RESULT=fail
   echo "cni-metrics-helper test took $((SECONDS - START)) seconds."
   if [[ "$TEST_RESULT" == fail ]]; then
       echo "Integration test failed."

scripts/run-ginkgo-integration-suite.sh

@@ -11,6 +11,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
 : "${SKIP_MAKE_TEST_BINARIES:=}"
 
+source "$SCRIPT_DIR"/lib/set_kubeconfig.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
 source "$SCRIPT_DIR"/lib/canary.sh
 
@@ -37,7 +38,7 @@ function load_test_parameters(){
 
 function run_ginkgo_test() {
   (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS -v --timeout 60m --no-color --fail-on-pending $GINKGO_TEST_BUILD/$SUITE_NAME.test -- \
-    --cluster-kubeconfig="$KUBE_CONFIG_PATH" \
+    --cluster-kubeconfig="$KUBECONFIG" \
     --cluster-name="$CLUSTER_NAME" \
     --aws-region="$REGION" \
     --aws-vpc-id="$VPC_ID" \

scripts/run-ipv6-canary-test.sh

@@ -7,14 +7,15 @@ set -e
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
 
+source "$SCRIPT_DIR"/lib/set_kubeconfig.sh
 source "$SCRIPT_DIR"/lib/add-on.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
 source "$SCRIPT_DIR"/lib/canary.sh
 
 function run_ginkgo_test() {
   local focus=$1
   echo "Running ginkgo tests with focus: $focus"
-  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --focus="$focus" -v --timeout 15m --no-color --fail-on-pending $GINKGO_TEST_BUILD/ipv6.test -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
+  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --focus="$focus" -v --timeout 15m --no-color --fail-on-pending $GINKGO_TEST_BUILD/ipv6.test -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
 }
 
 load_cluster_details

scripts/run-ipv6-integration-tests.sh

@@ -10,12 +10,13 @@ GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
 # TEST_IMAGE_REGISTRY is the registry in test-infra-* accounts where e2e test images are stored
 TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazonaws.com"}
 
+source "$SCRIPT_DIR"/lib/set_kubeconfig.sh
 source "$SCRIPT_DIR"/lib/add-on.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
 source "$SCRIPT_DIR"/lib/canary.sh
 
 function run_ginkgo_test() {
-  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS -v --timeout 30m --no-color --fail-on-pending $GINKGO_TEST_BUILD/ipv6.test -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux" --test-image-registry=$TEST_IMAGE_REGISTRY)
+  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS -v --timeout 30m --no-color --fail-on-pending $GINKGO_TEST_BUILD/ipv6.test -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux" --test-image-registry=$TEST_IMAGE_REGISTRY)
 }
 
 load_cluster_details

scripts/run-multus-tests.sh

@@ -7,13 +7,14 @@ set -e
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 INTEGRATION_TEST_DIR="$SCRIPT_DIR/../test/integration"
 
+source "$SCRIPT_DIR"/lib/set_kubeconfig.sh
 source "$SCRIPT_DIR"/lib/common.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
 source "$SCRIPT_DIR"/lib/canary.sh
 
 function run_ginkgo_test() {
   local focus=$1
-  (cd "$INTEGRATION_TEST_DIR/multus" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 20m --fail-on-pending -- --cluster-kubeconfig="$KUBE_CONFIG_PATH" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
+  (cd "$INTEGRATION_TEST_DIR/multus" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 20m --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
 }
 
 check_is_installed kubectl

scripts/run-soak-test.sh

@@ -14,6 +14,7 @@ TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazo
 # defined in lib/cluster.sh will add --eks-endpoint=$ENDPOINT to the ginkgo
 # test command
 
+source "$SCRIPT_DIR"/lib/set_kubeconfig.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
 source "$SCRIPT_DIR"/lib/canary.sh
 
@@ -22,7 +23,7 @@ function run_ginkgo_test() {
   echo "Running ginkgo tests with focus: $focus"
 
   (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 3h --fail-on-pending $GINKGO_TEST_BUILD/cni.test -- \
-      --cluster-kubeconfig="$KUBE_CONFIG_PATH" \
+      --cluster-kubeconfig="$KUBECONFIG" \
       --cluster-name="$CLUSTER_NAME" \
       --aws-region="$REGION" \
       --aws-vpc-id="$VPC_ID" \

scripts/run-static-canary.sh

@@ -14,6 +14,7 @@ TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazo
 # defined in lib/cluster.sh will add --eks-endpoint=$ENDPOINT to the ginkgo
 # test command
 
+source "$SCRIPT_DIR"/lib/set_kubeconfig.sh
 source "$SCRIPT_DIR"/lib/cluster.sh
 source "$SCRIPT_DIR"/lib/canary.sh
 
@@ -22,7 +23,7 @@ function run_ginkgo_test() {
   echo "Running ginkgo tests with focus: $focus"
 
   (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 10m --fail-on-pending $GINKGO_TEST_BUILD/az-traffic.test -- \
-      --cluster-kubeconfig="$KUBE_CONFIG_PATH" \
+      --cluster-kubeconfig="$KUBECONFIG" \
       --cluster-name="$CLUSTER_NAME" \
       --aws-region="$REGION" \
       --aws-vpc-id="$VPC_ID" \

test/integration/README.md

@@ -121,10 +121,10 @@ These tests require multus to be deployed to your cluster using the [manifest](h
 You can optionally provide multus tag to install the manifest. If not provided then it will use the default tag
 
 ```
-KUBE_CONFIG_PATH=/Users/cgadgil/.kube/config CLUSTER_NAME=eks-MultusInfra REGION=us-west-2 SKIP_MAKE_TEST_BINARIES=true ./scripts/run-multus-tests.sh v3.7.2-eksbuild.2
+KUBECONFIG=/Users/cgadgil/.kube/config CLUSTER_NAME=eks-MultusInfra REGION=us-west-2 SKIP_MAKE_TEST_BINARIES=true ./scripts/run-multus-tests.sh v3.7.2-eksbuild.2
 
 Running tests with the following variables
-KUBE_CONFIG_PATH:  /Users/cgadgil/.kube/config
+KUBECONFIG:  /Users/cgadgil/.kube/config
 CLUSTER_NAME: eks-MultusInfra
 REGION: us-west-2
 ENDPOINT: