chore: cleanup + refactor for test infra migration

.github/workflows/check-coverage.yml

@@ -6,13 +6,6 @@ permissions:
   id-token: write
   contents: read
 
-env:
-  SUBSCRIPTION_ID: "8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8"
-  RESOURCE_GROUP_NAME: "agentbaker-e2e-tests"
-  LOCATION: "eastus"
-  CLUSTER_NAME: "agentbaker-e2e-test-cluster"
-  AZURE_TENANT_ID: "72f988bf-86f1-41af-91ab-2d7cd011db47"
-
 jobs:
   unit_tests:
     runs-on: ubuntu-latest

.pipelines/.vsts-vhd-builder-release.yaml

@@ -1308,7 +1308,7 @@ stages:
               useOverrides: ${{ parameters.useOverrides }}
               overrideBranch: ${{ parameters.overrideBranch }}
               artifactName: 2204-minimal-gen2-containerd
-  - stage: Run_E2E_Tests
+  - stage: e2e
     condition: and(ne(variables.SKIP_E2E_TESTS, 'true'), eq('${{ parameters.dryrun }}', false))
     variables:
       VHD_BUILD_ID: $(Build.BuildId)

.pipelines/.vsts-vhd-builder.yaml

@@ -1,4 +1,4 @@
-name: $(Date:yyyyMMdd)$(Rev:.r)_$(OS_SKU)$(OS_VERSION)_$(HYPERV_GENERATION)_$(FEATURE_FLAGS)_$(Build.SourceBranchName)
+name: $(Date:yyyyMMdd)$(Rev:.r)_$(Build.SourceBranchName)_$(BuildID)
 trigger: none
 pr:
   branches:

.pipelines/templates/e2e-template.yaml

@@ -1,7 +1,7 @@
 jobs:
-  - job: E2E_Tests
+  - job: e2e
     pool:
-      name: nodesigtest-pool
+      name: $(POOL_NAME)
     timeoutInMinutes: 90
     displayName: Run AgentBaker E2E
     variables:
@@ -10,7 +10,7 @@ jobs:
       - bash: |
           set -ex
           az login --identity
-          az account set -s $(AZURE_SUBSCRIPTION_ID)
+          az account set -s $(SUBSCRIPTION_ID)
         displayName: Azure login
       - bash: bash .pipelines/scripts/setup_go.sh
         displayName: Setup go
@@ -32,7 +32,7 @@ jobs:
             export SIG_VERSION_TAG_VALUE=$VHD_BUILD_ID
             export IGNORE_SCENARIOS_WITH_MISSING_VHD=true
           else
-            echo "Build.BuildId is not specified. Running default e2e tests."
+            echo "VHD_BUILD_ID is not specified. Running tests with default SIG version tag selectors."
           fi
 
           cd e2e

e2e/config/azure.go

@@ -57,16 +57,16 @@ type AzureClient struct {
 	VirutalNetworkLinksClient *armprivatedns.VirtualNetworkLinksClient
 }
 
-func mustNewAzureClient(subscription string) *AzureClient {
-	client, err := NewAzureClient(subscription)
+func mustNewAzureClient(subscription, gallerySubscription string) *AzureClient {
+	client, err := NewAzureClient(subscription, gallerySubscription)
 	if err != nil {
 		panic(err)
 	}
 	return client
 
 }
 
-func NewAzureClient(subscription string) (*AzureClient, error) {
+func NewAzureClient(subscription, gallerySubscription string) (*AzureClient, error) {
 	httpClient := &http.Client{
 		// use a bunch of connections for load balancing
 		// ensure all timeouts are defined and reasonable
@@ -208,7 +208,7 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 		return nil, fmt.Errorf("create vnet client: %w", err)
 	}
 
-	cloud.GalleryImageVersion, err = armcompute.NewGalleryImageVersionsClient(subscription, credential, opts)
+	cloud.GalleryImageVersion, err = armcompute.NewGalleryImageVersionsClient(gallerySubscription, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create a new images client: %v", err)
 	}

e2e/config/config.go

@@ -1,6 +1,8 @@
 package config
 
 import (
+	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
@@ -10,7 +12,7 @@ import (
 
 var (
 	Config            = mustLoadConfig()
-	Azure             = mustNewAzureClient(Config.SubscriptionID)
+	Azure             = mustNewAzureClient(Config.SubscriptionID, Config.GallerySubscriptionID)
 	ResourceGroupName = "abe2e-" + Config.Location
 	VMIdentityName    = "abe2e-vm-identity"
 	PrivateACRName    = "privateacre2e"
@@ -21,23 +23,26 @@ var (
 )
 
 type Configuration struct {
-	AirgapNSGName                 string        `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup"`
-	DefaultSubnetName             string        `env:"DEFAULT_SUBNET_NAME" envDefault:"aks-subnet"`
-	BuildID                       string        `env:"BUILD_ID" envDefault:"local"`
-	Location                      string        `env:"LOCATION" envDefault:"westus3"`
-	SubscriptionID                string        `env:"SUBSCRIPTION_ID" envDefault:"8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8"`
-	SIGVersionTagName             string        `env:"SIG_VERSION_TAG_NAME" envDefault:"branch"`
-	SIGVersionTagValue            string        `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/master"`
-	TagsToRun                     string        `env:"TAGS_TO_RUN"`
-	TagsToSkip                    string        `env:"TAGS_TO_SKIP"`
-	TestTimeout                   time.Duration `env:"TEST_TIMEOUT" envDefault:"35m"`
-	E2ELoggingDir                 string        `env:"LOGGING_DIR" envDefault:"scenario-logs"`
-	IgnoreScenariosWithMissingVHD bool          `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
-	SkipTestsWithSKUCapacityIssue bool          `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`
-	KeepVMSS                      bool          `env:"KEEP_VMSS"`
-	BlobStorageAccountPrefix      string        `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e"`
-	BlobContainer                 string        `env:"BLOB_CONTAINER" envDefault:"abe2e"`
-	EnableAKSNodeControllerTest   bool          `env:"ENABLE_AKS_NODE_CONTROLLER_TEST"`
+	AirgapNSGName                 string        `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup" json:"airgapNSGName"`
+	DefaultSubnetName             string        `env:"DEFAULT_SUBNET_NAME" envDefault:"aks-subnet" json:"defaultSubnetName"`
+	BuildID                       string        `env:"BUILD_ID" envDefault:"local" json:"buildID"`
+	Location                      string        `env:"LOCATION" envDefault:"westus3" json:"location"`
+	SubscriptionID                string        `env:"SUBSCRIPTION_ID" envDefault:"8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8" json:"subscriptionID"`
+	GallerySubscriptionID         string        `env:"SUBSCRIPTION_ID" envDefault:"c4c3550e-a965-4993-a50c-628fd38cd3e1" json:"gallerySubscriptionID"`
+	GalleryResourceGroupName      string        `env:"GALLERY_RESOURCE_GROUP_NAME" envDefault:"aksvhdtestbuildrg" json:"galleryResourceGroupName"`
+	GalleryName                   string        `env:"GALLERY_NAME" envDefault:"PackerSigGalleryEastUS" json:"galleryName"`
+	SIGVersionTagName             string        `env:"SIG_VERSION_TAG_NAME" envDefault:"branch" json:"sigVersionTagName"`
+	SIGVersionTagValue            string        `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/master" json:"sigVersionTagValue"`
+	TagsToRun                     string        `env:"TAGS_TO_RUN" json:"tagsToRun"`
+	TagsToSkip                    string        `env:"TAGS_TO_SKIP" json:"tagsToSkip"`
+	TestTimeout                   time.Duration `env:"TEST_TIMEOUT" envDefault:"35m" json:"testTimeout"`
+	E2ELoggingDir                 string        `env:"LOGGING_DIR" envDefault:"scenario-logs" json:"e2eLoggingDir"`
+	IgnoreScenariosWithMissingVHD bool          `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD" json:"ignoreScenariosWithMissingVHD"`
+	SkipTestsWithSKUCapacityIssue bool          `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE" json:"skipTestsWithSKUCapacityIssue"`
+	KeepVMSS                      bool          `env:"KEEP_VMSS" json:"keepVMSS"`
+	BlobStorageAccountPrefix      string        `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e" json:"blobStorageAccountPrefix"`
+	BlobContainer                 string        `env:"BLOB_CONTAINER" envDefault:"abe2e" json:"blobContainer"`
+	EnableAKSNodeControllerTest   bool          `env:"ENABLE_AKS_NODE_CONTROLLER_TEST" json:"enableAKSNodeControllerTest"`
 }
 
 func (c *Configuration) BlobStorageAccount() string {
@@ -48,6 +53,18 @@ func (c *Configuration) BlobStorageAccountURL() string {
 	return "https://" + c.BlobStorageAccount() + ".blob.core.windows.net"
 }
 
+func (c *Configuration) E2EGalleryResourceID() string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s", c.GallerySubscriptionID, c.GalleryResourceGroupName, c.GalleryName)
+}
+
+func (c Configuration) String() string {
+	content, err := json.MarshalIndent(c, "", "	")
+	if err != nil {
+		panic(err)
+	}
+	return string(content)
+}
+
 func mustLoadConfig() Configuration {
 	_ = godotenv.Load(".env")
 	cfg := Configuration{}

e2e/config/vhd.go

@@ -14,7 +14,6 @@ import (
 )
 
 const (
-	imageGallery       = "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/"
 	noSelectionTagName = "abe2e-ignore"
 )
 
@@ -64,6 +63,7 @@ var (
 	// this is a particular 2204gen2containerd image originally built with private packages,
 	// if we ever want to update this then we'd need to run a new VHD build using private package overrides
 	VHDUbuntu2204Gen2ContainerdPrivateKubePkg = &Image{
+		// 2204Gen2 is a special image definition holding historical VHDs used by agentbaker e2e's.
 		Name:    "2204Gen2",
 		OS:      "ubuntu",
 		Arch:    "amd64",
@@ -73,7 +73,7 @@ var (
 
 	// without kubelet, kubectl, credential-provider and wasm
 	VHDUbuntu2204Gen2ContainerdAirgapped = &Image{
-		Name:    "2204gen2containerd",
+		Name:    "2204Gen2",
 		OS:      "ubuntu",
 		Arch:    "amd64",
 		Version: "1.1725612526.29638",
@@ -102,7 +102,7 @@ func (i *Image) String() string {
 
 func (i *Image) VHDResourceID(ctx context.Context, t *testing.T) (VHDResourceID, error) {
 	i.vhdOnce.Do(func() {
-		imageDefinitionResourceID := imageGallery + i.Name
+		imageDefinitionResourceID := fmt.Sprintf("%s/images/%s", Config.E2EGalleryResourceID(), i.Name)
 		if i.Version != "" {
 			i.vhd, i.vhdErr = ensureStaticSIGImageVersion(ctx, t, imageDefinitionResourceID+"/versions/"+i.Version)
 		} else {

e2e/scenario_test.go

@@ -16,6 +16,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
+	fmt.Printf("using E2E environment configuration:\n%s\n", config.Config)
 	// delete scenario-logs folder if it exists
 	if _, err := os.Stat("scenario-logs"); err == nil {
 		_ = os.RemoveAll("scenario-logs")

vhdbuilder/packer/init-variables.sh

@@ -81,7 +81,7 @@ fi
 
 if [ -z "${VNET_RG_NAME}" ]; then
 	if [ "$MODE" == "linuxVhdMode" ]; then
-		VNET_RG_NAME="nodesig-${ENVIRONMENT}-${PACKER_BUILD_LOCATION}-agent-pool"
+		VNET_RG_NAME="nodesig-${ENVIRONMENT}-${PACKER_BUILD_LOCATION}-pool-vnet-rg"
 	fi
 	if [ "$MODE" == "windowsVhdMode" ]; then
 		if [[ "${POOL_NAME}" == *nodesigprod* ]]; then

vhdbuilder/packer/test-scan-and-cleanup.sh

@@ -10,6 +10,7 @@ required_env_vars=(
   "SIG_IMAGE_NAME"
   "UMSI_RESOURCE_ID"
   "UMSI_PRINCIPAL_ID"
+  "AZURE_MSI_RESOURCE_STRING"
   "UMSI_CLIENT_ID"
   "BUILD_RUN_NUMBER"
 )

vhdbuilder/packer/trivy-scan.sh

@@ -31,13 +31,14 @@ SEVERITY=${17}
 MODULE_VERSION=${18}
 UMSI_PRINCIPAL_ID=${19}
 UMSI_CLIENT_ID=${20}
-BUILD_RUN_NUMBER=${21}
-export BUILD_REPOSITORY_NAME=${22}
-export BUILD_SOURCEBRANCH=${23}
-export BUILD_SOURCEVERSION=${24}
-export SYSTEM_COLLECTIONURI=${25}
-export SYSTEM_TEAMPROJECT=${26}
-export BUILD_BUILDID=${27}
+AZURE_MSI_RESOURCE_STRING=${21}
+BUILD_RUN_NUMBER=${22}
+export BUILD_REPOSITORY_NAME=${23}
+export BUILD_SOURCEBRANCH=${24}
+export BUILD_SOURCEVERSION=${25}
+export SYSTEM_COLLECTIONURI=${26}
+export SYSTEM_TEAMPROJECT=${27}
+export BUILD_BUILDID=${28}
 
 retrycmd_if_failure() {
     retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
@@ -93,13 +94,21 @@ install_azure_cli() {
     fi
 }
 
+login_with_user_assigned_managed_identity() {
+    local USERNAME=$1
+
+    LOGIN_FLAGS="--identity --username $USERNAME"
+    if [ "${ENABLE_TRUSTED_LAUNCH,,}" == "true" ]; then
+        LOGIN_FLAGS="$LOGIN_FLAGS --allow-no-subscriptions"
+    fi
+
+   echo "logging into azure with flags: $LOGIN_FLAGS"
+   az login $LOGIN_FLAGS
+}
+
 install_azure_cli $OS_SKU $OS_VERSION $ARCHITECTURE $TEST_VM_ADMIN_USERNAME
 
-if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then
-    az login --identity --allow-no-subscriptions --username ${UMSI_PRINCIPAL_ID}
-else
-    az login --identity
-fi
+login_with_user_assigned_managed_identity ${UMSI_PRINCIPAL_ID}
 
 arch="$(uname -m)"
 if [ "${arch,,}" == "arm64" ] || [ "${arch,,}" == "aarch64" ]; then
@@ -182,6 +191,8 @@ rm ./trivy
 chmod a+r "${TRIVY_REPORT_ROOTFS_JSON_PATH}"
 chmod a+r "${TRIVY_REPORT_IMAGE_TABLE_PATH}"
 
+login_with_user_assigned_managed_identity ${AZURE_MSI_RESOURCE_STRING}
+
 az storage blob upload --file ${TRIVY_REPORT_ROOTFS_JSON_PATH} \
     --container-name ${SIG_CONTAINER_NAME} \
     --name ${TRIVY_UPLOAD_REPORT_NAME} \

vhdbuilder/packer/vhd-scanning.sh

@@ -3,8 +3,15 @@ set -eux
 
 source ./parts/linux/cloud-init/artifacts/cse_benchmark_functions.sh
 
+# This variable is used to determine where we need to deploy the VM on which we'll run trivy.
+# We must be sure this location matches the location used by packer when delivering the output image
+# version to the staging gallery, as the particular image version will only have a single replica in this region.
+if [ -z "$PACKER_BUILD_LOCATION" ]; then
+    echo "PACKER_BUILD_LOCATION must be set to run VHD scanning"
+    exit 1
+fi
+
 TRIVY_SCRIPT_PATH="trivy-scan.sh"
-EXE_SCRIPT_PATH="vhd-scanning-exe-on-vm.sh"
 SCAN_RESOURCE_PREFIX="vhd-scanning"
 SCAN_VM_NAME="$SCAN_RESOURCE_PREFIX-vm-$(date +%s)-$RANDOM"
 VHD_IMAGE="$MANAGED_SIG_ID"
@@ -16,14 +23,6 @@ SCAN_VM_ADMIN_USERNAME="azureuser"
 VNET_NAME="nodesig-pool-vnet-${PACKER_BUILD_LOCATION}"
 SUBNET_NAME="scanning"
 
-# This variable is used to determine where we need to deploy the VM on which we'll run trivy.
-# We must be sure this location matches the location used by packer when delivering the output image
-# version to the staging gallery, as the particular image version will only have a single replica in this region.
-if [ -z "$PACKER_BUILD_LOCATION" ]; then
-    echo "PACKER_BUILD_LOCATION must be set to run VHD scanning"
-    exit 1
-fi
-
 # Use the domain name from the classic blob URL to get the storage account name.
 # If the CLASSIC_BLOB var is not set create a new var called BLOB_STORAGE_NAME in the pipeline.
 BLOB_URL_REGEX="^https:\/\/.+\.blob\.core\.windows\.net\/vhd(s)?$"
@@ -73,6 +72,10 @@ az vm create --resource-group $RESOURCE_GROUP_NAME \
     --assign-identity "${UMSI_RESOURCE_ID}"
 
 capture_benchmark "${SCRIPT_NAME}_create_scan_vm"
+set +x
+
+# for scanning storage account/container upload access
+az vm identity assign -g $RESOURCE_GROUP_NAME --name $SCAN_VM_NAME --identities $AZURE_MSI_RESOURCE_STRING
 
 FULL_PATH=$(realpath $0)
 CDIR=$(dirname $FULL_PATH)
@@ -109,6 +112,7 @@ az vm run-command invoke \
         "MODULE_VERSION"=${MODULE_VERSION} \
         "UMSI_PRINCIPAL_ID"=${UMSI_PRINCIPAL_ID} \
         "UMSI_CLIENT_ID"=${UMSI_CLIENT_ID} \
+        "AZURE_MSI_RESOURCE_STRING"=${AZURE_MSI_RESOURCE_STRING} \
         "BUILD_RUN_NUMBER"=${BUILD_RUN_NUMBER} \
         "BUILD_REPOSITORY_NAME"=${BUILD_REPOSITORY_NAME} \
         "BUILD_SOURCEBRANCH"=${GIT_BRANCH} \