feat: add retina mode support for perf-test and abstract common e2e setups

test/e2e/common/common.go

@@ -8,6 +8,7 @@ import (
 	"flag"
 	"os"
 	"os/user"
+	"path/filepath"
 	"strconv"
 	"testing"
 	"time"
@@ -30,6 +31,22 @@ var (
 	Architectures  = []string{"amd64", "arm64"}
 	CreateInfra    = flag.Bool("create-infra", true, "create a Resource group, vNET and AKS cluster for testing")
 	DeleteInfra    = flag.Bool("delete-infra", true, "delete a Resource group, vNET and AKS cluster for testing")
+
+	// kubeconfig: path to kubeconfig file, in not provided,
+	// a new k8s cluster will be created
+	KubeConfig = flag.String("kubeConfig", "", "Path to kubeconfig file")
+)
+
+var (
+	RetinaChartPath = func(rootDir string) string {
+		return filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina")
+	}
+	RetinaAdvancedProfilePath = func(rootDir string) string {
+		return filepath.Join(rootDir, "test", "profiles", "advanced", "values.yaml")
+	}
+	KubeConfigFilePath = func(rootDir string) string {
+		return filepath.Join(rootDir, "test", "e2e", "test.pem")
+	}
 )
 
 func ClusterNameForE2ETest(t *testing.T) string {

test/e2e/framework/kubernetes/validate-service.go

@@ -62,7 +62,7 @@ func (v *ValidateResource) Run() error {
 	return nil
 }
 
-func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceName, labels string) (bool, error) {
+func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, _, labels string) (bool, error) {
 	var serviceList *corev1.ServiceList
 	serviceList, err := clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labels})
 	if err != nil {

test/e2e/infra/azure_temp_infra_setup.go

@@ -0,0 +1,49 @@
+package infra
+
+import (
+	"context"
+	"crypto/rand"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/microsoft/retina/test/e2e/common"
+	"github.com/microsoft/retina/test/e2e/framework/types"
+	jobs "github.com/microsoft/retina/test/e2e/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+func CreateAzureTempK8sInfra(ctx context.Context, t *testing.T, rootDir string) string {
+	kubeConfigFilePath := common.KubeConfigFilePath(rootDir)
+	clusterName := common.ClusterNameForE2ETest(t)
+
+	subID := os.Getenv("AZURE_SUBSCRIPTION_ID")
+	require.NotEmpty(t, subID, "AZURE_SUBSCRIPTION_ID environment variable must be set")
+
+	location := os.Getenv("AZURE_LOCATION")
+	if location == "" {
+		nBig, err := rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations))))
+		if err != nil {
+			t.Fatal("Failed to generate a secure random index", err)
+		}
+		location = common.AzureLocations[nBig.Int64()]
+	}
+
+	rg := os.Getenv("AZURE_RESOURCE_GROUP")
+	if rg == "" {
+		// Use the cluster name as the resource group name by default.
+		rg = clusterName
+	}
+
+	// CreateTestInfra
+	createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, *common.CreateInfra))
+	createTestInfra.Run(ctx)
+
+	t.Cleanup(func() {
+		err := jobs.DeleteTestInfra(subID, rg, location, *common.DeleteInfra).Run()
+		if err != nil {
+			t.Logf("Failed to delete test infrastructure: %v", err)
+		}
+	})
+	return kubeConfigFilePath
+}

test/e2e/jobs/jobs.go

@@ -1,9 +1,6 @@
 package retina
 
 import (
-	"fmt"
-	"time"
-
 	"github.com/microsoft/retina/test/e2e/common"
 	"github.com/microsoft/retina/test/e2e/framework/azure"
 	"github.com/microsoft/retina/test/e2e/framework/generic"
@@ -13,7 +10,6 @@ import (
 	"github.com/microsoft/retina/test/e2e/scenarios/dns"
 	"github.com/microsoft/retina/test/e2e/scenarios/drop"
 	"github.com/microsoft/retina/test/e2e/scenarios/latency"
-	"github.com/microsoft/retina/test/e2e/scenarios/perf"
 	tcp "github.com/microsoft/retina/test/e2e/scenarios/tcp"
 	"github.com/microsoft/retina/test/e2e/scenarios/windows"
 )
@@ -59,23 +55,19 @@ func CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath string
 		}, nil)
 	}
 
-	job.AddStep(&generic.LoadFlags{
-		TagEnv:            generic.DefaultTagEnv,
-		ImageNamespaceEnv: generic.DefaultImageNamespace,
-		ImageRegistryEnv:  generic.DefaultImageRegistry,
-	}, nil)
-
 	return job
 }
 
-func DeleteTestInfra(subID, rg, clusterName, location string) *types.Job {
+func DeleteTestInfra(subID, rg, location string, deleteInfra bool) *types.Job {
 	job := types.NewJob("Delete e2e test infrastructure")
 
-	job.AddStep(&azure.DeleteResourceGroup{
-		SubscriptionID:    subID,
-		ResourceGroupName: rg,
-		Location:          location,
-	}, nil)
+	if deleteInfra {
+		job.AddStep(&azure.DeleteResourceGroup{
+			SubscriptionID:    subID,
+			ResourceGroupName: rg,
+			Location:          location,
+		}, nil)
+	}
 
 	return job
 }
@@ -274,50 +266,14 @@ func ValidateHubble(kubeConfigFilePath, chartPath string, testPodNamespace strin
 	return job
 }
 
-func RunPerfTest(kubeConfigFilePath string, chartPath string) *types.Job {
-	job := types.NewJob("Run performance tests")
-
-	benchmarkFile := fmt.Sprintf("netperf-benchmark-%s.json", time.Now().Format("20060102150405"))
-	resultFile := fmt.Sprintf("netperf-result-%s.json", time.Now().Format("20060102150405"))
-	regressionFile := fmt.Sprintf("netperf-regression-%s.json", time.Now().Format("20060102150405"))
+func LoadGenericFlags() *types.Job {
+	job := types.NewJob("Loading Generic Flags to env")
 
-	job.AddStep(&perf.GetNetworkPerformanceMeasures{
-		KubeConfigFilePath: kubeConfigFilePath,
-		ResultTag:          "no-retina",
-		JsonOutputFile:     benchmarkFile,
-	}, &types.StepOptions{
-		SkipSavingParametersToJob: true,
-	})
-
-	job.AddStep(&kubernetes.InstallHelmChart{
-		Namespace:          "kube-system",
-		ReleaseName:        "retina",
-		KubeConfigFilePath: kubeConfigFilePath,
-		ChartPath:          chartPath,
-		TagEnv:             generic.DefaultTagEnv,
+	job.AddStep(&generic.LoadFlags{
+		TagEnv:            generic.DefaultTagEnv,
+		ImageNamespaceEnv: generic.DefaultImageNamespace,
+		ImageRegistryEnv:  generic.DefaultImageRegistry,
 	}, nil)
 
-	job.AddStep(&perf.GetNetworkPerformanceMeasures{
-		KubeConfigFilePath: kubeConfigFilePath,
-		ResultTag:          "retina",
-		JsonOutputFile:     resultFile,
-	}, &types.StepOptions{
-		SkipSavingParametersToJob: true,
-	})
-
-	job.AddStep(&perf.GetNetworkRegressionResults{
-		BaseResultsFile:       benchmarkFile,
-		NewResultsFile:        resultFile,
-		RegressionResultsFile: regressionFile,
-	}, &types.StepOptions{
-		SkipSavingParametersToJob: true,
-	})
-
-	job.AddStep(&perf.PublishPerfResults{
-		ResultsFile: regressionFile,
-	}, &types.StepOptions{
-		SkipSavingParametersToJob: true,
-	})
-
 	return job
 }

test/e2e/jobs/perf.go

@@ -0,0 +1,73 @@
+package retina
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/microsoft/retina/test/e2e/framework/generic"
+	"github.com/microsoft/retina/test/e2e/framework/kubernetes"
+	"github.com/microsoft/retina/test/e2e/framework/types"
+	"github.com/microsoft/retina/test/e2e/scenarios/perf"
+)
+
+func RunPerfTest(kubeConfigFilePath, chartPath, advancedValuePath, retinaMode string) *types.Job {
+	job := types.NewJob("Run performance tests")
+
+	benchmarkFile := fmt.Sprintf("netperf-benchmark-%s.json", time.Now().Format("20060102150405"))
+	resultFile := fmt.Sprintf("netperf-result-%s.json", time.Now().Format("20060102150405"))
+	regressionFile := fmt.Sprintf("netperf-regression-%s.json", time.Now().Format("20060102150405"))
+
+	job.AddStep(&perf.GetNetworkPerformanceMeasures{
+		KubeConfigFilePath: kubeConfigFilePath,
+		ResultTag:          "no-retina",
+		JsonOutputFile:     benchmarkFile,
+	}, &types.StepOptions{
+		SkipSavingParametersToJob: true,
+	})
+
+	job.AddStep(&kubernetes.InstallHelmChart{
+		Namespace:          "kube-system",
+		ReleaseName:        "retina",
+		KubeConfigFilePath: kubeConfigFilePath,
+		ChartPath:          chartPath,
+		TagEnv:             generic.DefaultTagEnv,
+	}, nil)
+
+	if retinaMode == "advanced" {
+		job.AddStep(&kubernetes.UpgradeRetinaHelmChart{
+			Namespace:          "kube-system",
+			ReleaseName:        "retina",
+			KubeConfigFilePath: kubeConfigFilePath,
+			ChartPath:          chartPath,
+			ValuesFile:         advancedValuePath,
+			TagEnv:             generic.DefaultTagEnv,
+		}, &types.StepOptions{
+			SkipSavingParametersToJob: true,
+		})
+	}
+
+	job.AddStep(&perf.GetNetworkPerformanceMeasures{
+		KubeConfigFilePath: kubeConfigFilePath,
+		ResultTag:          "retina",
+		JsonOutputFile:     resultFile,
+	}, &types.StepOptions{
+		SkipSavingParametersToJob: true,
+	})
+
+	job.AddStep(&perf.GetNetworkRegressionResults{
+		BaseResultsFile:       benchmarkFile,
+		NewResultsFile:        resultFile,
+		RegressionResultsFile: regressionFile,
+	}, &types.StepOptions{
+		SkipSavingParametersToJob: true,
+	})
+
+	job.AddStep(&perf.PublishPerfResults{
+		ResultsFile: regressionFile,
+		RetinaMode:  retinaMode,
+	}, &types.StepOptions{
+		SkipSavingParametersToJob: true,
+	})
+
+	return job
+}

test/e2e/retina_e2e_test.go

@@ -3,15 +3,14 @@
 package retina
 
 import (
-	"crypto/rand"
-	"math/big"
 	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/microsoft/retina/test/e2e/common"
 	"github.com/microsoft/retina/test/e2e/framework/helpers"
 	"github.com/microsoft/retina/test/e2e/framework/types"
+	"github.com/microsoft/retina/test/e2e/infra"
 	jobs "github.com/microsoft/retina/test/e2e/jobs"
 	"github.com/stretchr/testify/require"
 )
@@ -21,57 +20,46 @@ func TestE2ERetina(t *testing.T) {
 	ctx, cancel := helpers.Context(t)
 	defer cancel()
 
-	// Truncate the username to 8 characters
-	clusterName := common.ClusterNameForE2ETest(t)
-
-	subID := os.Getenv("AZURE_SUBSCRIPTION_ID")
-	require.NotEmpty(t, subID)
-
-	location := os.Getenv("AZURE_LOCATION")
-	if location == "" {
-		nBig, err := rand.Int(rand.Reader, big.NewInt(int64(len(common.AzureLocations))))
-		if err != nil {
-			t.Fatalf("Failed to generate a secure random index: %v", err)
-		}
-		location = common.AzureLocations[nBig.Int64()]
-	}
-
-	rg := os.Getenv("AZURE_RESOURCE_GROUP")
-	if rg == "" {
-		// Use the cluster name as the resource group name by default.
-		rg = clusterName
-	}
-
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
 	// Get to root of the repo by going up two directories
 	rootDir := filepath.Dir(filepath.Dir(cwd))
 
-	chartPath := filepath.Join(rootDir, "deploy", "legacy", "manifests", "controller", "helm", "retina")
 	hubblechartPath := filepath.Join(rootDir, "deploy", "hubble", "manifests", "controller", "helm", "retina")
-	profilePath := filepath.Join(rootDir, "test", "profiles", "advanced", "values.yaml")
-	kubeConfigFilePath := filepath.Join(rootDir, "test", "e2e", "test.pem")
 
-	// CreateTestInfra
-	createTestInfra := types.NewRunner(t, jobs.CreateTestInfra(subID, rg, clusterName, location, kubeConfigFilePath, *common.CreateInfra))
-	createTestInfra.Run(ctx)
+	err = jobs.LoadGenericFlags().Run()
+	require.NoError(t, err, "failed to load generic flags")
 
-	t.Cleanup(func() {
-		if *common.DeleteInfra {
-			_ = jobs.DeleteTestInfra(subID, rg, clusterName, location).Run()
-		}
-	})
+	if *common.KubeConfig == "" {
+		*common.KubeConfig = infra.CreateAzureTempK8sInfra(ctx, t, rootDir)
+	}
 
 	// Install and test Retina basic metrics
-	basicMetricsE2E := types.NewRunner(t, jobs.InstallAndTestRetinaBasicMetrics(kubeConfigFilePath, chartPath, common.TestPodNamespace))
+	basicMetricsE2E := types.NewRunner(t,
+		jobs.InstallAndTestRetinaBasicMetrics(
+			common.KubeConfigFilePath(rootDir),
+			common.RetinaChartPath(rootDir),
+			common.TestPodNamespace),
+	)
 	basicMetricsE2E.Run(ctx)
 
 	// Upgrade and test Retina with advanced metrics
-	advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath, common.TestPodNamespace))
+	advanceMetricsE2E := types.NewRunner(t,
+		jobs.UpgradeAndTestRetinaAdvancedMetrics(
+			common.KubeConfigFilePath(rootDir),
+			common.RetinaChartPath(rootDir),
+			common.RetinaAdvancedProfilePath(rootDir),
+			common.TestPodNamespace),
+	)
 	advanceMetricsE2E.Run(ctx)
 
 	// Install and test Hubble basic metrics
-	validatehubble := types.NewRunner(t, jobs.ValidateHubble(kubeConfigFilePath, hubblechartPath, common.TestPodNamespace))
+	validatehubble := types.NewRunner(t,
+		jobs.ValidateHubble(
+			common.KubeConfigFilePath(rootDir),
+			hubblechartPath,
+			common.TestPodNamespace),
+	)
 	validatehubble.Run(ctx)
 }