From 4079e1c2571745a4e85ba4d9a04d321ee33979e6 Mon Sep 17 00:00:00 2001 From: Blaise Dias Date: Wed, 30 Oct 2024 12:41:04 +0000 Subject: [PATCH] chore: rationalise configuration file loading - common up code for loading base, platform and product configiration files - load configuration files preferentially from e2e_root_dir over openebs_e2e_root_dir - this makes overriding base configuration simple Signed-off-by: Blaise Dias --- common/e2e_config/e2e_config.go | 110 ++++++++++++++++-------------- configurations/selfci_config.yaml | 85 ----------------------- 2 files changed, 58 insertions(+), 137 deletions(-) delete mode 100644 configurations/selfci_config.yaml diff --git a/common/e2e_config/e2e_config.go b/common/e2e_config/e2e_config.go index d33fe8b..0359748 100644 --- a/common/e2e_config/e2e_config.go +++ b/common/e2e_config/e2e_config.go @@ -13,9 +13,12 @@ import ( // const ConfigDir = "../../configurations" // const PlatformConfigDir = "../../configurations/platforms/" -var ( - ConfigDir = "/configurations" - PlatformConfigDir = "/configurations/platforms/" +type configurationType int + +const ( + baseCfg configurationType = iota + productCfg + platformCfg ) type ConfigurationContext int @@ -244,20 +247,45 @@ func SetContext(ctx ConfigurationContext) { configContext = ctx } -// This function is called early from junit and various bits have not been initialised yet -// so we cannot use logf or Expect instead we use fmt.Print... and panic. +// getConfigFilePath given a configuration filename search for its existence +// in the expected locations +// - current directory +// - e2e_root_dir (if defined in environment) +// - openebs_e2e_root_dir (if defined in environment) +// +// Returns the fully qualified path to the file if found. +// Note: if the config filename is a fully qualified path and the file exists +// then that path will be returned +func getConfigFilePath(filename string, cfgType configurationType) (string, error) { + var subPath = map[configurationType]string{ + baseCfg: "/configurations", + productCfg: "/configurations/product", + platformCfg: "/configurations/platforms", + } + + subDir := subPath[cfgType] + configFile := path.Clean(filename) + e2eRootDir, haveE2ERootDir := os.LookupEnv("e2e_root_dir") + openebsE2eRootDir, haveOpenebsE2eRootDir := os.LookupEnv("openebs_e2e_root_dir") + info, err := os.Stat(configFile) + if os.IsNotExist(err) && haveE2ERootDir { + configFile = path.Clean(e2eRootDir + subDir + "/" + filename) + info, err = os.Stat(configFile) + } + if os.IsNotExist(err) && haveOpenebsE2eRootDir { + configFile = path.Clean(openebsE2eRootDir + subDir + "/" + filename) + info, err = os.Stat(configFile) + } + if err == nil && info.IsDir() { + err = fmt.Errorf("%v is not a file", configFile) + } + return configFile, err +} + +// GetConfig This function is called early from junit and various bits have not been initialised yet, +// we cannot use logf or Expect instead we use fmt.Print... and panic. func GetConfig() E2EConfig { once.Do(func() { - var err error - var info os.FileInfo - e2eRootDir, haveE2ERootDir := os.LookupEnv("e2e_root_dir") - openebsE2eRootDir, haveOpenebsE2eRootDir := os.LookupEnv("openebs_e2e_root_dir") - if configContext == E2eTesting { - if !haveOpenebsE2eRootDir { - panic("openebs_e2e_root environment variable not set") - } - } - // Initialise the configuration _ = cleanenv.ReadEnv(&e2eConfig) // We absorb the complexity of locating the configuration file here @@ -271,19 +299,11 @@ func GetConfig() E2EConfig { _, _ = fmt.Fprintln(os.Stderr, " Use environment variable \"e2e_config_file\" to specify configuration file.") } } else { - var configFile string = path.Clean(e2eConfig.ConfigPaths.ConfigFile) - info, err = os.Stat(configFile) - if os.IsNotExist(err) { - configFile = path.Clean(openebsE2eRootDir + ConfigDir + "/" + e2eConfig.ConfigPaths.ConfigFile) - info, err = os.Stat(configFile) - if err != nil { - panic(fmt.Sprintf("Unable to access configuration file %v", configFile)) - } - e2eConfig.ConfigPaths.ConfigFile = configFile - } - if info.IsDir() { - panic(fmt.Sprintf("%v is not a file", configFile)) + configFile, err := getConfigFilePath(e2eConfig.ConfigPaths.ConfigFile, baseCfg) + if err != nil { + panic(fmt.Sprintf("Unable to access configuration file %v, %v", e2eConfig.ConfigPaths.ConfigFile, err)) } + e2eConfig.ConfigPaths.ConfigFile = configFile _, _ = fmt.Fprintf(os.Stderr, "Using configuration file %s\n", configFile) err = cleanenv.ReadConfig(configFile, &e2eConfig) if err != nil { @@ -297,44 +317,29 @@ func GetConfig() E2EConfig { _, _ = fmt.Fprintln(os.Stderr, " Use environment variable \"e2e_platform_config_file\" to specify platform configuration.") } } else { - var platformCfg string = path.Clean(e2eConfig.ConfigPaths.PlatformConfigFile) - info, err = os.Stat(platformCfg) - if os.IsNotExist(err) { - platformCfg = path.Clean(openebsE2eRootDir + PlatformConfigDir + "/" + e2eConfig.ConfigPaths.PlatformConfigFile) - info, err = os.Stat(platformCfg) - if err != nil { - panic(fmt.Sprintf("Unable to access platform configuration file %v", platformCfg)) - } - e2eConfig.ConfigPaths.PlatformConfigFile = platformCfg - } - if info.IsDir() { - panic(fmt.Sprintf("%v is not a file", platformCfg)) + platformCfg, err := getConfigFilePath(e2eConfig.ConfigPaths.PlatformConfigFile, platformCfg) + if err != nil { + panic(fmt.Sprintf("Unable to access configuration file %v, %v", e2eConfig.ConfigPaths.PlatformConfigFile, err)) } + e2eConfig.ConfigPaths.PlatformConfigFile = platformCfg _, _ = fmt.Fprintf(os.Stderr, "Using platform configuration file %s\n", platformCfg) err = cleanenv.ReadConfig(platformCfg, &e2eConfig) if err != nil { panic(fmt.Sprintf("%v", err)) } } + if e2eConfig.ConfigPaths.ProductConfigFile == "" { if configContext == E2eTesting { _, _ = fmt.Fprintln(os.Stderr, "Product configuration file not specified, using defaults.") _, _ = fmt.Fprintln(os.Stderr, " Use environment variable \"e2e_product_config_file\" to specify product configuration.") } } else { - var productCfg string = path.Clean(e2eConfig.ConfigPaths.ProductConfigFile) - info, err = os.Stat(productCfg) - if os.IsNotExist(err) { - productCfg = path.Clean(e2eConfig.ConfigPaths.ProductConfigFile) - info, err = os.Stat(productCfg) - if err != nil { - panic(fmt.Sprintf("Unable to access product configuration file %v", productCfg)) - } - e2eConfig.ConfigPaths.ProductConfigFile = productCfg - } - if info.IsDir() { - panic(fmt.Sprintf("%v is not a file", productCfg)) + productCfg, err := getConfigFilePath(e2eConfig.ConfigPaths.ProductConfigFile, productCfg) + if err != nil { + panic(fmt.Sprintf("Unable to access configuration file %v, %v", e2eConfig.ConfigPaths.ProductConfigFile, err)) } + e2eConfig.ConfigPaths.ProductConfigFile = productCfg _, _ = fmt.Fprintf(os.Stderr, "Using product configuration file %s\n", productCfg) err = cleanenv.ReadConfig(productCfg, &e2eConfig) if err != nil { @@ -350,6 +355,7 @@ func GetConfig() E2EConfig { } } + e2eRootDir, haveE2ERootDir := os.LookupEnv("e2e_root_dir") if configContext == E2eTesting { artifactsDir := "" // if e2e root dir was specified record this in the configuration @@ -367,7 +373,7 @@ func GetConfig() E2EConfig { // The session directory is required for install and uninstall tests // create and use the default one. e2eConfig.SessionDir = artifactsDir + "/sessions/default" - err = os.MkdirAll(e2eConfig.SessionDir, os.ModeDir|os.ModePerm) + err := os.MkdirAll(e2eConfig.SessionDir, os.ModeDir|os.ModePerm) if err != nil { panic(err) } diff --git a/configurations/selfci_config.yaml b/configurations/selfci_config.yaml deleted file mode 100644 index 26157c4..0000000 --- a/configurations/selfci_config.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# Defining feature is that long running tests have attenuated durations -configName: selfci -grpcMandated: true -selfTest: true -failQuick: true -beforeEachCheckAndRestart: true -ioEngineNvmeTimeout: 110 -pvcStress: - replicas: 1 - cdCycles: 1 - crudCycles: 1 -ioSoakTest: - replicas: 1 - duration: 3m - # loadFactor is number of volumes for each mayastor instance - # volumes for disruptor pods are allocated from within this "pool" - loadFactor: 3 - protocols: - - nvmf - # fioStartDelay units are seconds - fioStartDelay: 90 - readyTimeout: 180s - disrupt: - # Number of disruptor pods - podCount: 3 - # faultAfter units are seconds - faultAfter: 51 - readyTimeout: 60s - # thinkTime units are microseconds - fioDutyCycles: - - thinkTime: 500000 - thinkTimeBlocks: 1000 - - thinkTime: 750000 - thinkTimeBlocks: 1000 - - thinkTime: 1250000 - thinkTimeBlocks: 2000 - - thinkTime: 1500000 - thinkTimeBlocks: 3000 - - thinkTime: 1750000 - thinkTimeBlocks: 3000 - - thinkTime: 2000000 - thinkTimeBlocks: 4000 -csi: - replicas: 1 - smallClaimSize: 50Mi - largeClaimSize: 500Mi -uninstall: - cleanup: 0 -basicVolumeIO: - replicas: 1 - # timeout units are seconds - fioLoopTimeout: 30 - # volSizeMb units are MiB - volSizeMb: 312 - # fsVolSizeMb units are MiB - fsVolSizeMb: 50 -multiVolumesPodIO: - replicas: 1 - volumeSizeMb: 312 -pvcReadWriteOnce: - # timeout units are seconds - fioTimeout: 20 -pvcDelete: - # volSizeMb units are MiB - volSizeMb: 312 - # fsVolSizeMb units are MiB - fsVolSizeMb: 312 -primitiveReplicas: - iterations: 2 - startSizeMb: 312 - endSizeMb: 512 - sizeStepMb: 64 -primitiveMaxVolsInPool: - volMb: 64 - volumeCountPerPool: 110 - replicas: 1 -primitiveMspState: - iterationCount: 10 -primitiveMspDelete: - iterations: 10 -PrimitiveMspStressTest: - iterations: 2 -maximumVolsIO: - volumeCountPerPod: 10 - podCount: 11