feat: Harvest should load templates from a set of conf directories (#…

…2329)

* feat: Harvest should load templates from a set of conf directories

* feat: address review comments

cmd/collectors/ems/ems_test.go

@@ -43,18 +43,17 @@ func BookendEmsTest(t *testing.T, e *Ems) {
 }
 
 func NewEms() *Ems {
-	// homepath is harvest directory level
+	// homePath is harvest directory level
 	homePath := "../../../"
-	emsConfgPath := homePath + "conf/ems/default.yaml"
-	emsPoller := "testEms"
+	emsConfigPath := homePath + "conf/ems/default.yaml"
 
 	conf.TestLoadHarvestConfig("testdata/config.yml")
-	opts := options.Options{
-		Poller:   emsPoller,
-		HomePath: homePath,
-		IsTest:   true,
-	}
-	ac := collector.New("Ems", "Ems", &opts, emsParams(emsConfgPath), nil)
+	opts := options.New(options.WithConfPath(homePath + "conf"))
+	opts.Poller = "testEms"
+	opts.HomePath = homePath
+	opts.IsTest = true
+
+	ac := collector.New("Ems", "Ems", opts, emsParams(emsConfigPath), nil)
 	e := &Ems{}
 	if err := e.Init(ac); err != nil {
 		log.Fatal().Err(err).Send()

cmd/collectors/rest/rest_test.go

@@ -152,12 +152,11 @@ func volumeEndpoints(e *endPoint) ([]gjson.Result, error) {
 
 func newRest(object string, path string) *Rest {
 	var err error
-	opts := options.Options{
-		Poller:   pollerName,
-		HomePath: "testdata",
-		IsTest:   true,
-	}
-	ac := collector.New("Rest", object, &opts, params(object, path), nil)
+	opts := options.New(options.WithConfPath("testdata/conf"))
+	opts.Poller = pollerName
+	opts.HomePath = "testdata"
+	opts.IsTest = true
+	ac := collector.New("Rest", object, opts, params(object, path), nil)
 	r := Rest{}
 	err = r.Init(ac)
 	if err != nil {

cmd/collectors/rest/templating.go

@@ -13,23 +13,13 @@ import (
 
 func (r *Rest) LoadTemplate() (string, error) {
 
-	var (
-		template     *node.Node
-		templatePath string
-		err          error
-	)
-
-	// import template
-	if template, templatePath, err = r.ImportSubTemplate(
-		"",
-		TemplateFn(r.Params, r.Object),
-		r.Client.Cluster().Version,
-	); err != nil {
+	template, path, err := r.ImportSubTemplate("", TemplateFn(r.Params, r.Object), r.Client.Cluster().Version)
+	if err != nil {
 		return "", err
 	}
 
 	r.Params.Union(template)
-	return templatePath, nil
+	return path, nil
 }
 
 func (r *Rest) InitCache() error {
@@ -94,10 +84,10 @@ func HandleDuration(value string) float64 {
 
 		seconds := 0.0
 
-		//years
-		//months
+		// years
+		// months
 
-		//days
+		// days
 		if matches[3] != "" {
 			f, err := strconv.ParseFloat(matches[3], 64)
 			if err != nil {
@@ -107,7 +97,7 @@ func HandleDuration(value string) float64 {
 			seconds += f * 24 * 60 * 60
 		}
 
-		//hours
+		// hours
 		if matches[4] != "" {
 			f, err := strconv.ParseFloat(matches[4], 64)
 			if err != nil {
@@ -117,7 +107,7 @@ func HandleDuration(value string) float64 {
 			seconds += f * 60 * 60
 		}
 
-		//minutes
+		// minutes
 		if matches[5] != "" {
 			f, err := strconv.ParseFloat(matches[5], 64)
 			if err != nil {
@@ -127,7 +117,7 @@ func HandleDuration(value string) float64 {
 			seconds += f * 60
 		}
 
-		//seconds & milliseconds
+		// seconds & milliseconds
 		if matches[6] != "" {
 			f, err := strconv.ParseFloat(matches[6], 64)
 			if err != nil {

cmd/collectors/restperf/restperf_test.go

@@ -205,12 +205,12 @@ func TestRestPerf_pollData(t *testing.T) {
 
 func newRestPerf(object string, path string) *RestPerf {
 	var err error
-	opts := options.Options{
-		Poller:   pollerName,
-		HomePath: "testdata",
-		IsTest:   true,
-	}
-	ac := collector.New("RestPerf", object, &opts, params(object, path), nil)
+	opts := options.New(options.WithConfPath("testdata/conf"))
+	opts.Poller = pollerName
+	opts.HomePath = "testdata"
+	opts.IsTest = true
+
+	ac := collector.New("RestPerf", object, opts, params(object, path), nil)
 	r := RestPerf{}
 	err = r.Init(ac)
 	if err != nil {

cmd/collectors/storagegrid/storagegrid.go

@@ -449,24 +449,18 @@ func (s *StorageGrid) InitProp() {
 
 func (s *StorageGrid) LoadTemplate() (string, error) {
 	var (
-		template     *node.Node
-		templatePath string
-		err          error
+		template *node.Node
+		path     string
+		err      error
 	)
 
-	// import template
-
-	template, templatePath, err = s.ImportSubTemplate(
-		"",
-		rest.TemplateFn(s.Params, s.Object),
-		s.client.Cluster.Version,
-	)
+	template, path, err = s.ImportSubTemplate("", rest.TemplateFn(s.Params, s.Object), s.client.Cluster.Version)
 	if err != nil {
 		return "", err
 	}
 
 	s.Params.Union(template)
-	return templatePath, nil
+	return path, nil
 }
 
 func (s *StorageGrid) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin {

cmd/collectors/zapi/collector/zapi.go

@@ -85,21 +85,20 @@ func (z *Zapi) Init(a *collector.AbstractCollector) error {
 }
 
 func (z *Zapi) InitVars() error {
-	var err error
-
 	// It's used for unit tests only
 	if z.Options.IsTest {
 		z.Client = client.NewTestClient()
 		templateName := z.Params.GetChildS("objects").GetChildContentS(z.Object)
-		template, templatePath, err := z.ImportSubTemplate("cdot", templateName, [3]int{9, 8, 0})
+		template, path, err := z.ImportSubTemplate("cdot", templateName, [3]int{9, 8, 0})
 		if err != nil {
-			return fmt.Errorf("unable to import template=[%s] %w", templatePath, err)
+			return err
 		}
-		z.TemplatePath = templatePath
+		z.TemplatePath = path
 		z.Params.Union(template)
 		return nil
 	}
 
+	var err error
 	if z.Client, err = client.New(conf.ZapiPoller(z.Params), z.Auth); err != nil { // convert to connection error, so poller aborts
 		return errs.New(errs.ErrConnection, err.Error())
 	}
@@ -122,12 +121,12 @@ func (z *Zapi) InitVars() error {
 	z.HostModel = model
 	templateName := z.Params.GetChildS("objects").GetChildContentS(z.Object)
 
-	template, templatePath, err := z.ImportSubTemplate(model, templateName, z.Client.Version())
+	template, path, err := z.ImportSubTemplate(model, templateName, z.Client.Version())
 	if err != nil {
-		return fmt.Errorf("unable to import template=[%s] %w", templatePath, err)
+		return err
 	}
 
-	z.TemplatePath = templatePath
+	z.TemplatePath = path
 
 	z.Params.Union(template)
 

cmd/collectors/zapiperf/zapiperf_test.go

@@ -36,16 +36,13 @@ func Test_ZapiPerf(t *testing.T) {
 func NewZapiPerf(object, path string) *ZapiPerf {
 	// homepath is harvest directory level
 	homePath := "../../../"
-	zapiperfPoller := "testZapiperf"
-
 	conf.TestLoadHarvestConfig("testdata/config.yml")
-	opts := options.Options{
-		Poller:   zapiperfPoller,
-		HomePath: homePath,
-		IsTest:   true,
-	}
+	opts := options.New(options.WithConfPath(homePath + "/conf"))
+	opts.Poller = "testZapiperf"
+	opts.HomePath = homePath
+	opts.IsTest = true
 
-	ac := collector.New("Zapiperf", object, &opts, params(object, path), nil)
+	ac := collector.New("Zapiperf", object, opts, params(object, path), nil)
 	z := &ZapiPerf{}
 	if err := z.Init(ac); err != nil {
 		log.Fatal().Err(err).Send()

cmd/exporters/influxdb/influxdb_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func setupInfluxDB(t *testing.T, exporterName string) *InfluxDB {
-	opts := &options.Options{}
+	opts := options.New()
 	opts.Debug = true
 
 	err := conf.LoadHarvestConfig("../../tools/doctor/testdata/testConfig.yml")

cmd/harvest/harvest.go

@@ -35,7 +35,7 @@ import (
 	_ "net/http/pprof" // #nosec since pprof is off by default
 	"os"
 	"os/exec"
-	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"syscall"
@@ -55,6 +55,7 @@ type options struct {
 	loglevel   int
 	logToFile  bool // only used when running in foreground
 	config     string
+	confPath   string
 	profiling  bool
 	longStatus bool
 	daemon     bool
@@ -358,50 +359,48 @@ func stopPoller(ps *util.PollerStatus) {
 func startPoller(pollerName string, promPort int, opts *options) {
 
 	argv := []string{
-		path.Join(HarvestHomePath, "bin", "poller"),
+		filepath.Join(HarvestHomePath, "bin", "poller"),
 		"--poller",
 		pollerName,
 		"--loglevel",
 		strconv.Itoa(opts.loglevel),
 	}
 
 	if promPort != 0 {
-		argv = append(argv, "--promPort")
-		argv = append(argv, strconv.Itoa(promPort))
+		argv = append(argv, "--promPort", strconv.Itoa(promPort))
 	}
 	if opts.debug {
 		argv = append(argv, "--debug")
 	}
 
 	if opts.config != HarvestConfigPath {
-		argv = append(argv, "--config")
-		argv = append(argv, opts.config)
+		argv = append(argv, "--config", opts.config)
+	}
+
+	if opts.confPath != conf.DefaultConfPath {
+		argv = append(argv, "--confpath", opts.confPath)
 	}
 
 	if opts.profiling {
 		if opts.foreground {
 			// Always pick the same port when profiling in foreground
-			argv = append(argv, "--profiling")
-			argv = append(argv, "6060")
+			argv = append(argv, "--profiling", "6060")
 		} else {
 			if port, err := freePort(); err != nil {
 				// No free port, log it and move on
 				fmt.Println("profiling disabled due to no free ports")
 			} else {
-				argv = append(argv, "--profiling")
-				argv = append(argv, strconv.Itoa(port))
+				argv = append(argv, "--profiling", strconv.Itoa(port))
 			}
 		}
 	}
 
 	if len(opts.collectors) > 0 {
-		argv = append(argv, "--collectors")
-		argv = append(argv, strings.Join(opts.collectors, ","))
+		argv = append(argv, "--collectors", strings.Join(opts.collectors, ","))
 	}
 
 	if len(opts.objects) > 0 {
-		argv = append(argv, "--objects")
-		argv = append(argv, strings.Join(opts.objects, ","))
+		argv = append(argv, "--objects", strings.Join(opts.objects, ","))
 	}
 
 	if opts.foreground {
@@ -545,6 +544,8 @@ func init() {
 	rootCmd.AddCommand(admin.Cmd())
 
 	rootCmd.PersistentFlags().StringVar(&opts.config, "config", "./harvest.yml", "Harvest config file path")
+	rootCmd.PersistentFlags().StringVar(&opts.confPath, "confpath", "conf", "colon-seperated paths to search for Harvest templates")
+
 	rootCmd.Version = version.String()
 	rootCmd.SetVersionTemplate(version.String())
 	rootCmd.SetUsageTemplate(rootCmd.UsageTemplate() + `