From 9a48e35b786ccbacd28eb03aa037aef561bdc15e Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Mon, 22 Jul 2019 13:16:31 +0300 Subject: [PATCH 1/4] feat: traverse symlinks to directories --- pkg/addon-operator/operator.go | 1 + pkg/module_manager/hook.go | 7 +- pkg/module_manager/module.go | 39 +++++---- pkg/utils/fschecksum.go | 79 +++++++++++++++++ pkg/utils/fswalk.go | 151 +++++++++++++++++++++++++++++++++ 5 files changed, 258 insertions(+), 19 deletions(-) create mode 100644 pkg/utils/fschecksum.go create mode 100644 pkg/utils/fswalk.go diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index 9527d8a8..7d1c47ed 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -401,6 +401,7 @@ func runDiscoverModulesState(t task.Task) error { ScheduledHooks = UpdateScheduleHooks(nil) // Enable kube events hooks for newly enabled modules + // FIXME convert to a task that run after AfterHelm if there is a flag in binding config. for _, moduleName := range modulesState.EnabledModules { err = KubeEventsHooks.EnableModuleHooks(moduleName, ModuleManager, KubeEventsManager) if err != nil { diff --git a/pkg/module_manager/hook.go b/pkg/module_manager/hook.go index a80316d8..7e1da835 100644 --- a/pkg/module_manager/hook.go +++ b/pkg/module_manager/hook.go @@ -31,8 +31,11 @@ type ModuleHook struct { } type Hook struct { - Name string // The unique name like 'global-hooks/startup_hook or 002-module/hooks/cleanup'. - Path string // The absolute path to the executable file. + // The unique name like 'global-hooks/startup_hook' or '002-module/hooks/cleanup'. + Name string + // The absolute path of the executable file. + Path string + Bindings []BindingType OrderByBinding map[BindingType]float64 diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index 76a5fd41..a2f5d50f 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -5,8 +5,10 @@ import ( "io/ioutil" "os" "os/exec" + "path" "path/filepath" "regexp" + "sort" "strings" "github.com/kennygrant/sanitize" @@ -16,7 +18,6 @@ import ( "github.com/flant/addon-operator/pkg/utils" "github.com/flant/shell-operator/pkg/executor" - utils_checksum "github.com/flant/shell-operator/pkg/utils/checksum" utils_file "github.com/flant/shell-operator/pkg/utils/file" ) @@ -107,7 +108,7 @@ func (m *Module) execRun() error { return err } - checksum, err := utils_checksum.CalculateChecksumOfPaths(runChartPath, valuesPath) + checksum, err := utils.CalculateChecksumOfPaths(runChartPath, valuesPath) if err != nil { return err } @@ -493,7 +494,7 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, } // initModulesIndex load all available modules from modules directory -// +// FIXME: Only 000-name modules are loaded, allow non-prefixed modules. func (mm *MainModuleManager) initModulesIndex() error { rlog.Debug("INIT: Search modules ...") @@ -607,27 +608,31 @@ func (mm *MainModuleManager) loadGlobalModulesValues() (utils.Values, error) { func getExecutableHooksFilesPaths(dir string) ([]string, error) { paths := make([]string, 0) - err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - if f.IsDir() { - return nil - } + nonExecutables := []string{} - if utils_file.IsFileExecutable(f) { - paths = append(paths, path) - } else { - return fmt.Errorf("found non-executable hook file '%s'", path) + // Find only executable files + files, err := utils.FilesFromRoot(dir, func(dir string, name string, info os.FileInfo) bool { + if info.Mode()&0111 != 0 { + return true } - - return nil + nonExecutables = append(nonExecutables, path.Join(dir, name)) + return false }) - if err != nil { return nil, err } + if len(nonExecutables) > 0 { + return nil, fmt.Errorf("found non-executable hook files '%+v'", nonExecutables) + } + + for dirPath, filePaths := range files { + for file := range filePaths { + paths = append(paths, path.Join(dirPath, file)) + } + } + + sort.Strings(paths) return paths, nil } diff --git a/pkg/utils/fschecksum.go b/pkg/utils/fschecksum.go new file mode 100644 index 00000000..19bdd3d7 --- /dev/null +++ b/pkg/utils/fschecksum.go @@ -0,0 +1,79 @@ +package utils + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io/ioutil" + "os" + "path" + "sort" +) + +func CalculateStringsChecksum(stringArr ...string) string { + hasher := md5.New() + sort.Strings(stringArr) + for _, value := range stringArr { + hasher.Write([]byte(value)) + } + return hex.EncodeToString(hasher.Sum(nil)) +} + +func CalculateChecksumOfFile(path string) (string, error) { + content, err := ioutil.ReadFile(path) + if err != nil { + return "", err + } + return CalculateStringsChecksum(string(content)), nil +} + +func CalculateChecksumOfDirectory(dir string) (string, error) { + res := "" + + var checkErr error + files, err := FilesFromRoot(dir, func(dir string, name string, info os.FileInfo) bool { + fPath := path.Join(dir, name) + checksum, err := CalculateChecksumOfFile(fPath) + if err != nil { + // return only bad files for logging + checkErr = err + return true + } + res = CalculateStringsChecksum(res, checksum) + // good files are skipped + return false + }) + if err != nil { + return "", err + } + if checkErr != nil { + return "", fmt.Errorf("calculate checksum of %+v: %v", files, err) + } + + return res, nil +} + +func CalculateChecksumOfPaths(pathes ...string) (string, error) { + res := "" + + for _, aPath := range pathes { + fileInfo, err := os.Stat(aPath) + if err != nil { + return "", err + } + + var checksum string + if fileInfo.IsDir() { + checksum, err = CalculateChecksumOfDirectory(aPath) + } else { + checksum, err = CalculateChecksumOfFile(aPath) + } + + if err != nil { + return "", err + } + res = CalculateStringsChecksum(res, checksum) + } + + return res, nil +} diff --git a/pkg/utils/fswalk.go b/pkg/utils/fswalk.go new file mode 100644 index 00000000..b005fa41 --- /dev/null +++ b/pkg/utils/fswalk.go @@ -0,0 +1,151 @@ +package utils + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" +) + +/* + * Example: + + files, err = FilesFromRoot("./dir", func(dir string, name string, info os.FileInfo) bool { + return info.Mode()&0111 != 0 + }) + if err != nil { + fmt.Printf("FilesFromRoot: %v", err) + } + dirPaths = []string{} + for dirPath := range files { + dirPaths = append(dirPaths, dirPath) + } + sort.Strings(dirPaths) + + for _, dirPath := range dirPaths { + fmt.Printf("%s\n", dirPath) + for file := range files[dirPath] { + fmt.Printf(" %s\n", file) + } + } + + */ + +// FilesFromRoot returns a map with path and array of files under it +func FilesFromRoot(root string, filterFn func(dir string, name string, info os.FileInfo) bool) (files map[string]map[string]string, err error) { + files = make(map[string]map[string]string, 0) + + symlinkedDirs, err := WalkSymlinks(root, "", files, filterFn) + if err != nil { + return nil, err + } + if len(symlinkedDirs) == 0 { + return files, nil + } + + walkedSymlinks := map[string]string{} + + // recurse list of symlinked directories + // limit depth to stop symlink loops + maxSymlinkDepth := 16 + for { + maxSymlinkDepth-- + if maxSymlinkDepth == 0 { + break + } + + newSymlinkedDirs := map[string]string{} + + for origPath, target := range symlinkedDirs { + symlinked, err := WalkSymlinks(target, origPath, files, filterFn) + if err != nil { + return nil, err + } + for k,v := range symlinked { + newSymlinkedDirs[k]=v + } + } + + for k := range walkedSymlinks { + if _, has := newSymlinkedDirs[k]; has { + delete(newSymlinkedDirs, k) + } + } + + if len(newSymlinkedDirs) == 0 { + break + } + + symlinkedDirs = newSymlinkedDirs + } + + return files, nil +} + +func SymlinkInfo(path string, info os.FileInfo) (target string, isDir bool, err error) { + // return empty path if not a symlink + if info.Mode()&os.ModeSymlink == 0 { + return "", false, nil + } + + // Eval symlink path and get stat of a target path + + target, err = filepath.EvalSymlinks(path) + if err != nil { + return "", false, err + } + // is it file or dir? + targetInfo, err := os.Lstat(target) + if err != nil { + return "", false, err + } + + return target, targetInfo.IsDir(), nil +} + +// WalkSymlinks walks a directory, updates files map and returns symlinked directories +func WalkSymlinks(target string, linkName string, files map[string]map[string]string, filterFn func(dir string, name string, info os.FileInfo) bool) (symlinkedDirectories map[string]string, err error) { + symlinkedDirectories = map[string]string{} + + err = filepath.Walk(target, func(foundPath string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + + resPath := foundPath + if linkName != "" { + // replace target with linkName in foundPath + resPath = path.Join(linkName, strings.TrimPrefix(foundPath, target)) + } + + target, isDir, err := SymlinkInfo(foundPath, info) + if err != nil { + return err + } + if target != "" && isDir { + // symlink to directory -> save it for future listing + symlinkedDirectories[resPath] = target + return nil + } + + // Walk found a file or symlink to file, just store it. + // FIXME symlink can have +x, but target file is not, so filterFn is not working properly + fDir := path.Dir(resPath) + fName := path.Base(resPath) + if filterFn == nil || filterFn(fDir, fName, info) { + if _, has := files[fDir]; !has { + files[fDir] = map[string]string{} + } + files[fDir][fName] = "" + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk symlinks dir %s: %v", target, err) + } + + return symlinkedDirectories, nil +} + From 825383f6651097ab05455ce5f1dac40cb53f1d94 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Tue, 13 Aug 2019 12:53:59 +0300 Subject: [PATCH 2/4] refactor: helm client, comments, small fixes, code cleanup - move helm client var to helm package - remove module hooks on errors - merge some methods - remove modulesHooksByName index map, use search method - remove unused module change types --- pkg/addon-operator/operator.go | 70 +++----------- pkg/addon-operator/operator_test.go | 29 +++--- pkg/helm/helm.go | 18 ++-- pkg/helm/helm_test.go | 2 +- .../kube_config_manager.go | 1 + pkg/module_manager/hook.go | 13 ++- pkg/module_manager/module.go | 74 +++++++------- pkg/module_manager/module_manager.go | 96 +++++++++---------- .../module_manager_internal_test.go | 10 +- pkg/module_manager/module_manager_test.go | 11 ++- pkg/utils/fswalk.go | 29 ++++++ pkg/utils/values.go | 2 + 12 files changed, 176 insertions(+), 179 deletions(-) diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index 7d1c47ed..981ef61d 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -52,9 +52,6 @@ var ( // ManagersEventsHandlerStopCh is the channel object for stopping infinite loop of the ManagersEventsHandler. ManagersEventsHandlerStopCh chan struct{} - - // HelmClient is the object for Helm client. - HelmClient helm.HelmClient ) const DefaultModulesDir = "modules" @@ -123,7 +120,7 @@ func Init() error { // Initializing helm. Installing Tiller, if it is missing. tillerNamespace := kube.AddonOperatorNamespace rlog.Infof("INIT: Namespace for Tiller: %s", tillerNamespace) - HelmClient, err = helm.Init(tillerNamespace) + err = helm.Init(tillerNamespace) if err != nil { rlog.Errorf("INIT: cannot initialize Tiller in ns/%s: %s", tillerNamespace, err) return err @@ -146,8 +143,7 @@ func Init() error { kube_config_manager.ValuesChecksumsAnnotation = ValuesChecksumsAnnotation module_manager.Init() ModuleManager = module_manager.NewMainModuleManager(). - WithDirectories(ModulesDir, GlobalHooksDir, TempDir). - WithHelmClient(HelmClient) + WithDirectories(ModulesDir, GlobalHooksDir, TempDir) err = ModuleManager.Init() if err != nil { rlog.Errorf("INIT: Cannot initialize module manager: %s", err) @@ -232,53 +228,15 @@ func ManagersEventsHandler() { // Some modules have changed. case module_manager.ModulesChanged: for _, moduleChange := range moduleEvent.ModulesChanges { - switch moduleChange.ChangeType { - case module_manager.Enabled: - // TODO этого события по сути нет. Нужно реализовать для вызова onStartup! - rlog.Infof("EVENT ModulesChanged, type=Enabled") - newTask := task.NewTask(task.ModuleRun, moduleChange.Name). - WithOnStartupHooks(true) - TasksQueue.Add(newTask) - rlog.Infof("QUEUE add ModuleRun %s", newTask.Name) - - err := KubeEventsHooks.EnableModuleHooks(moduleChange.Name, ModuleManager, KubeEventsManager) - if err != nil { - rlog.Errorf("MAIN_LOOP module '%s' enabled: cannot enable hooks: %s", moduleChange.Name, err) - } - - case module_manager.Changed: - rlog.Infof("EVENT ModulesChanged, type=Changed") - newTask := task.NewTask(task.ModuleRun, moduleChange.Name) - TasksQueue.Add(newTask) - rlog.Infof("QUEUE add ModuleRun %s", newTask.Name) - - case module_manager.Disabled: - rlog.Infof("EVENT ModulesChanged, type=Disabled") - newTask := task.NewTask(task.ModuleDelete, moduleChange.Name) - TasksQueue.Add(newTask) - rlog.Infof("QUEUE add ModuleDelete %s", newTask.Name) - - err := KubeEventsHooks.DisableModuleHooks(moduleChange.Name, ModuleManager, KubeEventsManager) - if err != nil { - rlog.Errorf("MAIN_LOOP module '%s' disabled: cannot disable hooks: %s", moduleChange.Name, err) - } - - case module_manager.Purged: - rlog.Infof("EVENT ModulesChanged, type=Purged") - newTask := task.NewTask(task.ModulePurge, moduleChange.Name) - TasksQueue.Add(newTask) - rlog.Infof("QUEUE add ModulePurge %s", newTask.Name) - - err := KubeEventsHooks.DisableModuleHooks(moduleChange.Name, ModuleManager, KubeEventsManager) - if err != nil { - rlog.Errorf("MAIN_LOOP module '%s' purged: cannot disable hooks: %s", moduleChange.Name, err) - } - } + rlog.Infof("EVENT ModulesChanged, type=Changed") + newTask := task.NewTask(task.ModuleRun, moduleChange.Name) + TasksQueue.Add(newTask) + rlog.Infof("QUEUE add ModuleRun %s", newTask.Name) } // As module list may have changed, hook schedule index must be re-created. ScheduledHooks = UpdateScheduleHooks(ScheduledHooks) - // As global values may have changed, all modules must be restarted. case module_manager.GlobalChanged: + // Global values are changed, all modules must be restarted. rlog.Infof("EVENT GlobalChanged") TasksQueue.ChangesDisable() CreateReloadAllTasks(false) @@ -286,7 +244,7 @@ func ManagersEventsHandler() { // Re-creating schedule hook index ScheduledHooks = UpdateScheduleHooks(ScheduledHooks) case module_manager.AmbigousState: - rlog.Infof("EVENT AmbigousState") + rlog.Infof("EVENT AmbiguousState") TasksQueue.ChangesDisable() // It is the error in the module manager. The task must be added to // the beginning of the queue so the module manager can restore its @@ -401,7 +359,7 @@ func runDiscoverModulesState(t task.Task) error { ScheduledHooks = UpdateScheduleHooks(nil) // Enable kube events hooks for newly enabled modules - // FIXME convert to a task that run after AfterHelm if there is a flag in binding config. + // FIXME convert to a task that run after AfterHelm if there is a flag in binding config to start informers after CRD installation. for _, moduleName := range modulesState.EnabledModules { err = KubeEventsHooks.EnableModuleHooks(moduleName, ModuleManager, KubeEventsManager) if err != nil { @@ -409,6 +367,7 @@ func runDiscoverModulesState(t task.Task) error { } } + // TODO is queue should be cleaned from hook run tasks of deleted module? // Disable kube events hooks for newly disabled modules for _, moduleName := range modulesState.ModulesToDisable { err = KubeEventsHooks.DisableModuleHooks(moduleName, ModuleManager, KubeEventsManager) @@ -423,9 +382,7 @@ func runDiscoverModulesState(t task.Task) error { // TasksRunner handle tasks in queue. // // Task handler may delay task processing by pushing delay to the queue. -// TODO пока только один обработчик, всё ок. Но лучше, чтобы очередь позволяла удалять только то, чему ранее был сделан peek. -// Т.е. кто взял в обработку задание, тот его и удалил из очереди. Сейчас Peek-нуть может одна го-рутина, другая добавит, -// первая Pop-нет задание — новое задание пропало, второй раз будет обработано одно и тоже. +// FIXME: For now, only one TaskRunner for a TasksQueue. There should be a lock between Peek and Pop to prevent Poping tasks from other TaskRunner func TasksRunner() { for { if TasksQueue.IsEmpty() { @@ -458,7 +415,7 @@ func TasksRunner() { if err != nil { MetricsStorage.SendCounterMetric(PrefixMetric("module_run_errors"), 1.0, map[string]string{"module": t.GetName()}) t.IncrementFailureCount() - rlog.Errorf("TASK_RUN %s '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetFailureCount(), err) + rlog.Errorf("TASK_RUN ModuleRun '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetName(), t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) rlog.Infof("QUEUE push FailedModuleDelay") } else { @@ -519,14 +476,13 @@ func TasksRunner() { case task.ModulePurge: rlog.Infof("TASK_RUN ModulePurge %s", t.GetName()) // Module for purge is unknown so log deletion error is enough. - err := HelmClient.DeleteRelease(t.GetName()) + err := helm.Client.DeleteRelease(t.GetName()) if err != nil { rlog.Errorf("TASK_RUN %s Helm delete '%s' failed. Error: %s", t.GetType(), t.GetName(), err) } TasksQueue.Pop() case task.ModuleManagerRetry: rlog.Infof("TASK_RUN ModuleManagerRetry") - // TODO метрику нужно отсылать из module_manager. Cделать metric_storage глобальным! MetricsStorage.SendCounterMetric(PrefixMetric("modules_discover_errors"), 1.0, map[string]string{}) ModuleManager.Retry() TasksQueue.Pop() diff --git a/pkg/addon-operator/operator_test.go b/pkg/addon-operator/operator_test.go index a01aca3e..1b52342d 100644 --- a/pkg/addon-operator/operator_test.go +++ b/pkg/addon-operator/operator_test.go @@ -256,11 +256,6 @@ func (m *ModuleManagerMock) WithDirectories(modulesDir string, globalHooksDir st return m } -func (m *ModuleManagerMock) WithHelmClient(helm helm.HelmClient) module_manager.ModuleManager { - fmt.Println("WithHelm") - return m -} - func (m *ModuleManagerMock) WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) module_manager.ModuleManager { fmt.Println("WithKubeConfigManager") return m @@ -350,7 +345,6 @@ func TestMain_ModulesEventsHandler(t *testing.T) { KubeEventsManager = &KubeEventsManagerMock{} KubeEventsHooks = &KubeEventsHooksControllerMock{} - assert.Equal(t, 0, 0) fmt.Println("Create queue") // Fill a queue TasksQueue = task.NewTasksQueue() @@ -358,6 +352,8 @@ func TestMain_ModulesEventsHandler(t *testing.T) { TasksQueue.AddWatcher(&QueueDumperTest{}) TasksQueue.ChangesEnable(true) + assert.Equal(t, 0, TasksQueue.Length()) + go func(ch chan module_manager.Event) { ch <- module_manager.Event{ Type: module_manager.ModulesChanged, @@ -368,7 +364,7 @@ func TestMain_ModulesEventsHandler(t *testing.T) { }, { Name: "test_module_2", - ChangeType: module_manager.Disabled, + ChangeType: module_manager.Changed, }, }, } @@ -376,12 +372,17 @@ func TestMain_ModulesEventsHandler(t *testing.T) { Type: module_manager.ModulesChanged, ModulesChanges: []module_manager.ModuleChange{ { - Name: "test_module_purged", - ChangeType: module_manager.Purged, + Name: "test_module_2", + ChangeType: module_manager.Changed, }, + }, + } + ch <- module_manager.Event{ + Type: module_manager.ModulesChanged, + ModulesChanges: []module_manager.ModuleChange{ { - Name: "test_module_enabled", - ChangeType: module_manager.Enabled, + Name: "test_module_1", + ChangeType: module_manager.Changed, }, }, } @@ -426,7 +427,7 @@ func TestMain_Run_With_InfiniteModuleError(t *testing.T) { // Сделать моки для всего, что нужно для запуска Run - HelmClient = MockHelmClient{ + helm.Client = MockHelmClient{ DeleteReleaseErrorsCount: 0, } @@ -486,7 +487,7 @@ func TestMain_Run_With_RecoverableErrors(t *testing.T) { // Сделать моки для всего, что нужно для запуска Run - HelmClient = MockHelmClient{ + helm.Client = MockHelmClient{ DeleteReleaseErrorsCount: 3, } @@ -564,7 +565,7 @@ func TestMain_ScheduledTasks(t *testing.T) { runOrder = []int{} - HelmClient = MockHelmClient{ + helm.Client = MockHelmClient{ DeleteReleaseErrorsCount: 3, } diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 4de810ca..b89e0b3b 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -36,30 +36,34 @@ type HelmClient interface { IsReleaseExists(releaseName string) (bool, error) } +var Client HelmClient + type CliHelm struct { tillerNamespace string } // Init starts Tiller installation. -func Init(tillerNamespace string) (HelmClient, error) { +func Init(tillerNamespace string) error { rlog.Info("Helm: run helm init") - helm := &CliHelm{tillerNamespace: tillerNamespace} + cliHelm := &CliHelm{tillerNamespace: tillerNamespace} - err := helm.InitTiller() + err := cliHelm.InitTiller() if err != nil { - return nil, err + return err } - stdout, stderr, err := helm.Cmd("version") + stdout, stderr, err := cliHelm.Cmd("version") if err != nil { - return nil, fmt.Errorf("unable to get helm version: %v\n%v %v", err, stdout, stderr) + return fmt.Errorf("unable to get helm version: %v\n%v %v", err, stdout, stderr) } rlog.Infof("Helm: helm version:\n%v %v", stdout, stderr) rlog.Info("Helm: successfully initialized") - return helm, nil + Client = cliHelm + + return nil } // InitTiller runs helm init with the same ServiceAccountName, NodeSelector and Tolerations diff --git a/pkg/helm/helm_test.go b/pkg/helm/helm_test.go index cadeb10a..b6f90593 100644 --- a/pkg/helm/helm_test.go +++ b/pkg/helm/helm_test.go @@ -70,7 +70,7 @@ func shouldUpgradeRelease(helm HelmClient, releaseName string, chart string, val } func TestHelm(t *testing.T) { - // Для теста требуется kubernetes + helm, поэтому skip + // Skip because this test needs a Kubernetes cluster and a helm binary. t.Skip() var err error diff --git a/pkg/kube_config_manager/kube_config_manager.go b/pkg/kube_config_manager/kube_config_manager.go index ca0292d0..fe7798bd 100644 --- a/pkg/kube_config_manager/kube_config_manager.go +++ b/pkg/kube_config_manager/kube_config_manager.go @@ -448,6 +448,7 @@ func (kcm *MainKubeConfigManager) handleCmDelete(obj *v1.ConfigMap) error { // Global values is already known to be empty. // So check each module values change separately, // and generate signals per-module. + // Note: Only ModuleName field is needed in ModuleConfig. moduleConfigsUpdate := make(ModuleConfigs) diff --git a/pkg/module_manager/hook.go b/pkg/module_manager/hook.go index 7e1da835..298ff008 100644 --- a/pkg/module_manager/hook.go +++ b/pkg/module_manager/hook.go @@ -178,8 +178,6 @@ func (mm *MainModuleManager) addModuleHook(moduleName, name, path string, config mm.addModulesHooksOrderByName(moduleName, KubeEvents, moduleHook) } - mm.modulesHooksByName[name] = moduleHook - return nil } @@ -190,6 +188,11 @@ func (mm *MainModuleManager) addModulesHooksOrderByName(moduleName string, bindi mm.modulesHooksOrderByName[moduleName][bindingType] = append(mm.modulesHooksOrderByName[moduleName][bindingType], moduleHook) } +func (mm *MainModuleManager) removeModuleHooks(moduleName string) { + delete(mm.modulesHooksOrderByName, moduleName) +} + + type globalValuesMergeResult struct { // Global values with the root "global" key. Values utils.Values @@ -665,6 +668,8 @@ func (mm *MainModuleManager) initModuleHooks(module *Module) error { }) if err != nil { + // cleanup hook indexes on error + mm.removeModuleHooks(module.Name) return err } @@ -676,7 +681,8 @@ func (mm *MainModuleManager) initHooks(hooksDir string, addHook func(hookPath st return nil } - hooksRelativePaths, err := getExecutableHooksFilesPaths(hooksDir) // returns a list of executable hooks sorted by filename + // retrieve a list of executable files in hooksDir sorted by filename + hooksRelativePaths, _, err := utils.FindExecutableFilesInPath(hooksDir) if err != nil { return err } @@ -693,6 +699,7 @@ func (mm *MainModuleManager) initHooks(hooksDir string, addHook func(hookPath st } } + return nil } diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index a2f5d50f..f4c852ad 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -5,10 +5,8 @@ import ( "io/ioutil" "os" "os/exec" - "path" "path/filepath" "regexp" - "sort" "strings" "github.com/kennygrant/sanitize" @@ -16,6 +14,7 @@ import ( "github.com/romana/rlog" "gopkg.in/yaml.v2" + "github.com/flant/addon-operator/pkg/helm" "github.com/flant/addon-operator/pkg/utils" "github.com/flant/shell-operator/pkg/executor" utils_file "github.com/flant/shell-operator/pkg/utils/file" @@ -30,7 +29,7 @@ type Module struct { moduleManager *MainModuleManager } -func (mm *MainModuleManager) NewModule() *Module { +func NewModule(mm *MainModuleManager) *Module { module := &Module{} module.moduleManager = mm return module @@ -40,7 +39,9 @@ func (m *Module) SafeName() string { return sanitize.BaseName(m.Name) } -func (m *Module) run(onStartup bool) error { +// Run is a phase of module lifecycle that runs onStartup and beforeHelm hooks, helm upgrade --install command and afterHelm hook. +// It is a handler of task MODULE_RUN +func (m *Module) Run(onStartup bool) error { if err := m.cleanup(); err != nil { return err } @@ -66,6 +67,34 @@ func (m *Module) run(onStartup bool) error { return nil } +// Delete removes helm release if it exists and runs afterDeleteHelm hooks. +// It is a handler for MODULE_DELETE task. +func (m *Module) Delete() error { + // Если есть chart, но нет релиза — warning + // если нет чарта — молча перейти к хукам + // если есть и chart и релиз — удалить + chartExists, _ := m.checkHelmChart() + if chartExists { + releaseExists, err := helm.Client.IsReleaseExists(m.generateHelmReleaseName()) + if !releaseExists { + if err != nil { + rlog.Warnf("Module delete: Cannot find helm release '%s' for module '%s'. Helm error: %s", m.generateHelmReleaseName(), m.Name, err) + } else { + rlog.Warnf("Module delete: Cannot find helm release '%s' for module '%s'.", m.generateHelmReleaseName(), m.Name) + } + } else { + // Chart and release are existed, so run helm delete command + err := helm.Client.DeleteRelease(m.generateHelmReleaseName()) + if err != nil { + return err + } + } + } + + return m.runHooksByBinding(AfterDeleteHelm) +} + + func (m *Module) cleanup() error { chartExists, err := m.checkHelmChart() if !chartExists { @@ -76,11 +105,11 @@ func (m *Module) cleanup() error { } //rlog.Infof("MODULE '%s': cleanup helm revisions...", m.Name) - if err := m.moduleManager.helm.DeleteSingleFailedRevision(m.generateHelmReleaseName()); err != nil { + if err := helm.Client.DeleteSingleFailedRevision(m.generateHelmReleaseName()); err != nil { return err } - if err := m.moduleManager.helm.DeleteOldFailedRevisions(m.generateHelmReleaseName()); err != nil { + if err := helm.Client.DeleteOldFailedRevisions(m.generateHelmReleaseName()); err != nil { return err } @@ -521,7 +550,7 @@ func (mm *MainModuleManager) initModulesIndex() error { modulePath := filepath.Join(mm.ModulesDir, file.Name()) - module := mm.NewModule() + module := NewModule(mm) module.Name = moduleName module.DirectoryName = file.Name() module.Path = modulePath @@ -606,37 +635,6 @@ func (mm *MainModuleManager) loadGlobalModulesValues() (utils.Values, error) { return utils.FormatValues(res) } -func getExecutableHooksFilesPaths(dir string) ([]string, error) { - paths := make([]string, 0) - - nonExecutables := []string{} - - // Find only executable files - files, err := utils.FilesFromRoot(dir, func(dir string, name string, info os.FileInfo) bool { - if info.Mode()&0111 != 0 { - return true - } - nonExecutables = append(nonExecutables, path.Join(dir, name)) - return false - }) - if err != nil { - return nil, err - } - if len(nonExecutables) > 0 { - return nil, fmt.Errorf("found non-executable hook files '%+v'", nonExecutables) - } - - for dirPath, filePaths := range files { - for file := range filePaths { - paths = append(paths, path.Join(dirPath, file)) - } - } - - sort.Strings(paths) - - return paths, nil -} - func dumpData(filePath string, data []byte) error { err := ioutil.WriteFile(filePath, data, 0644) if err != nil { diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index b7319498..7d67a54b 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -33,14 +33,17 @@ type ModuleManager interface { RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext) error Retry() WithDirectories(modulesDir string, globalHooksDir string, tempDir string) ModuleManager - WithHelmClient(helm helm.HelmClient) ModuleManager WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager } -// All modules are in the right order to run/disable/purge +// ModulesState is a result of Discovery process, that determines which +// modules should be enabled, disabled or purged. type ModulesState struct { + // modules that should be run EnabledModules []string + // modules that should be deleted ModulesToDisable []string + // modules that should be purged ReleasedUnknownModules []string } @@ -69,9 +72,8 @@ type MainModuleManager struct { // Index for searching global hooks by their bindings. globalHooksOrder map[BindingType][]*GlobalHook - // Index of all module hooks. Key is module hook name. - modulesHooksByName map[string]*ModuleHook - // Index for searching module hooks by module name and by bindings. + // module hooks by module name and binding type ordered by name + // Note: one module hook can have several binding types. modulesHooksOrderByName map[string]map[BindingType][]*ModuleHook // global static values from modules/values.yaml file @@ -161,14 +163,9 @@ const ( type ChangeType string const ( - // Deprecated: Module becomes enabled is handled by module discovery. - Enabled ChangeType = "MODULE_ENABLED" - // Deprecated: Module become disabled is handled by module discovery. - Disabled ChangeType = "MODULE_DISABLED" + // All other types are deprecated. This const can be removed in future versions. // Module values are changed Changed ChangeType = "MODULE_CHANGED" - // Deprecated: Module files are no longer available is handled by module discovery. - Purged ChangeType = "MODULE_PURGED" ) // ModuleChange contains module name and type of module changes. @@ -200,7 +197,6 @@ func NewMainModuleManager() *MainModuleManager { enabledModulesInOrder: make([]string, 0), globalHooksByName: make(map[string]*GlobalHook), globalHooksOrder: make(map[BindingType][]*GlobalHook), - modulesHooksByName: make(map[string]*ModuleHook), modulesHooksOrderByName: make(map[string]map[BindingType][]*ModuleHook), globalStaticValues: make(utils.Values), kubeGlobalConfigValues: make(utils.Values), @@ -211,7 +207,6 @@ func NewMainModuleManager() *MainModuleManager { moduleValuesChanged: make(chan string, 1), globalValuesChanged: make(chan bool, 1), - helm: nil, kubeConfigManager: nil, moduleConfigsUpdateBeforeAmbiguos: make(kube_config_manager.ModuleConfigs), @@ -518,15 +513,25 @@ func (mm *MainModuleManager) Retry() { mm.retryOnAmbigous <- true } -// discoverModulesState calculate new arrays for enabled modules, to be disabled modules and to be purged modules -// This method needs updated mm.enabledModulesByConfig -func (mm *MainModuleManager) discoverModulesState() (*ModulesState, error) { - // EnabledModules — modules that should be run - // ModulesToDisable — modules that should be deleted - // ReleasedUnknownModules — modules that should be purged - state := &ModulesState{} - releasedModules, err := mm.helm.ListReleasesNames(nil) +// DiscoverModulesState handles DiscoverModulesState event: it calculates new arrays of enabled modules, +// modules that should be disabled and modules that should be purged. +// +// This method requires that mm.enabledModulesByConfig and mm.kubeModulesConfigValues are updated. +func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err error) { + rlog.Debugf("DISCOVER state:\n"+ + " mm.enabledModulesByConfig: %v\n"+ + " mm.enabledModulesInOrder: %v\n", + mm.enabledModulesByConfig, + mm.enabledModulesInOrder) + + state = &ModulesState{ + EnabledModules: []string{}, + ModulesToDisable: []string{}, + ReleasedUnknownModules: []string{}, + } + + releasedModules, err := helm.Client.ListReleasesNames(nil) if err != nil { return nil, err } @@ -558,6 +563,8 @@ func (mm *MainModuleManager) discoverModulesState() (*ModulesState, error) { } state.EnabledModules = enabledModules + // save enabled modules for future usages + mm.enabledModulesInOrder = enabledModules // Calculate modules that has helm release and are disabled for now. // Sort them in reverse order for proper deletion. @@ -565,24 +572,6 @@ func (mm *MainModuleManager) discoverModulesState() (*ModulesState, error) { state.ModulesToDisable = utils.ListIntersection(state.ModulesToDisable, releasedModules) state.ModulesToDisable = utils.SortReverseByReference(state.ModulesToDisable, mm.allModulesNamesInOrder) - return state, nil -} - -// DiscoverModulesState handles DiscoverModulesState event. -// This method needs updated mm.enabledModulesByConfig and mm.kubeModulesConfigValues -func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err error) { - rlog.Debugf("DISCOVER state:\n"+ - " mm.enabledModulesByConfig: %v\n"+ - " mm.enabledModulesInOrder: %v\n", - mm.enabledModulesByConfig, - mm.enabledModulesInOrder) - - state, err = mm.discoverModulesState() - if err != nil { - return nil, err - } - mm.enabledModulesInOrder = state.EnabledModules - rlog.Debugf("DISCOVER state results:\n"+ " mm.enabledModulesByConfig: %v\n"+ " EnabledModules: %v\n"+ @@ -619,12 +608,16 @@ func (mm *MainModuleManager) GetGlobalHook(name string) (*GlobalHook, error) { } func (mm *MainModuleManager) GetModuleHook(name string) (*ModuleHook, error) { - moduleHook, exist := mm.modulesHooksByName[name] - if exist { - return moduleHook, nil - } else { - return nil, fmt.Errorf("module hook '%s' not found", name) + for _, bindingHooks := range mm.modulesHooksOrderByName { + for _, hooks := range bindingHooks { + for _, hook := range hooks { + if hook.Name == name { + return hook, nil + } + } + } } + return nil, fmt.Errorf("module hook '%s' is not found", name) } func (mm *MainModuleManager) GetGlobalHooksInOrder(bindingType BindingType) []string { @@ -679,20 +672,24 @@ func (mm *MainModuleManager) DeleteModule(moduleName string) error { return err } - if err := module.delete(); err != nil { + if err := module.Delete(); err != nil { return err } + // remove hooks structures + mm.removeModuleHooks(moduleName) + return nil } -func (mm *MainModuleManager) RunModule(moduleName string, onStartup bool) error { // запускает before-helm + helm + after-helm +// RunModule runs beforeHelm hook, helm upgrade --install and afterHelm or afterDeleteHelm hook +func (mm *MainModuleManager) RunModule(moduleName string, onStartup bool) error { module, err := mm.GetModule(moduleName) if err != nil { return err } - if err := module.run(onStartup); err != nil { + if err := module.Run(onStartup); err != nil { return err } @@ -774,11 +771,6 @@ func (mm *MainModuleManager) WithDirectories(modulesDir string, globalHooksDir s return mm } -func (mm *MainModuleManager) WithHelmClient(helm helm.HelmClient) ModuleManager { - mm.helm = helm - return mm -} - func (mm *MainModuleManager) WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager { mm.kubeConfigManager = kubeConfigManager return mm diff --git a/pkg/module_manager/module_manager_internal_test.go b/pkg/module_manager/module_manager_internal_test.go index 73c7d529..a974ca67 100644 --- a/pkg/module_manager/module_manager_internal_test.go +++ b/pkg/module_manager/module_manager_internal_test.go @@ -148,10 +148,9 @@ func TestMainModuleManager_GetModule2(t *testing.T) { } func TestMainModuleManager_EnabledModules(t *testing.T) { + helm.Client = &MockHelmClient{} mm := NewMainModuleManager() - mm.WithHelmClient(&MockHelmClient{}) - runInitModulesIndex(t, mm, "test_get_module_names_in_order") expectedModules := []string{ @@ -438,10 +437,10 @@ func TestMainModuleManager_RunModule(t *testing.T) { // TODO something wrong here with patches from afterHelm and beforeHelm hooks t.SkipNow() hc := &MockHelmClient{} + helm.Client = hc mm := NewMainModuleManager() - mm.WithHelmClient(hc) mm.WithKubeConfigManager(MockKubeConfigManager{}) runInitModulesIndex(t, mm, "test_run_module") @@ -485,9 +484,9 @@ func TestMainModuleManager_DeleteModule(t *testing.T) { // TODO check afterHelmDelete patch t.SkipNow() hc := &MockHelmClient{} + helm.Client = hc mm := NewMainModuleManager() - mm.WithHelmClient(hc) mm.WithKubeConfigManager(MockKubeConfigManager{}) runInitModulesIndex(t, mm, "test_delete_module") @@ -528,8 +527,8 @@ func TestMainModuleManager_DeleteModule(t *testing.T) { func TestMainModuleManager_RunModuleHook(t *testing.T) { // TODO hooks not found t.SkipNow() + helm.Client = &MockHelmClient{} mm := NewMainModuleManager() - mm.WithHelmClient(&MockHelmClient{}) mm.WithKubeConfigManager(MockKubeConfigManager{}) runInitModulesIndex(t, mm, "test_run_module_hook") @@ -853,6 +852,7 @@ func TestMainModuleManager_GetGlobalHooksInOrder2(t *testing.T) { } func TestMainModuleManager_RunGlobalHook(t *testing.T) { + helm.Client = &MockHelmClient{} mm := NewMainModuleManager() mm.WithHelmClient(&MockHelmClient{}) mm.WithKubeConfigManager(MockKubeConfigManager{}) diff --git a/pkg/module_manager/module_manager_test.go b/pkg/module_manager/module_manager_test.go index a734422b..8b97a9aa 100644 --- a/pkg/module_manager/module_manager_test.go +++ b/pkg/module_manager/module_manager_test.go @@ -5,6 +5,8 @@ import ( "reflect" "strings" "testing" + + "github.com/flant/addon-operator/pkg/helm" ) func TestMainModuleManager_GetModule(t *testing.T) { @@ -33,7 +35,12 @@ func TestMainModuleManager_GetModuleHook(t *testing.T) { mm := NewMainModuleManager() expectedModuleHook := &ModuleHook{Hook: &Hook{Name: "hook"}} - mm.modulesHooksByName["hook"] = expectedModuleHook + + mm.modulesHooksOrderByName["module"] = map[BindingType][]*ModuleHook{ + OnStartup: { + expectedModuleHook, + }, + } moduleHook, err := mm.GetModuleHook("hook") if err != nil { @@ -215,8 +222,8 @@ func (helm *mockDiscoverModulesHelmClient) ListReleasesNames(_ map[string]string } func TestMainModuleManager_DiscoverModulesState(t *testing.T) { + helm.Client = &mockDiscoverModulesHelmClient{} mm := NewMainModuleManager() - mm.WithHelmClient(&mockDiscoverModulesHelmClient{}) mm.allModulesByName = make(map[string]*Module) mm.allModulesByName["module-1"] = &Module{Name: "module-1", DirectoryName: "001-module-1", Path: "some/path/001-module-1"} diff --git a/pkg/utils/fswalk.go b/pkg/utils/fswalk.go index b005fa41..27e5d708 100644 --- a/pkg/utils/fswalk.go +++ b/pkg/utils/fswalk.go @@ -5,6 +5,7 @@ import ( "os" "path" "path/filepath" + "sort" "strings" ) @@ -149,3 +150,31 @@ func WalkSymlinks(target string, linkName string, files map[string]map[string]st return symlinkedDirectories, nil } +// FindExecutableFilesInPath returns a list of executable and a list of non-executable files in path +func FindExecutableFilesInPath(dir string) (executables []string, nonExecutables []string, err error) { + executables = make([]string, 0) + + nonExecutables = make([]string, 0) + + // Find only executable files + files, err := FilesFromRoot(dir, func(dir string, name string, info os.FileInfo) bool { + if info.Mode()&0111 != 0 { + return true + } + nonExecutables = append(nonExecutables, path.Join(dir, name)) + return false + }) + if err != nil { + return + } + + for dirPath, filePaths := range files { + for file := range filePaths { + executables = append(executables, path.Join(dirPath, file)) + } + } + + sort.Strings(executables) + + return +} diff --git a/pkg/utils/values.go b/pkg/utils/values.go index d2940a7d..5ad62f42 100644 --- a/pkg/utils/values.go +++ b/pkg/utils/values.go @@ -55,10 +55,12 @@ func (op *ValuesPatchOperation) ToString() string { return string(data) } +// ModuleNameToValuesKey returns camelCased name from kebab-cased (very-simple-module become verySimpleModule) func ModuleNameToValuesKey(moduleName string) string { return camelcase.Camelcase(moduleName) } +// ModuleNameFromValuesKey returns kebab-cased name from camelCased (verySimpleModule become ver-simple-module) func ModuleNameFromValuesKey(moduleValuesKey string) string { b := make([]byte, 0, 64) l := len(moduleValuesKey) From 07ff9ea5459d188a913832ed5b79a8ffe666c7f1 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Tue, 13 Aug 2019 13:36:58 +0300 Subject: [PATCH 3/4] refactor: simplify hook execution code - --- pkg/addon-operator/operator_test.go | 6 +- pkg/kube_config_manager/module_kube_config.go | 2 + pkg/module_manager/hook.go | 216 +++++++++------ .../hook/kube_event/hooks_controller.go | 10 +- pkg/module_manager/module.go | 247 +++++++----------- pkg/utils/values.go | 4 + 6 files changed, 247 insertions(+), 238 deletions(-) diff --git a/pkg/addon-operator/operator_test.go b/pkg/addon-operator/operator_test.go index 1b52342d..a7366f26 100644 --- a/pkg/addon-operator/operator_test.go +++ b/pkg/addon-operator/operator_test.go @@ -125,7 +125,7 @@ func (m *ModuleManagerMock) DiscoverModulesState() (*module_manager.ModulesState func (m *ModuleManagerMock) GetGlobalHook(name string) (*module_manager.GlobalHook, error) { if _, has_hook := scheduledHooks[name]; has_hook { return &module_manager.GlobalHook{ - Hook: &module_manager.Hook{ + CommonHook: &module_manager.CommonHook{ Name: name, Path: "/addon-operator/hooks/global_1", Bindings: []module_manager.BindingType{module_manager.Schedule}, @@ -142,7 +142,7 @@ func (m *ModuleManagerMock) GetGlobalHook(name string) (*module_manager.GlobalHo } else { // Global hook run task handler requires Path field return &module_manager.GlobalHook{ - Hook: &module_manager.Hook{ + CommonHook: &module_manager.CommonHook{ Name: name, Path: "/addon-operator/hooks/global_hook_1_1", Bindings: []module_manager.BindingType{module_manager.BeforeAll}, @@ -158,7 +158,7 @@ func (m *ModuleManagerMock) GetGlobalHook(name string) (*module_manager.GlobalHo func (m *ModuleManagerMock) GetModuleHook(name string) (*module_manager.ModuleHook, error) { if _, has_hook := scheduledHooks[name]; has_hook { return &module_manager.ModuleHook{ - Hook: &module_manager.Hook{ + CommonHook: &module_manager.CommonHook{ Name: name, Path: "/addon-operator/modules/000_test_modu", Bindings: []module_manager.BindingType{module_manager.Schedule}, diff --git a/pkg/kube_config_manager/module_kube_config.go b/pkg/kube_config_manager/module_kube_config.go index 951b7ac5..dab02df0 100644 --- a/pkg/kube_config_manager/module_kube_config.go +++ b/pkg/kube_config_manager/module_kube_config.go @@ -8,7 +8,9 @@ import ( "gopkg.in/yaml.v2" ) +// TODO make a method of KubeConfig // GetModulesNamesFromConfigData returns all keys in kube config except global +// modNameEnabled keys are also handled func GetModulesNamesFromConfigData(configData map[string]string) map[string]bool { res := make(map[string]bool, 0) diff --git a/pkg/module_manager/hook.go b/pkg/module_manager/hook.go index 298ff008..92489928 100644 --- a/pkg/module_manager/hook.go +++ b/pkg/module_manager/hook.go @@ -16,21 +16,28 @@ import ( "github.com/flant/shell-operator/pkg/schedule_manager" utils_data "github.com/flant/shell-operator/pkg/utils/data" + "github.com/flant/addon-operator/pkg/helm" "github.com/flant/addon-operator/pkg/utils" ) type GlobalHook struct { - *Hook + *CommonHook Config *GlobalHookConfig } type ModuleHook struct { - *Hook + *CommonHook Module *Module Config *ModuleHookConfig } -type Hook struct { +type Hook interface { + GetName() string + GetPath() string + PrepareTmpFilesForHookRun(context []BindingContext) (map[string]string, error) +} + +type CommonHook struct { // The unique name like 'global-hooks/startup_hook' or '002-module/hooks/cleanup'. Name string // The absolute path of the executable file. @@ -61,15 +68,15 @@ type HookConfig struct { OnKubernetesEvent []kube_events_manager.OnKubernetesEventConfig `json:"onKubernetesEvent"` } -func (mm *MainModuleManager) newGlobalHook(name, path string, config *GlobalHookConfig) *GlobalHook { +func NewGlobalHook(name, path string, config *GlobalHookConfig, mm *MainModuleManager) *GlobalHook { globalHook := &GlobalHook{} - globalHook.Hook = mm.newHook(name, path) + globalHook.CommonHook = NewHook(name, path, mm) globalHook.Config = config return globalHook } -func (mm *MainModuleManager) newHook(name, path string) *Hook { - hook := &Hook{} +func NewHook(name, path string, mm *MainModuleManager) *CommonHook { + hook := &CommonHook{} hook.moduleManager = mm hook.Name = name hook.Path = path @@ -77,16 +84,16 @@ func (mm *MainModuleManager) newHook(name, path string) *Hook { return hook } -func (mm *MainModuleManager) newModuleHook(name, path string, config *ModuleHookConfig) *ModuleHook { +func NewModuleHook(name, path string, config *ModuleHookConfig, mm *MainModuleManager) *ModuleHook { moduleHook := &ModuleHook{} - moduleHook.Hook = mm.newHook(name, path) + moduleHook.CommonHook = NewHook(name, path, mm) moduleHook.Config = config return moduleHook } func (mm *MainModuleManager) addGlobalHook(name, path string, config *GlobalHookConfig) (err error) { var ok bool - globalHook := mm.newGlobalHook(name, path, config) + globalHook := NewGlobalHook(name, path, config, mm) if config.BeforeAll != nil { globalHook.Bindings = append(globalHook.Bindings, BeforeAll) @@ -129,7 +136,7 @@ func (mm *MainModuleManager) addGlobalHook(name, path string, config *GlobalHook func (mm *MainModuleManager) addModuleHook(moduleName, name, path string, config *ModuleHookConfig) (err error) { var ok bool - moduleHook := mm.newModuleHook(name, path, config) + moduleHook := NewModuleHook(name, path, config, mm) if moduleHook.Module, err = mm.GetModule(moduleName); err != nil { return err @@ -238,12 +245,14 @@ func (h *GlobalHook) handleGlobalValuesPatch(currentValues utils.Values, valuesP func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) error { rlog.Infof("Running global hook '%s' binding '%s' ...", h.Name, bindingType) - configValuesPatch, valuesPatch, err := h.exec(context) + globalHookExecutor := NewHookExecutor(h, context) + patches, err := globalHookExecutor.Run() if err != nil { return fmt.Errorf("global hook '%s' failed: %s", h.Name, err) } - if configValuesPatch != nil { + configValuesPatch, has := patches[utils.ConfigMapPatch] + if has && configValuesPatch != nil { preparedConfigValues := utils.MergeValues( utils.Values{"global": map[string]interface{}{}}, h.moduleManager.kubeGlobalConfigValues, @@ -265,7 +274,8 @@ func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) erro } } - if valuesPatch != nil { + valuesPatch, has := patches[utils.MemoryValuesPatch] + if has && valuesPatch != nil { valuesPatchResult, err := h.handleGlobalValuesPatch(h.values(), *valuesPatch) if err != nil { return fmt.Errorf("global hook '%s': dynamic global values update error: %s", h.Name, err) @@ -279,32 +289,41 @@ func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) erro return nil } -func (h *GlobalHook) exec(context []BindingContext) (*utils.ValuesPatch, *utils.ValuesPatch, error) { - configValuesPath, err := h.prepareConfigValuesJsonFile() +// PrepareTmpFilesForHookRun creates temporary files for hook and returns environment variables with paths +func (h *GlobalHook) PrepareTmpFilesForHookRun(context []BindingContext) (tmpFiles map[string]string, err error) { + tmpFiles = make(map[string]string, 0) + + tmpFiles["CONFIG_VALUES_PATH"], err = h.prepareConfigValuesJsonFile() if err != nil { - return nil, nil, err + return } - valuesPath, err := h.prepareValuesJsonFile() + + tmpFiles["VALUES_PATH"], err = h.prepareValuesJsonFile() if err != nil { - return nil, nil, err + return } - contextPath, err := h.prepareBindingContextJsonFile(context) - if err != nil { - return nil, nil, err + + if len(context) > 0 { + tmpFiles["BINDING_CONTEXT_PATH"], err = h.prepareBindingContextJsonFile(context) + if err != nil { + return + } } - cmd := h.moduleManager.makeHookCommand("", configValuesPath, valuesPath, contextPath, h.Path, []string{}, []string{}) - configValuesPatchPath, err := h.prepareConfigValuesJsonPatchFile() + tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"], err= h.prepareConfigValuesJsonPatchFile() if err != nil { - return nil, nil, err + return } - valuesPatchPath, err := h.prepareValuesJsonPatchFile() + + tmpFiles["VALUES_JSON_PATCH_PATH"], err = h.prepareValuesJsonPatchFile() if err != nil { - return nil, nil, err + return } - return h.moduleManager.execHook(h.Name, configValuesPatchPath, valuesPatchPath, cmd) + + return } + func (h *GlobalHook) configValues() utils.Values { return utils.MergeValues( utils.Values{"global": map[string]interface{}{}}, @@ -417,10 +436,18 @@ type moduleValuesMergeResult struct { ValuesChanged bool } -func (h *Hook) SafeName() string { +func (h *CommonHook) SafeName() string { return sanitize.BaseName(h.Name) } +func (h *CommonHook) GetName() string { + return h.Name +} + +func (h *CommonHook) GetPath() string { + return h.Path +} + func (h *ModuleHook) handleModuleValuesPatch(currentValues utils.Values, valuesPatch utils.ValuesPatch) (*moduleValuesMergeResult, error) { moduleValuesKey := utils.ModuleNameToValuesKey(h.Module.Name) @@ -474,12 +501,14 @@ func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) erro moduleName := h.Module.Name rlog.Infof("Running module hook '%s' binding '%s' ...", h.Name, bindingType) - configValuesPatch, valuesPatch, err := h.exec(context) + moduleHookExecutor := NewHookExecutor(h, context) + patches, err := moduleHookExecutor.Run() if err != nil { return fmt.Errorf("module hook '%s' failed: %s", h.Name, err) } - if configValuesPatch != nil { + configValuesPatch, has := patches[utils.ConfigMapPatch] + if has && configValuesPatch != nil{ preparedConfigValues := utils.MergeValues( utils.Values{utils.ModuleNameToValuesKey(moduleName): map[string]interface{}{}}, h.moduleManager.kubeModulesConfigValues[moduleName], @@ -501,7 +530,8 @@ func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) erro } } - if valuesPatch != nil { + valuesPatch, has := patches[utils.MemoryValuesPatch] + if has && valuesPatch != nil { valuesPatchResult, err := h.handleModuleValuesPatch(h.values(), *valuesPatch) if err != nil { return fmt.Errorf("module hook '%s': dynamic module values update error: %s", h.Name, err) @@ -515,33 +545,41 @@ func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) erro return nil } -func (h *ModuleHook) exec(context []BindingContext) (*utils.ValuesPatch, *utils.ValuesPatch, error) { - configValuesPath, err := h.prepareConfigValuesJsonFile() +// PrepareTmpFilesForHookRun creates temporary files for hook and returns environment variables with paths +func (h *ModuleHook) PrepareTmpFilesForHookRun(context []BindingContext) (tmpFiles map[string]string, err error) { + tmpFiles = make(map[string]string, 0) + + tmpFiles["CONFIG_VALUES_PATH"], err = h.prepareConfigValuesJsonFile() if err != nil { - return nil, nil, err + return } - valuesPath, err := h.prepareValuesJsonFile() + + tmpFiles["VALUES_PATH"], err = h.prepareValuesJsonFile() if err != nil { - return nil, nil, err + return } - contextPath, err := h.prepareBindingContextJsonFile(context) - if err != nil { - return nil, nil, err + + if len(context) > 0 { + tmpFiles["BINDING_CONTEXT_PATH"], err = h.prepareBindingContextJsonFile(context) + if err != nil { + return + } } - cmd := h.moduleManager.makeHookCommand("", configValuesPath, valuesPath, contextPath, h.Path, []string{}, []string{}) - configValuesPatchPath, err := h.prepareConfigValuesJsonPatchFile() + tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"], err= h.prepareConfigValuesJsonPatchFile() if err != nil { - return nil, nil, err + return } - valuesPatchPath, err := h.prepareValuesJsonPatchFile() + + tmpFiles["VALUES_JSON_PATCH_PATH"], err = h.prepareValuesJsonPatchFile() if err != nil { - return nil, nil, err + return } - return h.moduleManager.execHook(h.Name, configValuesPatchPath, valuesPatchPath, cmd) + return } + func (h *ModuleHook) configValues() utils.Values { return h.Module.configValues() } @@ -676,7 +714,7 @@ func (mm *MainModuleManager) initModuleHooks(module *Module) error { return nil } -func (mm *MainModuleManager) initHooks(hooksDir string, addHook func(hookPath string, output []byte) error) error { +func (mm *MainModuleManager) initHooks(hooksDir string, addHookFn func(hookPath string, output []byte) error) error { if _, err := os.Stat(hooksDir); os.IsNotExist(err) { return nil } @@ -694,7 +732,7 @@ func (mm *MainModuleManager) initHooks(hooksDir string, addHook func(hookPath st return fmt.Errorf("cannot get config for hook '%s': %s", hookPath, err) } - if err := addHook(hookPath, output); err != nil { + if err := addHookFn(hookPath, output); err != nil { return err } } @@ -735,30 +773,6 @@ func (h *ModuleHook) prepareValuesJsonPatchFile() (string, error) { return path, nil } -func (mm *MainModuleManager) execHook(hookName string, configValuesJsonPatchPath string, valuesJsonPatchPath string, cmd *exec.Cmd) (*utils.ValuesPatch, *utils.ValuesPatch, error) { - cmd.Env = append( - cmd.Env, - fmt.Sprintf("CONFIG_VALUES_JSON_PATCH_PATH=%s", configValuesJsonPatchPath), - fmt.Sprintf("VALUES_JSON_PATCH_PATH=%s", valuesJsonPatchPath), - ) - - err := executor.Run(cmd, true) - if err != nil { - return nil, nil, fmt.Errorf("%s FAILED: %s", hookName, err) - } - - configValuesPatch, err := utils.ValuesPatchFromFile(configValuesJsonPatchPath) - if err != nil { - return nil, nil, fmt.Errorf("got bad config values json patch from hook %s: %s", hookName, err) - } - - valuesPatch, err := utils.ValuesPatchFromFile(valuesJsonPatchPath) - if err != nil { - return nil, nil, fmt.Errorf("got bad values json patch from hook %s: %s", hookName, err) - } - - return configValuesPatch, valuesPatch, nil -} func createHookResultValuesFile(filePath string) error { file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) @@ -790,11 +804,57 @@ func execCommandOutput(cmd *exec.Cmd) ([]byte, error) { return output, nil } -func (mm *MainModuleManager) makeHookCommand(dir string, configValuesPath string, valuesPath string, contextPath string, entrypoint string, args []string, envs []string) *exec.Cmd { - envs = append(envs, fmt.Sprintf("CONFIG_VALUES_PATH=%s", configValuesPath)) - envs = append(envs, fmt.Sprintf("VALUES_PATH=%s", valuesPath)) - if contextPath != "" { - envs = append(envs, fmt.Sprintf("BINDING_CONTEXT_PATH=%s", contextPath)) + +type HookExecutor struct { + Hook Hook + Context []BindingContext + ConfigValuesPath string + ValuesPath string + ContextPath string + ConfigValuesPatchPath string + ValuesPatchPath string +} + +func NewHookExecutor(h Hook, context []BindingContext) *HookExecutor { + return &HookExecutor{ + Hook: h, + Context: context, + } +} + +func (e *HookExecutor) Run() (patches map[utils.ValuesPatchType]*utils.ValuesPatch, err error) { + patches = make(map[utils.ValuesPatchType]*utils.ValuesPatch) + + tmpFiles, err := e.Hook.PrepareTmpFilesForHookRun(e.Context) + if err != nil { + return nil, err + } + e.ConfigValuesPatchPath = tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"] + e.ValuesPatchPath = tmpFiles["VALUES_JSON_PATCH_PATH"] + + envs := []string{} + envs = append(envs, os.Environ()...) + for envName, filePath := range tmpFiles { + envs = append(envs, fmt.Sprintf("%s=%s", envName, filePath)) + } + envs = append(envs, helm.Client.CommandEnv()...) + + cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{}, envs) + + err = executor.Run(cmd, true) + if err != nil { + return nil, fmt.Errorf("%s FAILED: %s", e.Hook.GetName(), err) } - return mm.makeCommand(dir, entrypoint, args, envs) + + patches[utils.ConfigMapPatch], err = utils.ValuesPatchFromFile(e.ConfigValuesPatchPath) + if err != nil { + return nil, fmt.Errorf("got bad config values json patch from hook %s: %s", e.Hook.GetName(), err) + } + + patches[utils.MemoryValuesPatch], err = utils.ValuesPatchFromFile(e.ValuesPatchPath) + if err != nil { + return nil, fmt.Errorf("got bad values json patch from hook %s: %s", e.Hook.GetName(), err) + } + + return patches, nil } diff --git a/pkg/module_manager/hook/kube_event/hooks_controller.go b/pkg/module_manager/hook/kube_event/hooks_controller.go index 0df8bd1a..83546509 100644 --- a/pkg/module_manager/hook/kube_event/hooks_controller.go +++ b/pkg/module_manager/hook/kube_event/hooks_controller.go @@ -12,7 +12,7 @@ import ( ) // MakeKubeEventHookDescriptors converts hook config into KubeEventHook structures -func MakeKubeEventHookDescriptors(hook *module_manager.Hook, hookConfig *module_manager.HookConfig) []*kube_event.KubeEventHook { +func MakeKubeEventHookDescriptors(hook module_manager.Hook, hookConfig *module_manager.HookConfig) []*kube_event.KubeEventHook { res := make([]*kube_event.KubeEventHook, 0) for _, config := range hookConfig.OnKubernetesEvent { @@ -28,9 +28,9 @@ func MakeKubeEventHookDescriptors(hook *module_manager.Hook, hookConfig *module_ return res } -func ConvertOnKubernetesEventToKubeEventHook(hook *module_manager.Hook, config kube_events_manager.OnKubernetesEventConfig, namespace string) *kube_event.KubeEventHook { +func ConvertOnKubernetesEventToKubeEventHook(hook module_manager.Hook, config kube_events_manager.OnKubernetesEventConfig, namespace string) *kube_event.KubeEventHook { return &kube_event.KubeEventHook{ - HookName: hook.Name, + HookName: hook.GetName(), Name: config.Name, EventTypes: config.EventTypes, Kind: config.Kind, @@ -71,7 +71,7 @@ func (obj *MainKubeEventsHooksController) EnableGlobalHooks(moduleManager module for _, globalHookName := range globalHooks { globalHook, _ := moduleManager.GetGlobalHook(globalHookName) - for _, desc := range MakeKubeEventHookDescriptors(globalHook.Hook, &globalHook.Config.HookConfig) { + for _, desc := range MakeKubeEventHookDescriptors(globalHook, &globalHook.Config.HookConfig) { configId, err := eventsManager.Run(desc.EventTypes, desc.Kind, desc.Namespace, desc.Selector, desc.ObjectName, desc.JqFilter, desc.Debug) if err != nil { return err @@ -102,7 +102,7 @@ func (obj *MainKubeEventsHooksController) EnableModuleHooks(moduleName string, m for _, moduleHookName := range moduleHooks { moduleHook, _ := moduleManager.GetModuleHook(moduleHookName) - for _, desc := range MakeKubeEventHookDescriptors(moduleHook.Hook, &moduleHook.Config.HookConfig) { + for _, desc := range MakeKubeEventHookDescriptors(moduleHook, &moduleHook.Config.HookConfig) { configId, err := eventsManager.Run(desc.EventTypes, desc.Kind, desc.Namespace, desc.Selector, desc.ObjectName, desc.JqFilter, desc.Debug) if err != nil { return err diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index f4c852ad..aa9da049 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -4,7 +4,6 @@ import ( "fmt" "io/ioutil" "os" - "os/exec" "path/filepath" "regexp" "strings" @@ -56,7 +55,7 @@ func (m *Module) Run(onStartup bool) error { return err } - if err := m.execRun(); err != nil { + if err := m.runHelmInstall(); err != nil { return err } @@ -116,156 +115,100 @@ func (m *Module) cleanup() error { return nil } -func (m *Module) execRun() error { - err := m.execHelm(func(valuesPath, helmReleaseName string) error { - var err error - - runChartPath := filepath.Join(m.moduleManager.TempDir, fmt.Sprintf("%s.chart", m.SafeName())) - - err = os.RemoveAll(runChartPath) - if err != nil { - return err - } - err = copy.Copy(m.Path, runChartPath) - if err != nil { - return err - } - - // Prepare dummy empty values.yaml for helm not to fail - err = os.Truncate(filepath.Join(runChartPath, "values.yaml"), 0) - if err != nil { - return err - } - - checksum, err := utils.CalculateChecksumOfPaths(runChartPath, valuesPath) - if err != nil { - return err - } - - doRelease := true - - isReleaseExists, err := m.moduleManager.helm.IsReleaseExists(helmReleaseName) +func (m *Module) runHelmInstall() error { + chartExists, err := m.checkHelmChart() + if !chartExists { if err != nil { - return err - } - - if isReleaseExists { - _, status, err := m.moduleManager.helm.LastReleaseStatus(helmReleaseName) - if err != nil { - return err - } - - // Skip helm release for unchanged modules only for non FAILED releases - if status != "FAILED" { - releaseValues, err := m.moduleManager.helm.GetReleaseValues(helmReleaseName) - if err != nil { - return err - } - - if recordedChecksum, hasKey := releaseValues["_addonOperatorModuleChecksum"]; hasKey { - if recordedChecksumStr, ok := recordedChecksum.(string); ok { - if recordedChecksumStr == checksum { - doRelease = false - rlog.Infof("MODULE_RUN '%s': helm release '%s' checksum '%s' is not changed: skip helm upgrade", m.Name, helmReleaseName, checksum) - } else { - rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s' is changed to '%s': upgrade helm release", m.Name, helmReleaseName, recordedChecksumStr, checksum) - } - } - } - } + rlog.Debugf("Module '%s': no Chart.yaml, helm is not needed: %s", m.Name, err) + return nil } + } - if doRelease { - rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s': installing/upgrading release", m.Name, helmReleaseName, checksum) - - return m.moduleManager.helm.UpgradeRelease( - helmReleaseName, runChartPath, - []string{valuesPath}, - []string{fmt.Sprintf("_addonOperatorModuleChecksum=%s", checksum)}, - m.moduleManager.helm.TillerNamespace(), - ) - } else { - rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s': release install/upgrade is skipped", m.Name, helmReleaseName, checksum) - } + helmReleaseName := m.generateHelmReleaseName() - return nil - }) + // valuesPath, err := values.Dump + //valuesPath := filepath.Join(m.moduleManager.TempDir, fmt.Sprintf("%s.module-values.yaml", m.SafeName())) + //err := values.Dump(m.values(), NewDumperToYamlFile(valuesPath)) + //err := m.values().Dump(values.ToYamlFile(valuesPath)) + valuesPath, err := m.prepareValuesYamlFile() if err != nil { return err } - return nil -} + // Create a temporary chart with empty values.yaml + runChartPath := filepath.Join(m.moduleManager.TempDir, fmt.Sprintf("%s.chart", m.SafeName())) -// delete removes helm release if it exists. -// -func (m *Module) delete() error { - // Если есть chart, но нет релиза — warning - // если нет чарта — молча перейти к хукам - // если есть и chart и релиз — удалить - chartExists, _ := m.checkHelmChart() - if chartExists { - releaseExists, err := m.moduleManager.helm.IsReleaseExists(m.generateHelmReleaseName()) - if !releaseExists { - if err != nil { - rlog.Warnf("Module delete: Cannot find helm release '%s' for module '%s'. Helm error: %s", m.generateHelmReleaseName(), m.Name, err) - } else { - rlog.Warnf("Module delete: Cannot find helm release '%s' for module '%s'.", m.generateHelmReleaseName(), m.Name) - } - } else { - // Есть чарт и есть релиз — запуск удаления - err := m.moduleManager.helm.DeleteRelease(m.generateHelmReleaseName()) - if err != nil { - return err - } - } + err = os.RemoveAll(runChartPath) + if err != nil { + return err + } + err = copy.Copy(m.Path, runChartPath) + if err != nil { + return err } - if err := m.runHooksByBinding(AfterDeleteHelm); err != nil { + // Prepare dummy empty values.yaml for helm not to fail + err = os.Truncate(filepath.Join(runChartPath, "values.yaml"), 0) + if err != nil { return err } - return nil -} + checksum, err := utils.CalculateChecksumOfPaths(runChartPath, valuesPath) + if err != nil { + return err + } -// execDelete -// -// Deprecated: no usages found -func (m *Module) execDelete() error { - err := m.execHelm(func(_, helmReleaseName string) error { - return m.moduleManager.helm.DeleteRelease(helmReleaseName) - }) + doRelease := true + isReleaseExists, err := helm.Client.IsReleaseExists(helmReleaseName) if err != nil { return err } - return nil -} - -func (m *Module) execHelm(executeHelm func(valuesPath, helmReleaseName string) error) error { - chartExists, err := m.checkHelmChart() - if !chartExists { + if isReleaseExists { + _, status, err := helm.Client.LastReleaseStatus(helmReleaseName) if err != nil { - rlog.Debugf("Module '%s': no Chart.yaml, helm is not needed: %s", m.Name, err) - return nil + return err } - } - helmReleaseName := m.generateHelmReleaseName() - valuesPath, err := m.prepareValuesYamlFile() - if err != nil { - return err + // Skip helm release for unchanged modules only for non FAILED releases + if status != "FAILED" { + releaseValues, err := helm.Client.GetReleaseValues(helmReleaseName) + if err != nil { + return err + } + + if recordedChecksum, hasKey := releaseValues["_addonOperatorModuleChecksum"]; hasKey { + if recordedChecksumStr, ok := recordedChecksum.(string); ok { + if recordedChecksumStr == checksum { + doRelease = false + rlog.Infof("MODULE_RUN '%s': helm release '%s' checksum '%s' is not changed: skip helm upgrade", m.Name, helmReleaseName, checksum) + } else { + rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s' is changed to '%s': upgrade helm release", m.Name, helmReleaseName, recordedChecksumStr, checksum) + } + } + } + } } - if err = executeHelm(valuesPath, helmReleaseName); err != nil { - return err + if doRelease { + rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s': installing/upgrading release", m.Name, helmReleaseName, checksum) + + return helm.Client.UpgradeRelease( + helmReleaseName, runChartPath, + []string{valuesPath}, + []string{fmt.Sprintf("_addonOperatorModuleChecksum=%s", checksum)}, + helm.Client.TillerNamespace(), + ) + } else { + rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s': release install/upgrade is skipped", m.Name, helmReleaseName, checksum) } return nil } + func (m *Module) runHooksByBinding(binding BindingType) error { moduleHooksAfterHelm, err := m.moduleManager.GetModuleHooksInOrder(m.Name, binding) if err != nil { @@ -361,6 +304,9 @@ func (m *Module) checkHelmChart() (bool, error) { return true, nil } +// generateHelmReleaseName returns a string that can be used as a helm release name. +// +// TODO Now it returns just a module name. Should it be cleaned from special symbols? func (m *Module) generateHelmReleaseName() string { return m.Name } @@ -379,12 +325,10 @@ func (m *Module) configValues() utils.Values { // constructValues returns effective values for module hook: // -// global: static + kube + patches from hooks +// global section: static + kube + patches from hooks // -// module: static + kube + patches from hooks -// -// global section also contains enabledModules key with previously enabled modules -func (m *Module) constructValues(enabledModules []string) utils.Values { +// module section: static + kube + patches from hooks +func (m *Module) constructValues() utils.Values { var err error res := utils.MergeValues( @@ -413,25 +357,31 @@ func (m *Module) constructValues(enabledModules []string) utils.Values { } } - res = utils.MergeValues(res, m.constructEnabledModulesValues(enabledModules)) - return res } -func (m *Module) constructEnabledModulesValues(enabledModules []string) utils.Values { - return utils.Values{ +// valuesForEnabledScript returns merged values for enabled script. +// There is enabledModules key in global section with previously enabled modules. +func (m *Module) valuesForEnabledScript(precedingEnabledModules []string) utils.Values { + res := m.constructValues() + res = utils.MergeValues(res, utils.Values{ "global": map[string]interface{}{ - "enabledModules": enabledModules, + "enabledModules": precedingEnabledModules, }, - } -} - -func (m *Module) valuesForEnabledScript(precedingEnabledModules []string) utils.Values { - return m.constructValues(precedingEnabledModules) + }) + return res } +// values returns merged values for hooks. +// There is enabledModules key in global section with all enabled modules. func (m *Module) values() utils.Values { - return m.constructValues(m.moduleManager.enabledModulesInOrder) + res := m.constructValues() + res = utils.MergeValues(res, utils.Values{ + "global": map[string]interface{}{ + "enabledModules": m.moduleManager.enabledModulesInOrder, + }, + }) + return res } func (m *Module) moduleValuesKey() string { @@ -495,14 +445,13 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, rlog.Infof("MODULE '%s': run enabled script '%s'...", m.Name, enabledScriptPath) - // dir is empty to run hook in process's current directory - // FIXME: set to hook's directory? - cmd := m.moduleManager.makeHookCommand( - "", configValuesPath, valuesPath, "", enabledScriptPath, []string{}, - []string{ - fmt.Sprintf("MODULE_ENABLED_RESULT=%s", enabledResultFilePath), - }, - ) + envs := make([]string, 0) + envs = append(envs, os.Environ()...) + envs = append(envs, fmt.Sprintf("CONFIG_VALUES_PATH=%s", configValuesPath)) + envs = append(envs, fmt.Sprintf("VALUES_PATH=%s", valuesPath)) + envs = append(envs, fmt.Sprintf("MODULE_ENABLED_RESULT=%s", enabledResultFilePath)) + + cmd := executor.MakeCommand("", enabledScriptPath, []string{}, envs) if err := executor.Run(cmd, true); err != nil { return false, err @@ -591,7 +540,7 @@ func (mm *MainModuleManager) initGlobalConfigValues() (err error) { } // loadStaticValues loads config for module from values.yaml -// Module is considered as enabled if values.yaml is not exists. +// Module is enabled if values.yaml is not exists. func (m *Module) loadStaticValues() error { valuesYamlPath := filepath.Join(m.Path, "values.yaml") @@ -642,9 +591,3 @@ func dumpData(filePath string, data []byte) error { } return nil } - -func (mm *MainModuleManager) makeCommand(dir string, entrypoint string, args []string, envs []string) *exec.Cmd { - envs = append(envs, os.Environ()...) - envs = append(envs, mm.helm.CommandEnv()...) - return executor.MakeCommand(dir, entrypoint, args, envs) -} diff --git a/pkg/utils/values.go b/pkg/utils/values.go index 5ad62f42..6ff11b17 100644 --- a/pkg/utils/values.go +++ b/pkg/utils/values.go @@ -20,6 +20,10 @@ const ( GlobalValuesKey = "global" ) +type ValuesPatchType string +const ConfigMapPatch ValuesPatchType = "CONFIG_MAP_PATCH" +const MemoryValuesPatch ValuesPatchType = "MEMORY_VALUES_PATCH" + // Values stores values for modules or hooks by name. type Values map[string]interface{} From 3c93346d7171adb1d02be9d8c8beac092dcbebfa Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Mon, 22 Jul 2019 14:07:32 +0300 Subject: [PATCH 4/4] feat: explicit enabled key - module disabled in values.yaml should be explicitly enabled in cm - no support of `moduleName: bool` setting - module is disabled by default if no values.yaml or no explicit Enabled --- .../kube_config_manager.go | 19 +- .../kube_config_manager_test.go | 165 +++++++++--------- pkg/kube_config_manager/module_kube_config.go | 71 ++++---- pkg/module_manager/hook.go | 2 +- pkg/module_manager/module.go | 4 +- pkg/module_manager/module_manager.go | 28 ++- .../module_manager_internal_test.go | 19 +- pkg/module_manager/module_manager_test.go | 18 +- .../modules/000-module-c/values.yaml | 1 + .../modules/200-module-b/values.yaml | 3 +- .../modules/300-module-disabled/values.yaml | 2 +- pkg/utils/module_config.go | 121 +++++++++++-- pkg/utils/module_config_test.go | 161 +++++++++++++++++ pkg/utils/values.go | 51 ++++++ pkg/utils/values_test.go | 112 ------------ 15 files changed, 487 insertions(+), 290 deletions(-) create mode 100644 pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/000-module-c/values.yaml create mode 100644 pkg/utils/module_config_test.go diff --git a/pkg/kube_config_manager/kube_config_manager.go b/pkg/kube_config_manager/kube_config_manager.go index fe7798bd..262a69f0 100644 --- a/pkg/kube_config_manager/kube_config_manager.go +++ b/pkg/kube_config_manager/kube_config_manager.go @@ -233,9 +233,9 @@ func (kcm *MainKubeConfigManager) initConfig() error { globalValuesChecksum = globalKubeConfig.Checksum } - for module := range GetModulesNamesFromConfigData(obj.Data) { + for moduleName := range GetModulesNamesFromConfigData(obj.Data) { // all GetModulesNamesFromConfigData must exist - moduleKubeConfig, err := ModuleKubeConfigMustExist(GetModuleKubeConfigFromConfigData(module, obj.Data)) + moduleKubeConfig, err := ExtractModuleKubeConfig(moduleName, obj.Data) if err != nil { return err } @@ -337,9 +337,9 @@ func (kcm *MainKubeConfigManager) handleNewCm(obj *v1.ConfigMap) error { // calculate new checksums of a module sections newModulesValuesChecksum := make(map[string]string) - for module := range GetModulesNamesFromConfigData(obj.Data) { + for moduleName := range GetModulesNamesFromConfigData(obj.Data) { // all GetModulesNamesFromConfigData must exist - moduleKubeConfig, err := ModuleKubeConfigMustExist(GetModuleKubeConfigFromConfigData(module, obj.Data)) + moduleKubeConfig, err := ExtractModuleKubeConfig(moduleName, obj.Data) if err != nil { return err } @@ -365,21 +365,21 @@ func (kcm *MainKubeConfigManager) handleNewCm(obj *v1.ConfigMap) error { // create ModuleConfig for each module in configData // IsUpdated flag set for updated configs - for module := range actualModulesNames { + for moduleName := range actualModulesNames { // all GetModulesNamesFromConfigData must exist - moduleKubeConfig, err := ModuleKubeConfigMustExist(GetModuleKubeConfigFromConfigData(module, obj.Data)) + moduleKubeConfig, err := ExtractModuleKubeConfig(moduleName, obj.Data) if err != nil { return err } - if moduleKubeConfig.Checksum != savedChecksums[module] && moduleKubeConfig.Checksum != kcm.ModulesValuesChecksum[module] { - kcm.ModulesValuesChecksum[module] = moduleKubeConfig.Checksum + if moduleKubeConfig.Checksum != savedChecksums[moduleName] && moduleKubeConfig.Checksum != kcm.ModulesValuesChecksum[moduleName] { + kcm.ModulesValuesChecksum[moduleName] = moduleKubeConfig.Checksum moduleKubeConfig.ModuleConfig.IsUpdated = true updatedCount++ } else { moduleKubeConfig.ModuleConfig.IsUpdated = false } - moduleConfigsActual[module] = moduleKubeConfig.ModuleConfig + moduleConfigsActual[moduleName] = moduleKubeConfig.ModuleConfig } // delete checksums for removed module sections @@ -460,7 +460,6 @@ func (kcm *MainKubeConfigManager) handleCmDelete(obj *v1.ConfigMap) error { delete(kcm.ModulesValuesChecksum, module) moduleConfigsUpdate[module] = utils.ModuleConfig{ ModuleName: module, - IsEnabled: true, Values: make(utils.Values), } } diff --git a/pkg/kube_config_manager/kube_config_manager_test.go b/pkg/kube_config_manager/kube_config_manager_test.go index 78690853..7d9a6c90 100644 --- a/pkg/kube_config_manager/kube_config_manager_test.go +++ b/pkg/kube_config_manager/kube_config_manager_test.go @@ -5,6 +5,7 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" "k8s.io/client-go/kubernetes" @@ -77,37 +78,40 @@ func (mockConfigMaps MockConfigMaps) Update(obj *v1.ConfigMap) (*v1.ConfigMap, e return nil, fmt.Errorf("no such resource '%s'", obj.Name) } -func TestInit(t *testing.T) { +func Test_Init(t *testing.T) { ConfigMapName = "addon-operator" + cmDataText := ` +global: | + project: tfprod + clusterName: main + clusterHostname: kube.flant.com + settings: + count: 2 + mysql: + user: myuser +nginxIngress: | + config: + hsts: true + setRealIPFrom: + - 1.1.1.1 + - 2.2.2.2 +nginxIngressEnabled: "true" +prometheus: | + adminPassword: qwerty + retentionDays: 20 + userPassword: qwerty +kubeLegoEnabled: "false" +` + cmData := map[string]string{} + err := yaml.Unmarshal([]byte(cmDataText), cmData) + assert.NoError(t, err) + mockConfigMapList = &v1.ConfigMapList{ Items: []v1.ConfigMap{ v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: ConfigMapName}, - Data: map[string]string{ - utils.GlobalValuesKey: ` -project: tfprod -clusterName: main -clusterHostname: kube.flant.com -settings: - count: 2 - mysql: - user: myuser`, - "nginxIngress": ` -config: - hsts: true - setRealIPFrom: - - 1.1.1.1 - - 2.2.2.2`, - "prometheus": ` -adminPassword: qwerty -estimatedNumberOfMetrics: 480000 -ingressHostname: prometheus.mysite.com -madisonAuthKey: 70cf58be013c93b5e7960716ea8538eb877808f88303c8a08f18f16582c81b61 -retentionDays: 20 -userPassword: qwerty`, - "kubeLego": "false", - }, + Data: cmData, }, }, } @@ -120,78 +124,68 @@ userPassword: qwerty`, } config := kcm.InitialConfig() - expectedData := utils.Values{ - "global": utils.Values{ - utils.GlobalValuesKey: map[string]interface{}{ - "project": "tfprod", - "clusterName": "main", - "clusterHostname": "kube.flant.com", - "settings": map[string]interface{}{ - "count": 2.0, - "mysql": map[string]interface{}{ - "user": "myuser", + expectations := map[string] struct { + isEnabled *bool + values utils.Values + }{ + "global": { + nil, + utils.Values{ + utils.GlobalValuesKey: map[string]interface{}{ + "project": "tfprod", + "clusterName": "main", + "clusterHostname": "kube.flant.com", + "settings": map[string]interface{}{ + "count": 2.0, + "mysql": map[string]interface{}{ + "user": "myuser", + }, }, }, }, }, - "nginx-ingress": utils.Values{ - utils.ModuleNameToValuesKey("nginx-ingress"): map[string]interface{}{ - "config": map[string]interface{}{ - "hsts": true, - "setRealIPFrom": []interface{}{ - "1.1.1.1", - "2.2.2.2", + "nginx-ingress": { + &utils.ModuleEnabled, + utils.Values{ + utils.ModuleNameToValuesKey("nginx-ingress"): map[string]interface{}{ + "config": map[string]interface{}{ + "hsts": true, + "setRealIPFrom": []interface{}{ + "1.1.1.1", + "2.2.2.2", + }, }, }, }, }, - "prometheus": utils.Values{ - utils.ModuleNameToValuesKey("prometheus"): map[string]interface{}{ - "adminPassword": "qwerty", - "estimatedNumberOfMetrics": 480000.0, - "ingressHostname": "prometheus.mysite.com", - "madisonAuthKey": "70cf58be013c93b5e7960716ea8538eb877808f88303c8a08f18f16582c81b61", - "retentionDays": 20.0, - "userPassword": "qwerty", + "prometheus": { + nil, + utils.Values{ + utils.ModuleNameToValuesKey("prometheus"): map[string]interface{}{ + "adminPassword": "qwerty", + "retentionDays": 20.0, + "userPassword": "qwerty", + }, }, }, + "kube-lego": { + &utils.ModuleDisabled, + utils.Values{}, + }, } - for key, data := range expectedData { - if key == "global" { - if !reflect.DeepEqual(data, config.Values) { - t.Errorf("Bad global values: expected %#v, got %#v", data, config.Values) - } - } else { - moduleName := key - moduleConfig, hasKey := config.ModuleConfigs[moduleName] - if !hasKey { - t.Errorf("Expected module %s values to be existing in config", moduleName) - } - if moduleConfig.ModuleName != moduleName { - t.Errorf("Expected %s module name, got %s", moduleName, moduleConfig.ModuleName) - } - if !moduleConfig.IsEnabled { - t.Errorf("Expected %s module to be enabled", moduleConfig.ModuleName) + for name, expect := range expectations { + t.Run(name, func(t *testing.T){ + if name == "global" { + assert.Equal(t, expect.values, config.Values) + } else { + // module + moduleConfig, hasConfig := config.ModuleConfigs[name] + assert.True(t, hasConfig) + assert.Equal(t, expect.isEnabled, moduleConfig.IsEnabled) + assert.Equal(t, expect.values, moduleConfig.Values) } - if !reflect.DeepEqual(data, moduleConfig.Values) { - t.Errorf("Bad %s module values: expected %+v, got %+v", moduleConfig.ModuleName, data, moduleConfig.Values) - } - } - } - - for moduleName, moduleConfig := range config.ModuleConfigs { - if _, hasKey := expectedData[moduleName]; hasKey { - continue - } - - if moduleConfig.ModuleName != moduleName { - t.Errorf("Expected %s module name in index, got %s", moduleName, moduleConfig.ModuleName) - } - - if moduleConfig.IsEnabled { - t.Errorf("Expected %s module to be disabled", moduleConfig.ModuleName) - } + }) } } @@ -239,7 +233,8 @@ func configDataShouldEqual(expectedValues utils.Values) error { return configRawDataShouldEqual(expectedDataRaw) } -func TestSetConfig(t *testing.T) { +// +func Test_SetConfig(t *testing.T) { mockConfigMapList = &v1.ConfigMapList{} kube.Kubernetes = &MockKubernetesClientset{} kcm := &MainKubeConfigManager{} diff --git a/pkg/kube_config_manager/module_kube_config.go b/pkg/kube_config_manager/module_kube_config.go index dab02df0..8f9b8fdd 100644 --- a/pkg/kube_config_manager/module_kube_config.go +++ b/pkg/kube_config_manager/module_kube_config.go @@ -2,6 +2,8 @@ package kube_config_manager import ( "fmt" + "strings" + "github.com/flant/addon-operator/pkg/utils" utils_checksum "github.com/flant/shell-operator/pkg/utils/checksum" "github.com/romana/rlog" @@ -15,13 +17,21 @@ func GetModulesNamesFromConfigData(configData map[string]string) map[string]bool res := make(map[string]bool, 0) for key := range configData { - if key != utils.GlobalValuesKey { - if utils.ModuleNameToValuesKey(utils.ModuleNameFromValuesKey(key)) != key { - rlog.Warnf("Bad module name '%s': should be camelCased module name: ignoring data", key) - continue - } - res[utils.ModuleNameFromValuesKey(key)] = true + if key == utils.GlobalValuesKey { + continue + } + + if strings.HasSuffix(key, "Enabled") { + key = strings.TrimSuffix(key, "Enabled") } + + modName := utils.ModuleNameFromValuesKey(key) + + if utils.ModuleNameToValuesKey(modName) != key { + rlog.Errorf("Bad module name '%s': should be camelCased module name: ignoring data", key) + continue + } + res[modName] = true } return res @@ -33,8 +43,11 @@ type ModuleKubeConfig struct { ConfigData map[string]string } +// TODO make a method of KubeConfig func GetModuleKubeConfigFromValues(moduleName string, values utils.Values) *ModuleKubeConfig { - moduleValues, hasKey := values[utils.ModuleNameToValuesKey(moduleName)] + mc := utils.NewModuleConfig(moduleName) + + moduleValues, hasKey := values[mc.ModuleConfigKey] if !hasKey { return nil } @@ -44,53 +57,31 @@ func GetModuleKubeConfigFromValues(moduleName string, values utils.Values) *Modu panic(fmt.Sprintf("cannot dump yaml for module '%s' kube config: %s\nfailed values data: %#v", moduleName, err, moduleValues)) } + return &ModuleKubeConfig{ ModuleConfig: utils.ModuleConfig{ ModuleName: moduleName, - IsEnabled: true, - Values: utils.Values{utils.ModuleNameToValuesKey(moduleName): moduleValues}, + Values: utils.Values{mc.ModuleConfigKey: moduleValues}, }, - ConfigData: map[string]string{utils.ModuleNameToValuesKey(moduleName): string(yamlData)}, + ConfigData: map[string]string{mc.ModuleConfigKey: string(yamlData)}, Checksum: utils_checksum.CalculateChecksum(string(yamlData)), } } -func ModuleKubeConfigMustExist(res *ModuleKubeConfig, err error) (*ModuleKubeConfig, error) { +// TODO make a method of KubeConfig +// ExtractModuleKubeConfig returns ModuleKubeConfig with values loaded from ConfigMap +func ExtractModuleKubeConfig(moduleName string, configData map[string]string) (*ModuleKubeConfig, error) { + moduleConfig, err := utils.NewModuleConfig(moduleName).FromKeyYamls(configData) if err != nil { - return res, err + return nil, fmt.Errorf("'%s' ConfigMap bad yaml at key '%s': %s", ConfigMapName, utils.ModuleNameToValuesKey(moduleName), err) } - if res == nil { + // NOTE this should never happen because of GetModulesNamesFromConfigData + if moduleConfig == nil { panic("module kube config must exist!") } - return res, err -} - -func GetModuleKubeConfigFromConfigData(moduleName string, configData map[string]string) (*ModuleKubeConfig, error) { - yamlData, hasKey := configData[utils.ModuleNameToValuesKey(moduleName)] - if !hasKey { - return nil, nil - } - - moduleConfig, err := NewModuleConfig(moduleName, yamlData) - if err != nil { - return nil, fmt.Errorf("'%s' ConfigMap bad yaml at key '%s': %s", ConfigMapName, utils.ModuleNameToValuesKey(moduleName), err) - } return &ModuleKubeConfig{ ModuleConfig: *moduleConfig, - Checksum: utils_checksum.CalculateChecksum(yamlData), + Checksum: moduleConfig.Checksum(), }, nil } - -func NewModuleConfig(moduleName string, moduleYamlData string) (*utils.ModuleConfig, error) { - var valuesAtModuleKey interface{} - - err := yaml.Unmarshal([]byte(moduleYamlData), &valuesAtModuleKey) - if err != nil { - return nil, err - } - - data := map[interface{}]interface{}{utils.ModuleNameToValuesKey(moduleName): valuesAtModuleKey} - - return utils.NewModuleConfig(moduleName).WithValues(data) -} diff --git a/pkg/module_manager/hook.go b/pkg/module_manager/hook.go index 92489928..bb837ba4 100644 --- a/pkg/module_manager/hook.go +++ b/pkg/module_manager/hook.go @@ -720,7 +720,7 @@ func (mm *MainModuleManager) initHooks(hooksDir string, addHookFn func(hookPath } // retrieve a list of executable files in hooksDir sorted by filename - hooksRelativePaths, _, err := utils.FindExecutableFilesInPath(hooksDir) + hooksRelativePaths, _, err := utils.FindExecutableFilesInPath(hooksDir) if err != nil { return err } diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index aa9da049..ca44041d 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -545,8 +545,8 @@ func (m *Module) loadStaticValues() error { valuesYamlPath := filepath.Join(m.Path, "values.yaml") if _, err := os.Stat(valuesYamlPath); os.IsNotExist(err) { - m.StaticConfig = utils.NewModuleConfig(m.Name).WithEnabled(true) - rlog.Debugf("module %s is enabled: no values.yaml exists", m.Name) + m.StaticConfig = utils.NewModuleConfig(m.Name) + rlog.Debugf("module %s is static disabled: no values.yaml exists", m.Name) return nil } diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index 7d67a54b..a4451213 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -301,7 +301,7 @@ func (mm *MainModuleManager) handleNewKubeModuleConfigs(moduleConfigs kube_confi updateAfterRemoval := make(map[string]bool, 0) for moduleName, module := range mm.allModulesByName { _, hasKubeConfig := moduleConfigs[moduleName] - if !hasKubeConfig && module.StaticConfig.IsEnabled { + if !hasKubeConfig && mergeEnabled(module.StaticConfig.IsEnabled) { if _, hasValues := mm.kubeModulesConfigValues[moduleName]; hasValues { updateAfterRemoval[moduleName] = true } @@ -377,7 +377,9 @@ func (mm *MainModuleManager) calculateEnabledModulesByConfig(moduleConfigs kube_ for moduleName, module := range mm.allModulesByName { kubeConfig, hasKubeConfig := moduleConfigs[moduleName] if hasKubeConfig { - if kubeConfig.IsEnabled { + isEnabled := mergeEnabled(module.StaticConfig.IsEnabled, kubeConfig.IsEnabled) + + if isEnabled { enabled = append(enabled, moduleName) values[moduleName] = kubeConfig.Values } @@ -387,7 +389,8 @@ func (mm *MainModuleManager) calculateEnabledModulesByConfig(moduleConfigs kube_ kubeConfig.IsEnabled, kubeConfig.IsUpdated) } else { - if module.StaticConfig.IsEnabled { + isEnabled := mergeEnabled(module.StaticConfig.IsEnabled) + if isEnabled { enabled = append(enabled, moduleName) } rlog.Debugf("Module %s: static enabled %v, no kubeConfig", module.Name, module.StaticConfig.IsEnabled) @@ -548,7 +551,7 @@ func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err er // modules finally enabled with enable script // no need to refresh mm.enabledModulesByConfig because - // it is updated before in Init or applyKubeUpdate + // it is updated before in Init or in applyKubeUpdate rlog.Infof("DISCOVER run `enabled` for %s", mm.enabledModulesByConfig) enabledModules, err := mm.determineEnableStateWithScript(mm.enabledModulesByConfig) rlog.Infof("DISCOVER enabled modules %s", enabledModules) @@ -775,3 +778,20 @@ func (mm *MainModuleManager) WithKubeConfigManager(kubeConfigManager kube_config mm.kubeConfigManager = kubeConfigManager return mm } + +// mergeEnabled merges enabled flags. Enabled flag can be nil. +// +// If all flags are nil, then false is returned — module is disabled by default. +// +func mergeEnabled(enabledFlags ... *bool) bool { + result := false + for _, enabled := range enabledFlags { + if enabled == nil { + continue + } else { + result = *enabled + } + } + + return result +} diff --git a/pkg/module_manager/module_manager_internal_test.go b/pkg/module_manager/module_manager_internal_test.go index a974ca67..20efddd0 100644 --- a/pkg/module_manager/module_manager_internal_test.go +++ b/pkg/module_manager/module_manager_internal_test.go @@ -3,13 +3,14 @@ package module_manager import ( "fmt" "io/ioutil" + "os" "path/filepath" "reflect" "runtime" "testing" - "github.com/stretchr/testify/assert" "github.com/romana/rlog" + "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -113,7 +114,7 @@ func TestMainModuleManager_modulesStaticValues(t *testing.T) { } } -func TestMainModuleManager_GetModule2(t *testing.T) { +func Test_MainModuleManager_GetModule2(t *testing.T) { mm := NewMainModuleManager() runInitModulesIndex(t, mm, "test_get_module") @@ -126,8 +127,11 @@ func TestMainModuleManager_GetModule2(t *testing.T) { StaticConfig: &utils.ModuleConfig{ ModuleName: "module", Values: utils.Values{}, - IsEnabled: true, + IsEnabled: nil, IsUpdated: false, + ModuleConfigKey: "module", + ModuleEnabledKey: "moduleEnabled", + RawConfig: []string{}, }, moduleManager: mm, }, @@ -148,6 +152,9 @@ func TestMainModuleManager_GetModule2(t *testing.T) { } func TestMainModuleManager_EnabledModules(t *testing.T) { + os.Setenv("RLOG_LOG_LEVEL", "DEBUG") + rlog.UpdateEnv() + helm.Client = &MockHelmClient{} mm := NewMainModuleManager() @@ -155,7 +162,6 @@ func TestMainModuleManager_EnabledModules(t *testing.T) { expectedModules := []string{ "module-c", - "module-a", "module-b", } @@ -187,7 +193,7 @@ func TestMainModuleManager_GetModuleHook2(t *testing.T) { orderByBindings[AfterDeleteHelm], } - moduleHook := mm.newModuleHook(name, filepath.Join(mm.ModulesDir, name), config) + moduleHook := NewModuleHook(name, filepath.Join(mm.ModulesDir, name), config, mm) var err error if moduleHook.Module, err = mm.GetModule(moduleName); err != nil { @@ -679,7 +685,7 @@ func TestMainModuleManager_GetGlobalHook2(t *testing.T) { orderByBindings[AfterAll], } - globalHook := mm.newGlobalHook(name, filepath.Join(mm.GlobalHooksDir, name), config) + globalHook := NewGlobalHook(name, filepath.Join(mm.GlobalHooksDir, name), config, mm) globalHook.Bindings = bindings for k, v := range orderByBindings { @@ -854,7 +860,6 @@ func TestMainModuleManager_GetGlobalHooksInOrder2(t *testing.T) { func TestMainModuleManager_RunGlobalHook(t *testing.T) { helm.Client = &MockHelmClient{} mm := NewMainModuleManager() - mm.WithHelmClient(&MockHelmClient{}) mm.WithKubeConfigManager(MockKubeConfigManager{}) runInitGlobalHooks(t, mm, "test_run_global_hook") diff --git a/pkg/module_manager/module_manager_test.go b/pkg/module_manager/module_manager_test.go index 8b97a9aa..842e36e7 100644 --- a/pkg/module_manager/module_manager_test.go +++ b/pkg/module_manager/module_manager_test.go @@ -34,7 +34,7 @@ func TestMainModuleManager_GetModule(t *testing.T) { func TestMainModuleManager_GetModuleHook(t *testing.T) { mm := NewMainModuleManager() - expectedModuleHook := &ModuleHook{Hook: &Hook{Name: "hook"}} + expectedModuleHook := &ModuleHook{CommonHook: &CommonHook{Name: "hook"}} mm.modulesHooksOrderByName["module"] = map[BindingType][]*ModuleHook{ OnStartup: { @@ -53,7 +53,7 @@ func TestMainModuleManager_GetModuleHook(t *testing.T) { _, err = mm.GetModuleHook("non-exist") if err == nil { t.Error("Expected error!") - } else if !strings.HasPrefix(err.Error(), "module hook 'non-exist' not found") { + } else if !strings.HasPrefix(err.Error(), "module hook 'non-exist' is not found") { t.Errorf("Got unexpected error: %s", err) } } @@ -66,19 +66,19 @@ func TestMainModuleManager_GetModuleHooksInOrder(t *testing.T) { "module": { BeforeHelm: []*ModuleHook{ { - Hook: &Hook{ + CommonHook: &CommonHook{ Name: "hook-1", OrderByBinding: map[BindingType]float64{BeforeHelm: 3}, }, }, { - Hook: &Hook{ + CommonHook: &CommonHook{ Name: "hook-2", OrderByBinding: map[BindingType]float64{BeforeHelm: 1}, }, }, { - Hook: &Hook{ + CommonHook: &CommonHook{ Name: "hook-3", OrderByBinding: map[BindingType]float64{BeforeHelm: 2}, }, @@ -144,7 +144,7 @@ func TestMainModuleManager_GetModuleHooksInOrder(t *testing.T) { func TestMainModuleManager_GetGlobalHook(t *testing.T) { mm := NewMainModuleManager() - expectedGlobalHook := &GlobalHook{Hook: &Hook{Name: "hook"}} + expectedGlobalHook := &GlobalHook{CommonHook: &CommonHook{Name: "hook"}} mm.globalHooksByName["hook"] = expectedGlobalHook moduleHook, err := mm.GetGlobalHook("hook") @@ -169,19 +169,19 @@ func TestMainModuleManager_GetGlobalHooksInOrder(t *testing.T) { mm.globalHooksOrder = map[BindingType][]*GlobalHook{ BeforeAll: { { - Hook: &Hook{ + CommonHook: &CommonHook{ Name: "hook-1", OrderByBinding: map[BindingType]float64{BeforeAll: 3}, }, }, { - Hook: &Hook{ + CommonHook: &CommonHook{ Name: "hook-2", OrderByBinding: map[BindingType]float64{BeforeAll: 1}, }, }, { - Hook: &Hook{ + CommonHook: &CommonHook{ Name: "hook-3", OrderByBinding: map[BindingType]float64{BeforeAll: 2}, }, diff --git a/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/000-module-c/values.yaml b/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/000-module-c/values.yaml new file mode 100644 index 00000000..8de59237 --- /dev/null +++ b/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/000-module-c/values.yaml @@ -0,0 +1 @@ +moduleCEnabled: true diff --git a/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/200-module-b/values.yaml b/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/200-module-b/values.yaml index f01c5155..291f7c23 100644 --- a/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/200-module-b/values.yaml +++ b/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/200-module-b/values.yaml @@ -1,2 +1,3 @@ moduleB: - param: value \ No newline at end of file + param: value +moduleBEnabled: true diff --git a/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/300-module-disabled/values.yaml b/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/300-module-disabled/values.yaml index 20286efb..e72df9ba 100755 --- a/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/300-module-disabled/values.yaml +++ b/pkg/module_manager/testdata/init_modules_index/test_get_module_names_in_order/modules/300-module-disabled/values.yaml @@ -1 +1 @@ -moduleDisabled: false +moduleDisabledEnabled: false diff --git a/pkg/utils/module_config.go b/pkg/utils/module_config.go index a555df0c..8bd6c9f7 100644 --- a/pkg/utils/module_config.go +++ b/pkg/utils/module_config.go @@ -3,14 +3,21 @@ package utils import ( "fmt" + utils_checksum "github.com/flant/shell-operator/pkg/utils/checksum" "github.com/go-yaml/yaml" ) +var ModuleEnabled = true +var ModuleDisabled = false + type ModuleConfig struct { ModuleName string - IsEnabled bool + IsEnabled *bool Values Values IsUpdated bool + ModuleConfigKey string + ModuleEnabledKey string + RawConfig []string } func (mc ModuleConfig) String() string { @@ -20,13 +27,20 @@ func (mc ModuleConfig) String() string { func NewModuleConfig(moduleName string) *ModuleConfig { return &ModuleConfig{ ModuleName: moduleName, - IsEnabled: true, + IsEnabled: nil, Values: make(Values), + ModuleConfigKey: ModuleNameToValuesKey(moduleName), + ModuleEnabledKey: ModuleNameToValuesKey(moduleName) + "Enabled", + RawConfig: make([]string, 0), } } func (mc *ModuleConfig) WithEnabled(v bool) *ModuleConfig { - mc.IsEnabled = v + if v { + mc.IsEnabled = &ModuleEnabled + } else { + mc.IsEnabled = &ModuleDisabled + } return mc } @@ -35,47 +49,118 @@ func (mc *ModuleConfig) WithUpdated(v bool) *ModuleConfig { return mc } -// WithValues loads module config from a map. +func (mc *ModuleConfig) WithValues(values Values) *ModuleConfig { + mc.Values = values + return mc +} + +// LoadValues loads module config from a map. // // Values for module in `values` map are addressed by a key. // This key should be produced with ModuleNameToValuesKey. -// -// A module is enabled if a key doesn't exist in values. -func (mc *ModuleConfig) WithValues(values map[interface{}]interface{}) (*ModuleConfig, error) { - moduleValuesKey := ModuleNameToValuesKey(mc.ModuleName) +func (mc *ModuleConfig) LoadValues(values map[interface{}]interface{}) (*ModuleConfig, error) { - if moduleValuesData, hasModuleData := values[moduleValuesKey]; hasModuleData { + if moduleValuesData, hasModuleData := values[mc.ModuleConfigKey]; hasModuleData { switch v := moduleValuesData.(type) { - case bool: - mc.IsEnabled = v case map[interface{}]interface{}, []interface{}: - data := map[interface{}]interface{}{moduleValuesKey: v} + data := map[interface{}]interface{}{mc.ModuleConfigKey: v} values, err := NewValues(data) if err != nil { return nil, err } - mc.IsEnabled = true mc.Values = values default: - return nil, fmt.Errorf("Module config should be bool, array or map. Got: %#v", moduleValuesData) + return nil, fmt.Errorf("Module config should be array or map. Got: %#v", moduleValuesData) + } + } + + if moduleEnabled, hasModuleEnabled := values[mc.ModuleEnabledKey]; hasModuleEnabled { + switch v := moduleEnabled.(type) { + case bool: + mc.WithEnabled(v) + default: + return nil, fmt.Errorf("Module enabled value should be bool. Got: %#v", moduleEnabled) } - } else { - mc.IsEnabled = true } return mc, nil } // FromYaml loads module config from a yaml string. +// +// Example: +// +// simpleModule: +// param1: 10 +// param2: 120 +// simpleModuleEnabled: true func (mc *ModuleConfig) FromYaml(yamlString []byte) (*ModuleConfig, error) { var values map[interface{}]interface{} err := yaml.Unmarshal(yamlString, &values) if err != nil { - return nil, fmt.Errorf("Module %s has errors in yaml: %s\n%s", mc.ModuleName, err, string(yamlString)) + return nil, fmt.Errorf("unmarshal module '%s' yaml config: %s\n%s", mc.ModuleName, err, string(yamlString)) + } + + mc.RawConfig = []string{string(yamlString)} + + return mc.LoadValues(values) +} + +// FromKeyYamls loads module config from a structure with string keys and yaml string values (ConfigMap) +// +// Example: +// +// simpleModule: | +// param1: 10 +// param2: 120 +// simpleModuleEnabled: "true" +func (mc *ModuleConfig) FromKeyYamls(configData map[string]string) (*ModuleConfig, error) { + // map with moduleNameKey and moduleEnabled keys + moduleConfigData := map[interface{}]interface{}{} + + // if there is data for module, unmarshal it and put into moduleConfigData + valuesYaml, hasKey := configData[mc.ModuleConfigKey] + if hasKey { + var values interface{} + + err := yaml.Unmarshal([]byte(valuesYaml), &values) + if err != nil { + return nil, fmt.Errorf("unmarshal yaml data in a module config key '%s': %v", mc.ModuleConfigKey, err) + } + + moduleConfigData[mc.ModuleConfigKey] = values + + mc.RawConfig = append(mc.RawConfig, valuesYaml) } - return mc.WithValues(values) + // if there is enabled key, treat it as boolean + enabledString, hasKey := configData[mc.ModuleEnabledKey] + if hasKey { + var enabled bool + + if enabledString == "true" { + enabled = true + } else if enabledString == "false" { + enabled = false + } else { + return nil, fmt.Errorf("module enabled key '%s' should have a boolean value, got '%v'", mc.ModuleEnabledKey, enabledString) + } + + moduleConfigData[mc.ModuleEnabledKey] = enabled + + mc.RawConfig = append(mc.RawConfig, enabledString) + } + + if len(moduleConfigData) == 0 { + return mc, nil + } + + return mc.LoadValues(moduleConfigData) +} + +func (mc *ModuleConfig) Checksum() string { + return utils_checksum.CalculateChecksum(mc.RawConfig...) } diff --git a/pkg/utils/module_config_test.go b/pkg/utils/module_config_test.go new file mode 100644 index 00000000..c90ab7d6 --- /dev/null +++ b/pkg/utils/module_config_test.go @@ -0,0 +1,161 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Test_FromYaml creates ModuleConfig objects from different input yaml strings +func Test_FromYaml(t *testing.T) { + var config *ModuleConfig + var err error + + tests := []struct { + name string + yaml string + assertFn func() + }{ + { + "simple config", + ` +testModule: + poaram1: "1234" +`, + func() { + assert.NoError(t, err) + assert.NotNil(t, config) + assert.Nil(t, config.IsEnabled) + }, + }, + { + "bad type", + `testModule: 1234`, + func() { + assert.Nil(t, config) + assert.Error(t, err) + assert.Containsf(t, err.Error(), "Module config should be array or map", "got unexpected error") + }, + }, + { + "disabled module", + `testModuleEnabled: false`, + func() { + assert.NoError(t, err) + assert.NotNil(t, config) + assert.Empty(t, config.Values) + assert.False(t, *config.IsEnabled) + }, + }, + { + "enabled module", + `testModuleEnabled: true`, + func() { + assert.NoError(t, err) + assert.NotNil(t, config) + assert.Empty(t, config.Values) + assert.True(t, *config.IsEnabled) + }, + }, + { + "full module config", + ` +testModule: + hello: world + 4: "123" + 5: 5 + aaa: + numbers: + - one + - two + - three +testModuleEnabled: true +`, + func() { + assert.NoError(t, err) + assert.NotNil(t, config) + assert.True(t, *config.IsEnabled) + assert.Contains(t, config.Values, "testModule") + modVals := config.Values["testModule"] + //assert.IsType(t, interface{}{}, modVals) + + modValsMap, ok := modVals.(map[string]interface{}) + assert.True(t, ok) + assert.Equal(t, "world", modValsMap["hello"]) + + assert.Contains(t, modValsMap, "4") + assert.Equal(t, "123", modValsMap["4"]) + assert.Contains(t, modVals, "5") + assert.Equal(t, 5.0, modValsMap["5"]) + + assert.Contains(t, modVals, "aaa") + aaa, ok := modValsMap["aaa"].(map[string]interface{}) + assert.True(t, ok) + + assert.Contains(t, aaa, "numbers") + noArray, ok := aaa["numbers"].([]interface{}) + assert.True(t, ok) + + assert.Len(t, noArray, 3) + + }, + }, + { + "array config", + ` +testModule: + - a: 1 + - b: 2 +`, + func() { + assert.NoError(t, err) + assert.NotNil(t, config) + assert.Contains(t, config.Values, "testModule") + + vals, ok := config.Values["testModule"].([]interface{}) + assert.True(t, ok, "testModule should be []interface{}") + assert.Len(t, vals, 2) + + vals0, ok := vals[0].(map[string]interface{}) + assert.True(t, ok) + assert.Equal(t, vals0["a"], 1.0) + vals1, ok := vals[1].(map[string]interface{}) + assert.True(t, ok) + assert.Equal(t, vals1["b"], 2.0) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config = nil + err = nil + config, err = NewModuleConfig("test-module").FromYaml([]byte(test.yaml)) + test.assertFn() + }) + } + +} + + +func Test_LoadValues(t *testing.T) { + var config *ModuleConfig + var err error + + inputData := map[interface{}]interface{}{ + "testModule": map[interface{}]interface{}{ + "hello": "world", 4: "123", 5: 5, + "aaa": map[interface{}]interface{}{"no": []interface{}{"one", "two", "three"}}, + }, + } + expectedData := Values{ + "testModule": map[string]interface{}{ + "hello": "world", "4": "123", "5": 5.0, + "aaa": map[string]interface{}{"no": []interface{}{"one", "two", "three"}}, + }, + } + + config, err = NewModuleConfig("test-module").LoadValues(inputData) + assert.NoError(t, err) + assert.NotNil(t, config) + assert.Equal(t, expectedData, config.Values) +} diff --git a/pkg/utils/values.go b/pkg/utils/values.go index 6ff11b17..921841db 100644 --- a/pkg/utils/values.go +++ b/pkg/utils/values.go @@ -268,3 +268,54 @@ func DumpValuesYaml(values Values) ([]byte, error) { func DumpValuesJson(values Values) ([]byte, error) { return json.Marshal(values) } + +type ValuesLoader interface { + Read() (Values, error) +} + +type ValuesDumper interface { + Write(values Values) error +} + +// Load values by specific key from loader +func Load(key string, loader ValuesLoader) (Values, error) { + return nil, nil +} + +// LoadAll loads values from all keys from loader +func LoadAll(loader ValuesLoader) (Values, error) { + return nil, nil +} + +func Dump(values Values, dumper ValuesDumper) error { + return nil +} + +type ValuesDumperToJsonFile struct { + FileName string +} + +func NewDumperToJsonFile(path string) ValuesDumper { + return &ValuesDumperToJsonFile{ + FileName: path, + } +} + +func (*ValuesDumperToJsonFile) Write(values Values) error { + return fmt.Errorf("implement Write in ValuesDumperToJsonFile") +} + +type ValuesLoaderFromJsonFile struct { + FileName string +} + +func NewLoaderFromJsonFile(path string) ValuesLoader { + return &ValuesLoaderFromJsonFile{ + FileName: path, + } +} + +func (*ValuesLoaderFromJsonFile) Read() (Values, error) { + return nil, fmt.Errorf("implement Read methoid") +} + diff --git a/pkg/utils/values_test.go b/pkg/utils/values_test.go index 293dfdb5..2d8344ec 100644 --- a/pkg/utils/values_test.go +++ b/pkg/utils/values_test.go @@ -3,121 +3,9 @@ package utils import ( "fmt" "reflect" - "strings" "testing" ) -func TestModuleConfig(t *testing.T) { - var config *ModuleConfig - var err error - - config, err = NewModuleConfig("test-module").WithValues(map[interface{}]interface{}{"testModule": 1234}) - if err == nil { - t.Errorf("Expected error, got ModuleConfig: %v", config) - } else if !strings.HasPrefix(err.Error(), "Module config should be bool, array or map") { - t.Errorf("Got unexpected error: %s", err) - } - - config, err = NewModuleConfig("test-module").WithValues(map[interface{}]interface{}{"testModule": false}) - if err != nil { - t.Error(err) - } - if config.IsEnabled { - t.Errorf("Expected module to be disabled, got: %v", config) - } - - config, err = NewModuleConfig("test-module").WithValues(map[interface{}]interface{}{"testModule": true}) - if err != nil { - t.Error(err) - } - if !config.IsEnabled { - t.Errorf("Expected module to be enabled") - } - - inputData := map[interface{}]interface{}{ - "testModule": map[interface{}]interface{}{ - "hello": "world", 4: "123", 5: 5, - "aaa": map[interface{}]interface{}{"no": []interface{}{"one", "two", "three"}}, - }, - } - expectedData := Values{ - "testModule": map[string]interface{}{ - "hello": "world", "4": "123", "5": 5.0, - "aaa": map[string]interface{}{"no": []interface{}{"one", "two", "three"}}, - }, - } - - config, err = NewModuleConfig("test-module").WithValues(inputData) - if err != nil { - t.Error(err) - } - if !config.IsEnabled { - t.Errorf("Expected module to be enabled") - } - - if !reflect.DeepEqual(config.Values, expectedData) { - t.Errorf("Got unexpected config values: %+v", config.Values) - } -} - -func TestNewModuleConfigByValuesYamlData(t *testing.T) { - configStr := ` -testModule: - a: 1 - b: 2 -` - expectedData := Values{ - "testModule": map[string]interface{}{ - "a": 1.0, "b": 2.0, - }, - } - config, err := NewModuleConfig("test-module").FromYaml([]byte(configStr)) - if err != nil { - t.Error(err) - } - if !config.IsEnabled { - t.Errorf("Expected module to be enabled") - } - if !reflect.DeepEqual(config.Values, expectedData) { - t.Errorf("Got unexpected config values: %+v", config.Values) - } - - config, err = NewModuleConfig("test-module").FromYaml([]byte("testModule: false\n")) - if err != nil { - t.Error(err) - } - if config.IsEnabled { - t.Errorf("Expected module to be disabled") - } - - _, err = NewModuleConfig("test-module").FromYaml([]byte("testModule: falsee\n")) - if !strings.HasPrefix(err.Error(), "Module config should be bool, array or map") { - t.Errorf("Got unexpected error: %s", err.Error()) - } - - configStr = ` -testModule: - - a: 1 - - b: 2 -` - expectedData = Values{ - "testModule": []interface{}{ - map[string]interface{}{"a": 1.0}, - map[string]interface{}{"b": 2.0}, - }, - } - config, err = NewModuleConfig("test-module").FromYaml([]byte(configStr)) - if err != nil { - t.Error(err) - } - if !config.IsEnabled { - t.Errorf("Expected module to be enabled") - } - if !reflect.DeepEqual(config.Values, expectedData) { - t.Errorf("Got unexpected config values: %+v", config.Values) - } - -} func TestMergeValues(t *testing.T) { expectations := []struct {