From cd38f5ec95c5cc870cffb060050a9baaf63073bd Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Thu, 9 Dec 2021 13:52:54 +0000 Subject: [PATCH 01/12] (GH-35) Add validate.yml check to `prm exec` --- cmd/exec/exec.go | 37 +++++++++++--------- go.mod | 1 + pkg/prm/docker.go | 1 - pkg/prm/exec.go | 27 ++++++++------- pkg/prm/prm.go | 79 ++++++++++++++++++++++++++++++++++-------- pkg/prm/tool_groups.go | 23 ++++++++++++ 6 files changed, 123 insertions(+), 45 deletions(-) create mode 100644 pkg/prm/tool_groups.go diff --git a/cmd/exec/exec.go b/cmd/exec/exec.go index 9d28fff..3890898 100644 --- a/cmd/exec/exec.go +++ b/cmd/exec/exec.go @@ -63,6 +63,10 @@ func CreateCommand(parent *prm.Prm) *cobra.Command { err = viper.BindPFlag("toolpath", tmp.Flags().Lookup("toolpath")) cobra.CheckErr(err) + tmp.Flags().StringVar(&prmApi.CodeDir, "codedir", "", "location of code to execute against") + err = viper.BindPFlag("codedir", tmp.Flags().Lookup("codedir")) + cobra.CheckErr(err) + return tmp } @@ -75,11 +79,6 @@ func preExecute(cmd *cobra.Command, args []string) error { } func validateArgCount(cmd *cobra.Command, args []string) error { - // show available tools if user runs `prm exec` - if len(args) == 0 && !listTools { - listTools = true - } - if len(args) >= 1 { if len(strings.Split(args[0], "/")) != 2 { return fmt.Errorf("Selected tool must be in AUTHOR/ID format") @@ -139,21 +138,27 @@ func execute(cmd *cobra.Command, args []string) error { return nil } - matchingTools := prmApi.FilterFiles(cachedTools, func(f prm.ToolConfig) bool { - return fmt.Sprintf("%s/%s", f.Plugin.Author, f.Plugin.Id) == selectedTool - }) - - if len(matchingTools) == 1 { - matchingTool := matchingTools[0] - selectedToolDirPath = filepath.Join(localToolPath, matchingTool.Plugin.Author, matchingTool.Plugin.Id, matchingTool.Plugin.Version) - tool, err := prmApi.Get(selectedToolDirPath) + if selectedTool != "" { + // get the tool from the cache + cachedTool, ok := prmApi.IsToolAvailable(selectedTool) + if !ok { + return fmt.Errorf("Tool %s not found in cache", selectedTool) + } + // execute! + err := prmApi.Exec(cachedTool, args[1:]) + if err != nil { + return err + } + } else { + // No tool specified, so check if their code contains a validate.yml, which returns the list of tools + // Their code is expected to be in the directory where the executable is run from + toolList, err := prmApi.CheckLocalConfig() if err != nil { return err } - return prmApi.Exec(&tool, args[1:]) + log.Info().Msgf("Found tools: %v ", toolList) - } else { - return fmt.Errorf("Couldn't find an installed tool that matches '%s'", selectedTool) } + } diff --git a/go.mod b/go.mod index 7ccdcee..dbe1893 100644 --- a/go.mod +++ b/go.mod @@ -32,4 +32,5 @@ require ( google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index e7d32c6..ad68d55 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -34,7 +34,6 @@ func (*Docker) Exec(tool *Tool, args []string) (ToolExitCode, error) { return FAILURE, nil } -// nolint:unused func (d *Docker) initClient() (err error) { if d.Client == nil { cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv) diff --git a/pkg/prm/exec.go b/pkg/prm/exec.go index 943d741..7f90380 100644 --- a/pkg/prm/exec.go +++ b/pkg/prm/exec.go @@ -1,8 +1,6 @@ //nolint:structcheck,unused package prm -import "fmt" - type ExecExitCode int64 const ( @@ -12,22 +10,25 @@ const ( ) // Executes a tool with the given arguments, against the codeDir. -func (*Prm) Exec(tool *Tool, args []string) error { +func (p *Prm) Exec(tool *Tool, args []string) error { - if tool.Cfg.Gem != nil { - fmt.Printf("GEM") - } + var toolList []string - if tool.Cfg.Puppet != nil { - fmt.Printf("PUPPET") - } + // perform a check for validate.yml + + // flatten the tool list + p.flattenToolList(&toolList) + + var backend BackendI - if tool.Cfg.Binary != nil { - fmt.Printf("BINARY") + switch RunningConfig.Backend { + case DOCKER: + backend = &Docker{} + default: } - if tool.Cfg.Container != nil { - fmt.Printf("CONTAINER") + for _, toolName := range toolList { + tool, err := backend.GetTool(toolName, RunningConfig) } return nil diff --git a/pkg/prm/prm.go b/pkg/prm/prm.go index 686f987..79071aa 100644 --- a/pkg/prm/prm.go +++ b/pkg/prm/prm.go @@ -4,9 +4,7 @@ package prm import ( "bytes" "fmt" - "os" "path/filepath" - "reflect" "sort" "strings" @@ -17,6 +15,7 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/afero" "github.com/spf13/viper" + "gopkg.in/yaml.v3" ) const ( @@ -41,29 +40,79 @@ type PuppetVersion struct { version semver.Version } -// Given a list of tool names, check if these are groups, and return -// an expanded list containing all the toolNames +type ValidateYmlContent struct { + Tools []string `yaml:"tools"` +} + +// checkGroups takes a slice of tool names and iterates through each +// checking against a map of toolGroups. If a toolGroup name is found +// the toolGroup is expanded and the list of tools is updated. func (*Prm) checkGroups(tools []string) []string { - // TODO - return []string{} + for index, toolName := range tools { + if toolGroup, ok := ToolGroups[toolName]; ok { + // remove the group from the list + tools = append(tools[:index], tools[index+1:]...) + // add the expanded toolgroup to the list + tools = append(tools, toolGroup...) + } + } + + // remove duplicates + allKeys := make(map[string]bool) + clean := []string{} + for _, item := range tools { + if _, value := allKeys[item]; !value { + allKeys[item] = true + clean = append(clean, item) + } + } + + return clean } -// Look within codeDir for a "validate.yml" containing -// a list of tools and/or tool groups that should be run against -// code within codeDir. -func (*Prm) checkLocalConfig() []string { - // TODO - return []string{} +// Check if the code being executed against contains a yalidate.yml. Read in the +// list of tool names from validate.yml into a list. Pass the list of +// tool names to flattenToolList to expand out any groups. Then return +// the complete list. +func (p *Prm) CheckLocalConfig() ([]string, error) { + // check if validate.yml exits in the codeDir + validateFile := filepath.Join(p.CodeDir, "validate.yml") + if _, err := p.AFS.Stat(validateFile); err != nil { + log.Error().Msgf("validate.yml not found in %s", p.CodeDir) + return []string{}, err + } + + // read in validate.yml + contents, err := p.AFS.ReadFile(validateFile) + if err != nil { + log.Error().Msgf("Error reading validate.yml: %s", err) + return []string{}, err + } + + // parse validate.yml to our temporary struct + var userList ValidateYmlContent + err = yaml.Unmarshal(contents, &userList) + if err != nil { + log.Error().Msgf("validate.yml is not formated correctly: %s", err) + return []string{}, err + } + + return p.checkGroups(userList.Tools), nil } // Check to see if the requested tool can be found installed. // If installed read the tool configuration and return -func (*Prm) isToolAvailable(tool string) (Tool, bool) { - return Tool{}, false +func (p *Prm) IsToolAvailable(tool string) (*Tool, bool) { + + if p.Cache[tool] != nil { + return p.Cache[tool], true + } + + return nil, false } // Check to see if the tool is ready to execute -func (*Prm) isToolReady(tool *Tool) bool { +func (*Prm) IsToolReady(tool *Tool) bool { return false } diff --git a/pkg/prm/tool_groups.go b/pkg/prm/tool_groups.go new file mode 100644 index 0000000..a33b857 --- /dev/null +++ b/pkg/prm/tool_groups.go @@ -0,0 +1,23 @@ +package prm + +/* + This package contains ToolGroups, which defines + what inidividual tools make up a specific ToolGroup. + + This allows users to specify 'group/mygroup' and have PRM + execute against a larger list of tools, without needing to define + or understand what is being called. +*/ + +var ( + ToolGroups = map[string][]string{ + // TODO: we may need to define group as a reserved word + "group/modules": { + "puppetlabs/rubocop", + "puppetlabs/rspec-puppet", + "puppetlabs/puppet-lint", + "puppetlabs/puppet-syntax", + "puppetlabs/puppet-strings", + }, + } +) From e2fc52b7fa160313c871507c0730ef23d13724b0 Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Thu, 9 Dec 2021 12:51:39 +0000 Subject: [PATCH 02/12] (GH-35) Refactor to preExecute to store to PRM tool cache --- cmd/exec/exec.go | 22 +- go.mod | 1 + pkg/prm/docker.go | 20 ++ pkg/prm/exec.go | 35 ++- pkg/prm/prm.go | 55 ++--- pkg/prm/prm_test.go | 514 +++++++++----------------------------------- 6 files changed, 186 insertions(+), 461 deletions(-) diff --git a/cmd/exec/exec.go b/cmd/exec/exec.go index 3890898..756f823 100644 --- a/cmd/exec/exec.go +++ b/cmd/exec/exec.go @@ -2,7 +2,6 @@ package exec import ( "fmt" - "path/filepath" "strings" "github.com/puppetlabs/prm/internal/pkg/utils" @@ -20,9 +19,8 @@ var ( selectedTool string selectedToolDirPath string // selectedToolInfo string - listTools bool - prmApi *prm.Prm - cachedTools []prm.ToolConfig + listTools bool + prmApi *prm.Prm ) func CreateCommand(parent *prm.Prm) *cobra.Command { @@ -45,10 +43,6 @@ func CreateCommand(parent *prm.Prm) *cobra.Command { err := tmp.RegisterFlagCompletionFunc("list", flagCompletion) cobra.CheckErr(err) - // tmp.Flags().StringVarP(&selectedToolInfo, "info", "i", "", "display the selected template's configuration and default values") - // err = tmp.RegisterFlagCompletionFunc("info", flagCompletion) - // cobra.CheckErr(err) - tmp.Flags().StringVar(&format, "format", "table", "display output in table or json format") err = tmp.RegisterFlagCompletionFunc("format", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { if len(args) != 0 { @@ -74,7 +68,7 @@ func preExecute(cmd *cobra.Command, args []string) error { if localToolPath == "" { localToolPath = prmApi.RunningConfig.ToolPath } - cachedTools = prmApi.List(localToolPath, "") + prmApi.List(localToolPath, "") return nil } @@ -107,10 +101,9 @@ func flagCompletion(cmd *cobra.Command, args []string, toComplete string) ([]str func completeName(cache string, match string) []string { var names []string - for _, tool := range cachedTools { - namespacedTemplate := fmt.Sprintf("%s/%s", tool.Plugin.Author, tool.Plugin.Id) - if strings.HasPrefix(namespacedTemplate, match) { - m := namespacedTemplate + "\t" + tool.Plugin.Display + for toolName, tool := range prmApi.Cache { + if strings.HasPrefix(toolName, match) { + m := toolName + "\t" + tool.Cfg.Plugin.Display names = append(names, m) } } @@ -129,7 +122,7 @@ func execute(cmd *cobra.Command, args []string) error { log.Trace().Msgf("Selected Tool: %v", selectedTool) if listTools { - formattedTemplates, err := prmApi.FormatTools(cachedTools, format) + formattedTemplates, err := prmApi.FormatTools(prmApi.Cache, format) if err != nil { return err } @@ -161,4 +154,5 @@ func execute(cmd *cobra.Command, args []string) error { } + return nil } diff --git a/go.mod b/go.mod index dbe1893..176c8e0 100644 --- a/go.mod +++ b/go.mod @@ -33,4 +33,5 @@ require ( gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gotest.tools/v3 v3.0.3 // indirect ) diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index ad68d55..0dcc5c4 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -30,6 +30,26 @@ func (*Docker) Validate(tool *Tool) (ToolExitCode, error) { } func (*Docker) Exec(tool *Tool, args []string) (ToolExitCode, error) { + + log.Info().Msgf("Executing docker exec command") + log.Info().Msgf("Tool: %v", tool.Cfg.Plugin) + + if tool.Cfg.Gem != nil { + log.Info().Msgf("GEM") + } + + if tool.Cfg.Puppet != nil { + log.Info().Msgf("PUPPET") + } + + if tool.Cfg.Binary != nil { + log.Info().Msgf("BINARY") + } + + if tool.Cfg.Container != nil { + log.Info().Msgf("CONTAINER") + } + // TODO return FAILURE, nil } diff --git a/pkg/prm/exec.go b/pkg/prm/exec.go index 7f90380..4dccae6 100644 --- a/pkg/prm/exec.go +++ b/pkg/prm/exec.go @@ -1,6 +1,8 @@ //nolint:structcheck,unused package prm +import "github.com/rs/zerolog/log" + type ExecExitCode int64 const ( @@ -12,23 +14,38 @@ const ( // Executes a tool with the given arguments, against the codeDir. func (p *Prm) Exec(tool *Tool, args []string) error { - var toolList []string - - // perform a check for validate.yml - - // flatten the tool list - p.flattenToolList(&toolList) - var backend BackendI switch RunningConfig.Backend { case DOCKER: backend = &Docker{} default: + backend = &Docker{} } - for _, toolName := range toolList { - tool, err := backend.GetTool(toolName, RunningConfig) + exit, err := backend.Exec(tool, args) + + if err != nil { + log.Error().Msgf("Error executing tool %s/%s: %s", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id, err.Error()) + return err + } + + switch exit { + case SUCCESS: + log.Info().Msgf("Tool %s/%s executed successfully", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) + break + case FAILURE: + log.Error().Msgf("Tool %s/%s failed to execute", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) + break + case TOOL_ERROR: + log.Error().Msgf("Tool %s/%s encountered an error", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) + break + case TOOL_NOT_FOUND: + log.Error().Msgf("Tool %s/%s not found", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) + break + default: + log.Info().Msgf("Tool %s/%s exited with code %d", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id, exit) + break } return nil diff --git a/pkg/prm/prm.go b/pkg/prm/prm.go index 79071aa..2df60e8 100644 --- a/pkg/prm/prm.go +++ b/pkg/prm/prm.go @@ -32,10 +32,6 @@ type Prm struct { Backend BackendI } -type toolCache struct { - toolName string - tool *Tool -} type PuppetVersion struct { version semver.Version } @@ -127,19 +123,6 @@ func (*Prm) getPuppetVersion() PuppetVersion { return PuppetVersion{} } -func (p *Prm) Get(toolDirPath string) (Tool, error) { - file := filepath.Join(toolDirPath, ToolConfigFileName) - _, err := p.AFS.Stat(file) - if os.IsNotExist(err) { - return Tool{}, fmt.Errorf("Couldn't find an installed tool at '%s'", toolDirPath) - } - i := p.readToolConfig(file) - if reflect.DeepEqual(i, Tool{}) { - return Tool{}, fmt.Errorf("Couldn't parse tool config at '%s'", file) - } - return i, nil -} - func (p *Prm) readToolConfig(configFile string) Tool { file, err := p.AFS.ReadFile(configFile) if err != nil { @@ -167,7 +150,7 @@ func (p *Prm) readToolConfig(configFile string) Tool { // List lists all templates in a given path and parses their configuration. Does // not return any errors from parsing invalid templates, but returns them as // debug log events -func (p *Prm) List(toolPath string, toolName string) []ToolConfig { +func (p *Prm) List(toolPath string, toolName string) { log.Debug().Msgf("Searching %+v for tool configs", toolPath) // Triple glob to match author/id/version/ToolConfigFileName matches, _ := p.IOFS.Glob(toolPath + "/**/**/**/" + ToolConfigFileName) @@ -188,7 +171,9 @@ func (p *Prm) List(toolPath string, toolName string) []ToolConfig { tmpls = p.filterNewestVersions(tmpls) - return tmpls + // cache for use with the rest of the program + // this is a seperate cache from the one used by the CLI + p.createToolCache(tmpls) } func (p *Prm) filterNewestVersions(tt []ToolConfig) (ret []ToolConfig) { @@ -238,9 +223,23 @@ func (p *Prm) FilterFiles(ss []ToolConfig, test func(ToolConfig) bool) (ret []To return } +func (p *Prm) createToolCache(tmpls []ToolConfig) { + // initialise the cache + p.Cache = make(map[string]*Tool) + // Iterate through the list of tool configs and + // add them to the map + for _, t := range tmpls { + name := t.Plugin.Author + "/" + t.Plugin.Id + tool := Tool{ + Cfg: t, + } + p.Cache[name] = &tool + } +} + // FormatTools formats one or more templates to display on the console in // table format or json format. -func (*Prm) FormatTools(tools []ToolConfig, jsonOutput string) (string, error) { +func (*Prm) FormatTools(tools map[string]*Tool, jsonOutput string) (string, error) { output := "" switch jsonOutput { case "table": @@ -249,19 +248,21 @@ func (*Prm) FormatTools(tools []ToolConfig, jsonOutput string) (string, error) { log.Warn().Msgf("Could not locate any tools at %+v", viper.GetString("toolpath")) } else if count == 1 { stringBuilder := &strings.Builder{} - stringBuilder.WriteString(fmt.Sprintf("DisplayName: %v\n", tools[0].Plugin.Display)) - stringBuilder.WriteString(fmt.Sprintf("Author: %v\n", tools[0].Plugin.Author)) - stringBuilder.WriteString(fmt.Sprintf("Name: %v\n", tools[0].Plugin.Id)) - stringBuilder.WriteString(fmt.Sprintf("Project_URL: %v\n", tools[0].Plugin.UpstreamProjUrl)) - stringBuilder.WriteString(fmt.Sprintf("Version: %v\n", tools[0].Plugin.Version)) + for key := range tools { + stringBuilder.WriteString(fmt.Sprintf("DisplayName: %v\n", tools[key].Cfg.Plugin.Display)) + stringBuilder.WriteString(fmt.Sprintf("Author: %v\n", tools[key].Cfg.Plugin.Author)) + stringBuilder.WriteString(fmt.Sprintf("Name: %v\n", tools[key].Cfg.Plugin.Id)) + stringBuilder.WriteString(fmt.Sprintf("Project_URL: %v\n", tools[key].Cfg.Plugin.UpstreamProjUrl)) + stringBuilder.WriteString(fmt.Sprintf("Version: %v\n", tools[key].Cfg.Plugin.Version)) + } output = stringBuilder.String() } else { stringBuilder := &strings.Builder{} table := tablewriter.NewWriter(stringBuilder) table.SetHeader([]string{"DisplayName", "Author", "Name", "Project_URL", "Version"}) table.SetBorder(false) - for _, v := range tools { - table.Append([]string{v.Plugin.Display, v.Plugin.Author, v.Plugin.Id, v.Plugin.UpstreamProjUrl, v.Plugin.Version}) + for key := range tools { + table.Append([]string{tools[key].Cfg.Plugin.Display, tools[key].Cfg.Plugin.Author, tools[key].Cfg.Plugin.Id, tools[key].Cfg.Plugin.UpstreamProjUrl, tools[key].Cfg.Plugin.Version}) } table.Render() output = stringBuilder.String() diff --git a/pkg/prm/prm_test.go b/pkg/prm/prm_test.go index cd57460..930b380 100644 --- a/pkg/prm/prm_test.go +++ b/pkg/prm/prm_test.go @@ -1,7 +1,6 @@ package prm_test import ( - "fmt" "io/ioutil" "os" "path/filepath" @@ -21,336 +20,9 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestGet(t *testing.T) { - type args struct { - toolDirPath string - setup bool - toolConfig string - } - tests := []struct { - name string - args args - want prm.Tool - wantErr bool - expectedErr string - }{ - { - name: "returns error for non-existent tool", - args: args{ - toolDirPath: "tools/author/i-dont-exist/0.1.0", - setup: false, - }, - wantErr: true, - expectedErr: "Couldn't find an installed tool at 'tools/author/i-dont-exist/0.1.0'", - }, - { - name: "returns config for existent tool", - args: args{ - toolDirPath: "tools/author/jeans/0.1.0", - setup: true, - toolConfig: `--- -plugin: - id: jeans - author: JoeBloggs - display: Jeans - version: 0.1.0 - upstream_project_url: https://github.com/joebloggs/prm-jeans -`, - }, - want: prm.Tool{ - Cfg: prm.ToolConfig{ - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "jeans", - Author: "JoeBloggs", - Version: "0.1.0", - }, - Display: "Jeans", - UpstreamProjUrl: "https://github.com/joebloggs/prm-jeans", - }, - }, - }, - wantErr: false, - }, - { - name: "returns config for existent tool with a malformed config", - args: args{ - toolDirPath: "tools/author/dud/0.1.0", - setup: true, - toolConfig: ``, - }, - want: prm.Tool{ - Cfg: prm.ToolConfig{}, - }, - wantErr: true, - expectedErr: fmt.Sprintf("Couldn't parse tool config at '%s'", filepath.Join("tools/author/dud/0.1.0", "prm-config.yml")), - }, - { - name: "returns config for existent GEM tool", - args: args{ - toolDirPath: "tools/author/jeans/0.1.0", - setup: true, - toolConfig: `--- -plugin: - id: jeans - author: JoeBloggs - display: Jeans - version: 0.1.0 - upstream_project_url: https://github.com/joebloggs/prm-jeans - -gem: - name: ['prmjeans', 'jeans-belt'] - executable: jeans - compatibility: - - 2.4: ['~> 0.1.0'] - - 2.5: ['>= 1.3.2', '<= 1.5.7'] -`, - }, - want: prm.Tool{ - Cfg: prm.ToolConfig{ - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "jeans", - Author: "JoeBloggs", - Version: "0.1.0", - }, - Display: "Jeans", - UpstreamProjUrl: "https://github.com/joebloggs/prm-jeans", - }, - Gem: &prm.GemConfig{ - Name: []string{"prmjeans", "jeans-belt"}, - Executable: "jeans", - BuildTools: false, - Compatibility: map[float32][]string{ - 2.4: {"~> 0.1.0"}, - 2.5: {">= 1.3.2", "<= 1.5.7"}, - }, - }, - }, - }, - wantErr: false, - }, - { - name: "returns config for existent CONTAINER tool", - args: args{ - toolDirPath: "tools/author/jeans/0.1.0", - setup: true, - toolConfig: `--- -plugin: - id: jeans - author: JoeBloggs - display: Jeans - version: 0.1.0 - upstream_project_url: https://github.com/joebloggs/prm-jeans - -container: - name: 'prmjeans' - tag: 'latest' -`, - }, - want: prm.Tool{ - Cfg: prm.ToolConfig{ - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "jeans", - Author: "JoeBloggs", - Version: "0.1.0", - }, - Display: "Jeans", - UpstreamProjUrl: "https://github.com/joebloggs/prm-jeans", - }, - Container: &prm.ContainerConfig{ - Name: "prmjeans", - Tag: "latest", - }, - }, - }, - wantErr: false, - }, - { - name: "returns config for existent BINARY tool", - args: args{ - toolDirPath: "tools/author/jeans/0.1.0", - setup: true, - toolConfig: `--- -plugin: - id: jeans - author: JoeBloggs - display: Jeans - version: 0.1.0 - upstream_project_url: https://github.com/joebloggs/prm-jeans - -binary: - name: 'prmjeans' - install_steps: - windows: | - curl http://github.com/joebloggs/prm-jeans/raw/master/bin/windows/prmjeans.exe -o prmjeans.exe - ./prmjeans.exe - linux: | - curl http://github.com/joebloggs/prm-jeans/raw/master/bin/linux/prmjeans -o prmjeans - ./prmjeans - darwin: | - curl http://github.com/joebloggs/prm-jeans/raw/master/bin/darwin/prmjeans -o prmjeans - ./prmjeans -`, - }, - want: prm.Tool{ - Cfg: prm.ToolConfig{ - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "jeans", - Author: "JoeBloggs", - Version: "0.1.0", - }, - Display: "Jeans", - UpstreamProjUrl: "https://github.com/joebloggs/prm-jeans", - }, - Binary: &prm.BinaryConfig{ - Name: "prmjeans", - InstallSteps: &prm.InstallSteps{ - Windows: "curl http://github.com/joebloggs/prm-jeans/raw/master/bin/windows/prmjeans.exe -o prmjeans.exe\n./prmjeans.exe\n", - Linux: "curl http://github.com/joebloggs/prm-jeans/raw/master/bin/linux/prmjeans -o prmjeans\n./prmjeans\n", - Darwin: "curl http://github.com/joebloggs/prm-jeans/raw/master/bin/darwin/prmjeans -o prmjeans\n./prmjeans\n", - }, - }, - }, - }, - wantErr: false, - }, - { - name: "returns config for existent PUPPET tool", - args: args{ - toolDirPath: "tools/author/jeans/0.1.0", - setup: true, - toolConfig: `--- -plugin: - id: jeans - author: JoeBloggs - display: Jeans - version: 0.1.0 - upstream_project_url: https://github.com/joebloggs/prm-jeans - -puppet: - enabled: true -`, - }, - want: prm.Tool{ - Cfg: prm.ToolConfig{ - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "jeans", - Author: "JoeBloggs", - Version: "0.1.0", - }, - Display: "Jeans", - UpstreamProjUrl: "https://github.com/joebloggs/prm-jeans", - }, - Puppet: &prm.PuppetConfig{ - Enabled: true, - }, - }, - }, - wantErr: false, - }, - { - name: "returns config for existent tool with common config items", - args: args{ - toolDirPath: "tools/author/jeans/0.1.0", - setup: true, - toolConfig: `--- -plugin: - id: jeans - author: JoeBloggs - display: Jeans - version: 0.1.0 - upstream_project_url: https://github.com/joebloggs/prm-jeans - -puppet: - enabled: true - -common: - can_validate: true - needs_write_access: false - use_script: "entrypoint" - requires_git: true - default_args: ["--verbose"] - help_arg: "--help" - success_exit_code: 0 - interleave_stdout: false - output_mode: - json: "--output json" - yaml: "-y" - junit: "--output xml" -`, - }, - want: prm.Tool{ - Cfg: prm.ToolConfig{ - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "jeans", - Author: "JoeBloggs", - Version: "0.1.0", - }, - Display: "Jeans", - UpstreamProjUrl: "https://github.com/joebloggs/prm-jeans", - }, - Puppet: &prm.PuppetConfig{ - Enabled: true, - }, - Common: prm.CommonConfig{ - CanValidate: true, - NeedsWriteAccess: false, - UseScript: "entrypoint", - RequiresGit: true, - DefaultArgs: []string{"--verbose"}, - HelpArg: "--help", - SuccessExitCode: 0, - InterleaveStdOutErr: false, - OutputMode: &prm.OutputModes{ - Json: "--output json", - Yaml: "-y", - Junit: "--output xml", - }, - }, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fs := afero.NewMemMapFs() - afs := &afero.Afero{Fs: fs} - iofs := &afero.IOFS{Fs: fs} - - if tt.args.setup { - // Create tool config - config, _ := afs.Create(filepath.Join(tt.args.toolDirPath, "prm-config.yml")) - config.Write([]byte(tt.args.toolConfig)) //nolint:errcheck - } - - p := &prm.Prm{ - AFS: afs, - IOFS: iofs, - } - - got, err := p.Get(tt.args.toolDirPath) - - if tt.wantErr { - assert.Equal(t, tt.expectedErr, err.Error()) - } - if (err != nil) != tt.wantErr { - t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr) - return - } - assert.Equal(t, tt.want, got) - }) - } -} - func TestFormatTools(t *testing.T) { type args struct { - tools []prm.ToolConfig + tools map[string]*prm.Tool jsonOutput string } tests := []struct { @@ -362,7 +34,7 @@ func TestFormatTools(t *testing.T) { { name: "When no tools are passed", args: args{ - tools: []prm.ToolConfig{}, + tools: map[string]*prm.Tool{}, jsonOutput: "table", }, matches: []string{}, @@ -370,16 +42,18 @@ func TestFormatTools(t *testing.T) { { name: "When only one tool is passed", args: args{ - tools: []prm.ToolConfig{ - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "foo", - Author: "bar", - Version: "0.1.0", + tools: map[string]*prm.Tool{ + "bar/foo": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Id: "foo", + Author: "bar", + Version: "0.1.0", + }, + Display: "Foo Item", + UpstreamProjUrl: "https://github.com/bar/pct-foo", }, - Display: "Foo Item", - UpstreamProjUrl: "https://github.com/bar/pct-foo", }, }, }, @@ -396,27 +70,31 @@ func TestFormatTools(t *testing.T) { { name: "When more than one tool is passed", args: args{ - tools: []prm.ToolConfig{ - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "foo", - Author: "baz", - Version: "0.1.0", + tools: map[string]*prm.Tool{ + "baz/foo": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Id: "foo", + Author: "baz", + Version: "0.1.0", + }, + Display: "Foo Item", + UpstreamProjUrl: "https://github.com/baz/pct-foo", }, - Display: "Foo Item", - UpstreamProjUrl: "https://github.com/baz/pct-foo", }, }, - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "bar", - Author: "baz", - Version: "0.1.0", + "baz/bar": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Id: "bar", + Version: "0.1.0", + Author: "baz", + }, + Display: "Bar Item", + UpstreamProjUrl: "https://github.com/baz/pct-bar", }, - Display: "Bar Item", - UpstreamProjUrl: "https://github.com/baz/pct-bar", }, }, }, @@ -431,27 +109,31 @@ func TestFormatTools(t *testing.T) { { name: "When format is specified as json", args: args{ - tools: []prm.ToolConfig{ - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "foo", - Author: "baz", - Version: "0.1.0", + tools: map[string]*prm.Tool{ + "baz/foo": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Id: "foo", + Version: "0.1.0", + Author: "baz", + }, + Display: "Foo Item", + UpstreamProjUrl: "https://github.com/baz/pct-foo", }, - Display: "Foo Item", - UpstreamProjUrl: "https://github.com/baz/pct-foo", }, }, - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Id: "bar", - Author: "baz", - Version: "0.1.0", + "baz/bar": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Id: "bar", + Author: "baz", + Version: "0.1.0", + }, + Display: "Bar Item", + UpstreamProjUrl: "https://github.com/baz/pct-bar", }, - Display: "Bar Item", - UpstreamProjUrl: "https://github.com/baz/pct-bar", }, }, }, @@ -498,13 +180,14 @@ func TestList(t *testing.T) { tests := []struct { name string args args - want []prm.ToolConfig + want map[string]*prm.Tool }{ { name: "when no tools are found", args: args{ toolPath: "stubbed/tools/none", }, + want: map[string]*prm.Tool{}, }, { name: "when an invalid tool config is found", @@ -517,6 +200,7 @@ func TestList(t *testing.T) { }, }, }, + want: map[string]*prm.Tool{}, }, { name: "when valid tool configs are found", @@ -547,27 +231,31 @@ plugin: }, }, }, - want: []prm.ToolConfig{ - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Author: "some_author", - Id: "first", - Version: "0.1.0", + want: map[string]*prm.Tool{ + "some_author/first": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Author: "some_author", + Id: "first", + Version: "0.1.0", + }, + Display: "First Tool", + UpstreamProjUrl: "https://github.com/some_author/pct-first-tool", }, - Display: "First Tool", - UpstreamProjUrl: "https://github.com/some_author/pct-first-tool", }, }, - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Author: "some_author", - Id: "second", - Version: "0.1.0", + "some_author/second": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Author: "some_author", + Id: "second", + Version: "0.1.0", + }, + Display: "Second Tool", + UpstreamProjUrl: "https://github.com/some_author/pct-second-tool", }, - Display: "Second Tool", - UpstreamProjUrl: "https://github.com/some_author/pct-second-tool", }, }, }, @@ -601,16 +289,18 @@ plugin: }, }, }, - want: []prm.ToolConfig{ - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Author: "some_author", - Id: "first", - Version: "0.2.0", + want: map[string]*prm.Tool{ + "some_author/first": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Author: "some_author", + Version: "0.2.0", + Id: "first", + }, + Display: "First Tool", + UpstreamProjUrl: "https://github.com/some_author/pct-first-tool", }, - Display: "First Tool", - UpstreamProjUrl: "https://github.com/some_author/pct-first-tool", }, }, }, @@ -645,16 +335,18 @@ plugin: }, }, }, - want: []prm.ToolConfig{ - { - Plugin: &prm.PluginConfig{ - ConfigParams: install.ConfigParams{ - Author: "some_author", - Id: "first", - Version: "0.1.0", + want: map[string]*prm.Tool{ + "some_author/first": { + Cfg: prm.ToolConfig{ + Plugin: &prm.PluginConfig{ + ConfigParams: install.ConfigParams{ + Author: "some_author", + Id: "first", + Version: "0.1.0", + }, + Display: "First Tool", + UpstreamProjUrl: "https://github.com/some_author/pct-first-tool", }, - Display: "First Tool", - UpstreamProjUrl: "https://github.com/some_author/pct-first-tool", }, }, }, @@ -679,8 +371,8 @@ plugin: IOFS: iofs, } - got := p.List(tt.args.toolPath, tt.args.toolName) - assert.Equal(t, tt.want, got) + p.List(tt.args.toolPath, tt.args.toolName) + assert.Equal(t, tt.want, p.Cache) }) } } From 0327df7e3539f9aab53fc4274cd6ef65da3d3cb6 Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Mon, 6 Dec 2021 16:13:33 +0000 Subject: [PATCH 03/12] (GH-25) Add docker commands for Gem --- cmd/exec/exec.go | 8 ++++++++ go.mod | 1 - pkg/prm/docker.go | 4 +++- pkg/prm/exec.go | 12 +----------- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/exec/exec.go b/cmd/exec/exec.go index 756f823..20741a2 100644 --- a/cmd/exec/exec.go +++ b/cmd/exec/exec.go @@ -68,6 +68,14 @@ func preExecute(cmd *cobra.Command, args []string) error { if localToolPath == "" { localToolPath = prmApi.RunningConfig.ToolPath } + + switch prm.RunningConfig.Backend { + case prm.DOCKER: + prmApi.Backend = &prm.Docker{} + default: + prmApi.Backend = &prm.Docker{} + } + prmApi.List(localToolPath, "") return nil } diff --git a/go.mod b/go.mod index 176c8e0..dbe1893 100644 --- a/go.mod +++ b/go.mod @@ -33,5 +33,4 @@ require ( gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - gotest.tools/v3 v3.0.3 // indirect ) diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index 0dcc5c4..fce2dd3 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -29,7 +29,9 @@ func (*Docker) Validate(tool *Tool) (ToolExitCode, error) { return FAILURE, nil } -func (*Docker) Exec(tool *Tool, args []string) (ToolExitCode, error) { +func (d *Docker) Exec(tool *Tool, args []string) (ToolExitCode, error) { + + d.initClient() log.Info().Msgf("Executing docker exec command") log.Info().Msgf("Tool: %v", tool.Cfg.Plugin) diff --git a/pkg/prm/exec.go b/pkg/prm/exec.go index 4dccae6..eb16b9b 100644 --- a/pkg/prm/exec.go +++ b/pkg/prm/exec.go @@ -13,17 +13,7 @@ const ( // Executes a tool with the given arguments, against the codeDir. func (p *Prm) Exec(tool *Tool, args []string) error { - - var backend BackendI - - switch RunningConfig.Backend { - case DOCKER: - backend = &Docker{} - default: - backend = &Docker{} - } - - exit, err := backend.Exec(tool, args) + exit, err := p.Backend.Exec(tool, args) if err != nil { log.Error().Msgf("Error executing tool %s/%s: %s", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id, err.Error()) From ace4daa7dd92506be719a7496d9bc51f2b666146 Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Thu, 9 Dec 2021 10:32:17 +0000 Subject: [PATCH 04/12] (GH-35) Image exists This commit provides the basic flow when the image already exists. --- cmd/exec/exec.go | 17 ++++- internal/pkg/mock/backend.go | 6 +- pkg/prm/backend.go | 9 ++- pkg/prm/docker.go | 131 +++++++++++++++++++++++++++++------ pkg/prm/exec.go | 10 ++- pkg/prm/prm.go | 19 +++-- 6 files changed, 154 insertions(+), 38 deletions(-) diff --git a/cmd/exec/exec.go b/cmd/exec/exec.go index 20741a2..df76370 100644 --- a/cmd/exec/exec.go +++ b/cmd/exec/exec.go @@ -2,6 +2,8 @@ package exec import ( "fmt" + "os/user" + "path/filepath" "strings" "github.com/puppetlabs/prm/internal/pkg/utils" @@ -18,6 +20,8 @@ var ( format string selectedTool string selectedToolDirPath string + codedir string + cachedir string // selectedToolInfo string listTools bool prmApi *prm.Prm @@ -61,6 +65,10 @@ func CreateCommand(parent *prm.Prm) *cobra.Command { err = viper.BindPFlag("codedir", tmp.Flags().Lookup("codedir")) cobra.CheckErr(err) + tmp.Flags().StringVar(&prmApi.CacheDir, "cachedir", "", "location of cache used by PRM") + err = viper.BindPFlag("cachedir", tmp.Flags().Lookup("cachedir")) + cobra.CheckErr(err) + return tmp } @@ -76,6 +84,13 @@ func preExecute(cmd *cobra.Command, args []string) error { prmApi.Backend = &prm.Docker{} } + // handle the default cachepath + if cachedir == "" { + usr, _ := user.Current() + dir := usr.HomeDir + prmApi.CacheDir = filepath.Join(dir, ".pdk/prm/cache") + } + prmApi.List(localToolPath, "") return nil } @@ -143,7 +158,7 @@ func execute(cmd *cobra.Command, args []string) error { // get the tool from the cache cachedTool, ok := prmApi.IsToolAvailable(selectedTool) if !ok { - return fmt.Errorf("Tool %s not found in cache", selectedTool) + return fmt.Errorf("Tool %s not found in cache", selectedTool) } // execute! err := prmApi.Exec(cachedTool, args[1:]) diff --git a/internal/pkg/mock/backend.go b/internal/pkg/mock/backend.go index acac82d..6b44da0 100644 --- a/internal/pkg/mock/backend.go +++ b/internal/pkg/mock/backend.go @@ -14,8 +14,8 @@ func (m *MockBackend) Status() prm.BackendStatus { } // Implement when needed -func (m *MockBackend) GetTool(toolName string, prmConfig prm.Config) (prm.Tool, error) { - return prm.Tool{}, nil +func (m *MockBackend) GetTool(tool *prm.Tool, prmConfig prm.Config) error{ + return nil } // Implement when needed @@ -24,6 +24,6 @@ func (m *MockBackend) Validate(tool *prm.Tool) (prm.ToolExitCode, error) { } // Implement when needed -func (m *MockBackend) Exec(tool *prm.Tool, args []string) (prm.ToolExitCode, error) { +func (m *MockBackend) Exec(tool *prm.Tool, args []string, prmConfig prm.Config, paths prm.DirectoryPaths) (prm.ToolExitCode, error) { return prm.FAILURE, nil } diff --git a/pkg/prm/backend.go b/pkg/prm/backend.go index e04e3d6..a7236bd 100644 --- a/pkg/prm/backend.go +++ b/pkg/prm/backend.go @@ -8,9 +8,9 @@ const ( ) type BackendI interface { - GetTool(toolName string, prmConfig Config) (Tool, error) + GetTool(tool *Tool, prmConfig Config) error Validate(tool *Tool) (ToolExitCode, error) - Exec(tool *Tool, args []string) (ToolExitCode, error) + Exec(tool *Tool, args []string, prmConfig Config, paths DirectoryPaths) (ToolExitCode, error) Status() BackendStatus } @@ -21,3 +21,8 @@ type BackendStatus struct { IsAvailable bool StatusMsg string } + +type DirectoryPaths struct { + codeDir string + cacheDir string +} diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index fce2dd3..05e1529 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -3,15 +3,23 @@ package prm import ( "context" "fmt" + "os" + "path/filepath" "strings" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" dockerClient "github.com/docker/docker/client" + "github.com/docker/docker/pkg/stdcopy" + "github.com/rs/zerolog/log" ) type Docker struct { // We need to be able to mock the docker client in testing - Client DockerClientI + Client DockerClientI + OrigClient *dockerClient.Client + Context context.Context } type DockerClientI interface { @@ -19,9 +27,46 @@ type DockerClientI interface { ServerVersion(context.Context) (types.Version, error) } -func (*Docker) GetTool(toolName string, prmConfig Config) (Tool, error) { - // TODO - return Tool{}, nil +func (d *Docker) GetTool(tool *Tool, prmConfig Config) error { + + // initialise the docker client + err := d.initClient() + if err != nil { + return err + } + + // what are we looking for? + toolImageName := d.ImageName(tool, prmConfig) + + // find out if docker knows about our tool + list, err := d.OrigClient.ImageList(d.Context, types.ImageListOptions{}) + + if err != nil { + log.Debug().Msgf("Error listing containers: %v", err) + return err + } + + for _, image := range list { + for _, tag := range image.RepoTags { + if tag == toolImageName { + log.Info().Msgf("Found container: %s", image.ID) + return nil + } + } + } + + // No image found with that configuration + // we must create it + // d.createDockerfile(tool, prmConfig) + + return fmt.Errorf("No image found %s", toolImageName) +} + +// Creates a unique name for the image based on the tool and the PRM configuration +func (d *Docker) ImageName(tool *Tool, prmConfig Config) string { + // build up a name based on the tool and puppet version + imageName := fmt.Sprintf("pdk:puppet-%s_%s-%s_%s", prmConfig.PuppetVersion.String(), tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id, tool.Cfg.Plugin.Version) + return imageName } func (*Docker) Validate(tool *Tool) (ToolExitCode, error) { @@ -29,41 +74,85 @@ func (*Docker) Validate(tool *Tool) (ToolExitCode, error) { return FAILURE, nil } -func (d *Docker) Exec(tool *Tool, args []string) (ToolExitCode, error) { +func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths DirectoryPaths) (ToolExitCode, error) { + // is Docker up and running? + status := d.Status() + if !status.IsAvailable { + log.Error().Msgf("Docker is not available") + return FAILURE, fmt.Errorf("%s", status.StatusMsg) + } - d.initClient() + // clean up paths + codeDir, _ := filepath.Abs(paths.codeDir) + log.Info().Msgf("Code path: %s", codeDir) + cacheDir, _ := filepath.Abs(paths.cacheDir) + log.Info().Msgf("Cache path: %s", cacheDir) + + // stand up a container + resp, err := d.OrigClient.ContainerCreate(d.Context, &container.Config{ + Image: d.ImageName(tool, prmConfig), + Cmd: args, + Tty: false, + }, + &container.HostConfig{ + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: codeDir, + Target: "/code", + }, + { + Type: mount.TypeBind, + Source: cacheDir, + Target: "/cache", + }, + }, + }, nil, nil, "") - log.Info().Msgf("Executing docker exec command") - log.Info().Msgf("Tool: %v", tool.Cfg.Plugin) + if err != nil { + return FAILURE, err + } + defer d.OrigClient.ContainerRemove(d.Context, resp.ID, types.ContainerRemoveOptions{}) - if tool.Cfg.Gem != nil { - log.Info().Msgf("GEM") + if err := d.OrigClient.ContainerStart(d.Context, resp.ID, types.ContainerStartOptions{}); err != nil { + return FAILURE, err } - if tool.Cfg.Puppet != nil { - log.Info().Msgf("PUPPET") + statusCh, errCh := d.OrigClient.ContainerWait(d.Context, resp.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + return FAILURE, err + } + case <-statusCh: } - if tool.Cfg.Binary != nil { - log.Info().Msgf("BINARY") + out, err := d.OrigClient.ContainerLogs(d.Context, resp.ID, types.ContainerLogsOptions{ShowStdout: true}) + if err != nil { + return FAILURE, err } - if tool.Cfg.Container != nil { - log.Info().Msgf("CONTAINER") + _, err = stdcopy.StdCopy(os.Stdout, os.Stderr, out) + if err != nil { + return FAILURE, err } - // TODO - return FAILURE, nil + return SUCCESS, nil } func (d *Docker) initClient() (err error) { if d.Client == nil { cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv) + if err != nil { + return err + } + d.Client = cli - return err + d.OrigClient = cli // TODO: remove this when we know all the functions that need added to the interface + d.Context = context.Background() } - return err + return nil } // Check to see if the Docker runtime is available: @@ -79,7 +168,7 @@ func (d *Docker) Status() BackendStatus { } // The client does not error on creation if the background service is not running, // but attempting to list the containers does. - dockerInfo, err := d.Client.ServerVersion(context.Background()) + dockerInfo, err := d.Client.ServerVersion(d.Context) if err != nil { // message := fmt.Sprintf("%s", err) message := err.Error() diff --git a/pkg/prm/exec.go b/pkg/prm/exec.go index eb16b9b..4692177 100644 --- a/pkg/prm/exec.go +++ b/pkg/prm/exec.go @@ -13,8 +13,16 @@ const ( // Executes a tool with the given arguments, against the codeDir. func (p *Prm) Exec(tool *Tool, args []string) error { - exit, err := p.Backend.Exec(tool, args) + // is the tool available? + err := p.Backend.GetTool(tool, RunningConfig) + if err != nil { + log.Error().Msgf("Failed to exec tool: %s/%s", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) + return err + } + + // the tool is available so execute against it + exit, err := p.Backend.Exec(tool, args, RunningConfig, DirectoryPaths{codeDir: p.CodeDir, cacheDir: p.CacheDir}) if err != nil { log.Error().Msgf("Error executing tool %s/%s: %s", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id, err.Error()) return err diff --git a/pkg/prm/prm.go b/pkg/prm/prm.go index 2df60e8..7c98a01 100644 --- a/pkg/prm/prm.go +++ b/pkg/prm/prm.go @@ -28,8 +28,9 @@ type Prm struct { IOFS *afero.IOFS RunningConfig Config codeDir string - cache []toolCache - Backend BackendI + CacheDir string + Cache map[string]*Tool + Backend BackendI } type PuppetVersion struct { @@ -108,14 +109,12 @@ func (p *Prm) IsToolAvailable(tool string) (*Tool, bool) { } // Check to see if the tool is ready to execute -func (*Prm) IsToolReady(tool *Tool) bool { - return false -} - -// save traversing to the filesystem -func (*Prm) cacheTool(tool *Tool) error { - // TODO - return nil +func (p *Prm) IsToolReady(tool *Tool) bool { + err := p.Backend.GetTool(tool, RunningConfig) + if err != nil { + return false + } + return true } // What version of Puppet is requested by the user From e4c900acdca6556d15a0f8b17736e5fc87fbd4ce Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Wed, 8 Dec 2021 13:53:14 +0000 Subject: [PATCH 05/12] (GH-35) Create image This commit allows an image to be created when it doesn't exist on the system --- cmd/exec/exec.go | 11 ++++ go.mod | 1 + go.sum | 11 ++++ pkg/prm/docker.go | 125 ++++++++++++++++++++++++++++++++++++++++++-- pkg/prm/prm.go | 1 + pkg/prm/prm_test.go | 4 ++ pkg/prm/tool.go | 20 +++---- 7 files changed, 159 insertions(+), 14 deletions(-) diff --git a/cmd/exec/exec.go b/cmd/exec/exec.go index df76370..c6db087 100644 --- a/cmd/exec/exec.go +++ b/cmd/exec/exec.go @@ -175,6 +175,17 @@ func execute(cmd *cobra.Command, args []string) error { log.Info().Msgf("Found tools: %v ", toolList) + for _, tool := range toolList { + cachedTool, ok := prmApi.IsToolAvailable(tool) + if !ok { + return fmt.Errorf("Tool %s not found in cache", tool) + } + err := prmApi.Exec(cachedTool, args) // todo: do we want to allow folk to specify args from validate.yml? + if err != nil { + return err + } + } + } return nil diff --git a/go.mod b/go.mod index dbe1893..c764f9c 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/moby/sys/mount v0.3.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/olekukonko/tablewriter v0.0.5 diff --git a/go.sum b/go.sum index 942c740..2a23cde 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,7 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.23 h1:47MSwtKGXet80aIn+7h4YI6fwPmwIghAnsx2aOUrG2M= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= @@ -155,6 +156,7 @@ github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1 github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= @@ -182,6 +184,7 @@ github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= @@ -342,6 +345,7 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -551,8 +555,12 @@ github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGg github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/mount v0.3.0 h1:bXZYMmq7DBQPwHRxH/MG+u9+XF90ZOwoXpHTOznMGp0= +github.com/moby/sys/mount v0.3.0/go.mod h1:U2Z3ur2rXPFrFmy4q6WMwWrBOAQGYtYTRVM8BIvzbwk= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= @@ -603,6 +611,7 @@ github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59P github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -781,6 +790,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/otel v1.2.0 h1:YOQDvxO1FayUcT9MIhJhgMyNO1WqoduiyvQHzGN0kUQ= go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= @@ -1028,6 +1038,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index 05e1529..676bbf9 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -1,8 +1,11 @@ package prm import ( + "bufio" "context" + "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" @@ -11,6 +14,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" dockerClient "github.com/docker/docker/client" + "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/stdcopy" "github.com/rs/zerolog/log" ) @@ -42,24 +46,133 @@ func (d *Docker) GetTool(tool *Tool, prmConfig Config) error { list, err := d.OrigClient.ImageList(d.Context, types.ImageListOptions{}) if err != nil { - log.Debug().Msgf("Error listing containers: %v", err) + log.Debug().Msgf("Error listing images: %v", err) return err } for _, image := range list { for _, tag := range image.RepoTags { if tag == toolImageName { - log.Info().Msgf("Found container: %s", image.ID) + log.Info().Msgf("Found image: %s", image.ID) return nil } } } + log.Info().Msg("Creating new image. Please wait...") + // No image found with that configuration // we must create it - // d.createDockerfile(tool, prmConfig) + fileString := d.createDockerfile(tool, prmConfig) + log.Debug().Msgf("Creating Dockerfile\n--------------------\n%s--------------------\n", fileString) + reader := strings.NewReader(fileString) + + // write the contents of fileString to a Dockerfile stored in the + // tool path + filePath := filepath.Join(tool.Cfg.Path, "generated.Dockerfile") + file, err := os.Create(filePath) + if err != nil { + log.Error().Msgf("Error creating Dockerfile: %v", err) + return err + } + defer file.Close() + + _, err = io.Copy(file, reader) + if err != nil { + log.Error().Msgf("Error copying Dockerfile: %v", err) + return err + } + + // create a tar of the tool directory *shrug* + tar, err := archive.TarWithOptions(tool.Cfg.Path, &archive.TarOptions{}) + if err != nil { + return err + } + + // build the image + imageBuildResponse, err := d.OrigClient.ImageBuild( + d.Context, + tar, + types.ImageBuildOptions{ + Dockerfile: "generated.Dockerfile", + Tags: []string{toolImageName}, + Remove: true, + }) + + if err != nil { + log.Error().Msgf("Unable to build docker image") + return err + } - return fmt.Errorf("No image found %s", toolImageName) + defer imageBuildResponse.Body.Close() + + // Parse the output from Docker, cleaning up where possible + scanner := bufio.NewScanner(imageBuildResponse.Body) + for scanner.Scan() { + var line map[string]string + json.Unmarshal(scanner.Bytes(), &line) + printLine := strings.TrimSuffix(line["stream"], "\n") + if printLine != "" { + log.Debug().Msgf("%s", printLine) + } + } + + return nil +} + +func (d *Docker) createDockerfile(tool *Tool, prmConfig Config) string { + // create a dockerfile from the Tool and prmConfig + dockerfile := strings.Builder{} + dockerfile.WriteString(fmt.Sprintf("FROM puppet/puppet-agent:%s\n", prmConfig.PuppetVersion.String())) + + if tool.Cfg.Common.RequiresGit || (tool.Cfg.Gem != nil && tool.Cfg.Gem.BuildTools) { + dockerfile.WriteString("RUN apt update\n") + } + + if tool.Cfg.Common.RequiresGit { + dockerfile.WriteString("RUN apt install git -y\n") + } + + if tool.Cfg.Gem != nil { + if tool.Cfg.Gem.BuildTools { + dockerfile.WriteString("RUN apt install build-essential -y\n") + } + + dockerfile.WriteString("RUN /opt/puppetlabs/puppet/bin/gem install bundler --no-document\n") + + for _, gem := range tool.Cfg.Gem.Name { + dockerfile.WriteString(fmt.Sprintf("RUN /opt/puppetlabs/puppet/bin/gem install %s -f --conservative --minimal-deps --no-document\n", gem)) + } + } + + for key, val := range tool.Cfg.Common.Env { + dockerfile.WriteString(fmt.Sprintf("ENV %s=%s\n", key, val)) + } + + // Copy the tools content into the image + // contentPath := filepath.Join(tool.Cfg.Path, "content", "*") + // dockerfile.WriteString(fmt.Sprintf("COPY %s /tmp/ \n", contentPath)) + if _, err := os.Stat(filepath.Join(tool.Cfg.Path, "/content")); err == nil { + dockerfile.WriteString("COPY ./content/* /tmp/ \n") + } + + dockerfile.WriteString("VOLUME [ /code, /cache ]\n") + dockerfile.WriteString("WORKDIR /code\n") + + if tool.Cfg.Common.UseScript != "" { + // todo: handle ps1 scripts + dockerfile.WriteString(fmt.Sprintf("ENTRYPOINT [\"/tmp/%s.sh\"]\n", tool.Cfg.Common.UseScript)) + } else { + if tool.Cfg.Gem != nil { + dockerfile.WriteString(fmt.Sprintf("ENTRYPOINT [ \"/opt/puppetlabs/puppet/bin/%s\"]\n", tool.Cfg.Gem.Executable)) + } + } + + if len(tool.Cfg.Common.DefaultArgs) > 0 { + dockerfile.WriteString(fmt.Sprintf("CMD %q\n", tool.Cfg.Common.DefaultArgs)) + } + + return dockerfile.String() } // Creates a unique name for the image based on the tool and the PRM configuration @@ -88,6 +201,8 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo cacheDir, _ := filepath.Abs(paths.cacheDir) log.Info().Msgf("Cache path: %s", cacheDir) + log.Info().Msgf("Additional Args: %v", args) + // stand up a container resp, err := d.OrigClient.ContainerCreate(d.Context, &container.Config{ Image: d.ImageName(tool, prmConfig), @@ -127,7 +242,7 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo case <-statusCh: } - out, err := d.OrigClient.ContainerLogs(d.Context, resp.ID, types.ContainerLogsOptions{ShowStdout: true}) + out, err := d.OrigClient.ContainerLogs(d.Context, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}) if err != nil { return FAILURE, err } diff --git a/pkg/prm/prm.go b/pkg/prm/prm.go index 7c98a01..8ced415 100644 --- a/pkg/prm/prm.go +++ b/pkg/prm/prm.go @@ -159,6 +159,7 @@ func (p *Prm) List(toolPath string, toolName string) { log.Debug().Msgf("Found: %+v", file) i := p.readToolConfig(file) if i.Cfg.Plugin != nil { + i.Cfg.Path = filepath.Dir(file) tmpls = append(tmpls, i.Cfg) } } diff --git a/pkg/prm/prm_test.go b/pkg/prm/prm_test.go index 930b380..46487f4 100644 --- a/pkg/prm/prm_test.go +++ b/pkg/prm/prm_test.go @@ -234,6 +234,7 @@ plugin: want: map[string]*prm.Tool{ "some_author/first": { Cfg: prm.ToolConfig{ + Path: filepath.Join("stubbed/tools/valid/some_author/first/0.1.0"), Plugin: &prm.PluginConfig{ ConfigParams: install.ConfigParams{ Author: "some_author", @@ -247,6 +248,7 @@ plugin: }, "some_author/second": { Cfg: prm.ToolConfig{ + Path: filepath.Join("stubbed/tools/valid/some_author/second/0.1.0"), Plugin: &prm.PluginConfig{ ConfigParams: install.ConfigParams{ Author: "some_author", @@ -292,6 +294,7 @@ plugin: want: map[string]*prm.Tool{ "some_author/first": { Cfg: prm.ToolConfig{ + Path: filepath.Join("stubbed/tools/multiversion/some_author/first/0.2.0"), Plugin: &prm.PluginConfig{ ConfigParams: install.ConfigParams{ Author: "some_author", @@ -338,6 +341,7 @@ plugin: want: map[string]*prm.Tool{ "some_author/first": { Cfg: prm.ToolConfig{ + Path: filepath.Join("stubbed/tools/named/some_author/first/0.1.0"), Plugin: &prm.PluginConfig{ ConfigParams: install.ConfigParams{ Author: "some_author", diff --git a/pkg/prm/tool.go b/pkg/prm/tool.go index 46ad6bf..6ff8000 100644 --- a/pkg/prm/tool.go +++ b/pkg/prm/tool.go @@ -24,6 +24,7 @@ const ( ) type ToolConfig struct { + Path string Plugin *PluginConfig `mapstructure:"plugin"` Gem *GemConfig `mapstructure:"gem"` Container *ContainerConfig `mapstructure:"container"` @@ -72,15 +73,16 @@ type PuppetConfig struct { } type CommonConfig struct { - CanValidate bool `mapstructure:"can_validate"` - NeedsWriteAccess bool `mapstructure:"needs_write_access"` - UseScript string `mapstructure:"use_script"` - RequiresGit bool `mapstructure:"requires_git"` - DefaultArgs []string `mapstructure:"default_args"` - HelpArg string `mapstructure:"help_arg"` - SuccessExitCode int `mapstructure:"success_exit_code"` - InterleaveStdOutErr bool `mapstructure:"interleave_stdout"` - OutputMode *OutputModes `mapstructure:"output_mode"` + CanValidate bool `mapstructure:"can_validate"` + NeedsWriteAccess bool `mapstructure:"needs_write_access"` + UseScript string `mapstructure:"use_script"` + RequiresGit bool `mapstructure:"requires_git"` + DefaultArgs []string `mapstructure:"default_args"` + HelpArg string `mapstructure:"help_arg"` + SuccessExitCode int `mapstructure:"success_exit_code"` + InterleaveStdOutErr bool `mapstructure:"interleave_stdout"` + OutputMode *OutputModes `mapstructure:"output_mode"` + Env map[string]string `mapstructure:"env"` } type OutputModes struct { From a3f58b80919223d0a272951aa11ff7ac3646b46e Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Tue, 7 Dec 2021 22:39:03 +0000 Subject: [PATCH 06/12] (maint) linting fixes --- cmd/exec/exec.go | 11 ++++------- pkg/prm/docker.go | 15 ++++++++++++--- pkg/prm/exec.go | 5 ----- pkg/prm/prm.go | 5 +---- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/cmd/exec/exec.go b/cmd/exec/exec.go index c6db087..2c9b359 100644 --- a/cmd/exec/exec.go +++ b/cmd/exec/exec.go @@ -16,12 +16,9 @@ import ( ) var ( - localToolPath string - format string - selectedTool string - selectedToolDirPath string - codedir string - cachedir string + localToolPath string + format string + selectedTool string // selectedToolInfo string listTools bool prmApi *prm.Prm @@ -85,7 +82,7 @@ func preExecute(cmd *cobra.Command, args []string) error { } // handle the default cachepath - if cachedir == "" { + if prmApi.CacheDir == "" { usr, _ := user.Current() dir := usr.HomeDir prmApi.CacheDir = filepath.Join(dir, ".pdk/prm/cache") diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index 676bbf9..e6b5e43 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -75,7 +75,11 @@ func (d *Docker) GetTool(tool *Tool, prmConfig Config) error { log.Error().Msgf("Error creating Dockerfile: %v", err) return err } - defer file.Close() + defer func() { + if err := file.Close(); err != nil { + log.Error().Msgf("Error closing file: %s", err) + } + }() _, err = io.Copy(file, reader) if err != nil { @@ -110,7 +114,7 @@ func (d *Docker) GetTool(tool *Tool, prmConfig Config) error { scanner := bufio.NewScanner(imageBuildResponse.Body) for scanner.Scan() { var line map[string]string - json.Unmarshal(scanner.Bytes(), &line) + _ = json.Unmarshal(scanner.Bytes(), &line) // nolint:errcheck // we don't care about the error here printLine := strings.TrimSuffix(line["stream"], "\n") if printLine != "" { log.Debug().Msgf("%s", printLine) @@ -227,7 +231,12 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo if err != nil { return FAILURE, err } - defer d.OrigClient.ContainerRemove(d.Context, resp.ID, types.ContainerRemoveOptions{}) + defer func() { + err := d.OrigClient.ContainerRemove(d.Context, resp.ID, types.ContainerRemoveOptions{}) + if err != nil { + log.Error().Msgf("Error removing container: %s", err) + } + }() if err := d.OrigClient.ContainerStart(d.Context, resp.ID, types.ContainerStartOptions{}); err != nil { return FAILURE, err diff --git a/pkg/prm/exec.go b/pkg/prm/exec.go index 4692177..2a073c0 100644 --- a/pkg/prm/exec.go +++ b/pkg/prm/exec.go @@ -31,19 +31,14 @@ func (p *Prm) Exec(tool *Tool, args []string) error { switch exit { case SUCCESS: log.Info().Msgf("Tool %s/%s executed successfully", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) - break case FAILURE: log.Error().Msgf("Tool %s/%s failed to execute", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) - break case TOOL_ERROR: log.Error().Msgf("Tool %s/%s encountered an error", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) - break case TOOL_NOT_FOUND: log.Error().Msgf("Tool %s/%s not found", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) - break default: log.Info().Msgf("Tool %s/%s exited with code %d", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id, exit) - break } return nil diff --git a/pkg/prm/prm.go b/pkg/prm/prm.go index 8ced415..76ea149 100644 --- a/pkg/prm/prm.go +++ b/pkg/prm/prm.go @@ -111,10 +111,7 @@ func (p *Prm) IsToolAvailable(tool string) (*Tool, bool) { // Check to see if the tool is ready to execute func (p *Prm) IsToolReady(tool *Tool) bool { err := p.Backend.GetTool(tool, RunningConfig) - if err != nil { - return false - } - return true + return err == nil } // What version of Puppet is requested by the user From 861eb85807403eb6a1d25140221fbe844c20db39 Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Wed, 8 Dec 2021 10:18:03 +0000 Subject: [PATCH 07/12] (maint) Support for running config change --- cmd/exec/exec.go | 2 +- pkg/prm/exec.go | 4 ++-- pkg/prm/prm.go | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/exec/exec.go b/cmd/exec/exec.go index 2c9b359..e8eb008 100644 --- a/cmd/exec/exec.go +++ b/cmd/exec/exec.go @@ -74,7 +74,7 @@ func preExecute(cmd *cobra.Command, args []string) error { localToolPath = prmApi.RunningConfig.ToolPath } - switch prm.RunningConfig.Backend { + switch prmApi.RunningConfig.Backend { case prm.DOCKER: prmApi.Backend = &prm.Docker{} default: diff --git a/pkg/prm/exec.go b/pkg/prm/exec.go index 2a073c0..ed05458 100644 --- a/pkg/prm/exec.go +++ b/pkg/prm/exec.go @@ -15,14 +15,14 @@ const ( func (p *Prm) Exec(tool *Tool, args []string) error { // is the tool available? - err := p.Backend.GetTool(tool, RunningConfig) + err := p.Backend.GetTool(tool, p.RunningConfig) if err != nil { log.Error().Msgf("Failed to exec tool: %s/%s", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id) return err } // the tool is available so execute against it - exit, err := p.Backend.Exec(tool, args, RunningConfig, DirectoryPaths{codeDir: p.CodeDir, cacheDir: p.CacheDir}) + exit, err := p.Backend.Exec(tool, args, p.RunningConfig, DirectoryPaths{codeDir: p.CodeDir, cacheDir: p.CacheDir}) if err != nil { log.Error().Msgf("Error executing tool %s/%s: %s", tool.Cfg.Plugin.Author, tool.Cfg.Plugin.Id, err.Error()) return err diff --git a/pkg/prm/prm.go b/pkg/prm/prm.go index 76ea149..35bb618 100644 --- a/pkg/prm/prm.go +++ b/pkg/prm/prm.go @@ -27,10 +27,10 @@ type Prm struct { AFS *afero.Afero IOFS *afero.IOFS RunningConfig Config - codeDir string - CacheDir string - Cache map[string]*Tool - Backend BackendI + CodeDir string + CacheDir string + Cache map[string]*Tool + Backend BackendI } type PuppetVersion struct { @@ -110,7 +110,7 @@ func (p *Prm) IsToolAvailable(tool string) (*Tool, bool) { // Check to see if the tool is ready to execute func (p *Prm) IsToolReady(tool *Tool) bool { - err := p.Backend.GetTool(tool, RunningConfig) + err := p.Backend.GetTool(tool, p.RunningConfig) return err == nil } From 54fa9568273cd4b60cf247bdf2563f5a077ec398 Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Thu, 9 Dec 2021 15:19:31 +0000 Subject: [PATCH 08/12] (GH-35) Handle gem compatibility ``` gem: compatibility: - 2.5: - "puppetlabs_spec_helper": "2.15.0" - 2.4: - "puppetlabs_spec_helper": "2.15.0" ``` --- pkg/prm/docker.go | 46 ++++++++++++++++++++++++++++++++++++++++++---- pkg/prm/tool.go | 8 ++++---- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index e6b5e43..e8236c7 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" + "github.com/Masterminds/semver" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" @@ -129,6 +130,12 @@ func (d *Docker) createDockerfile(tool *Tool, prmConfig Config) string { dockerfile := strings.Builder{} dockerfile.WriteString(fmt.Sprintf("FROM puppet/puppet-agent:%s\n", prmConfig.PuppetVersion.String())) + rubyVersion := getRubyVersion(prmConfig.PuppetVersion) + + if prmConfig.PuppetVersion.Major() == 5 { + dockerfile.WriteString("RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 4528B6CD9E61EF26\n") + } + if tool.Cfg.Common.RequiresGit || (tool.Cfg.Gem != nil && tool.Cfg.Gem.BuildTools) { dockerfile.WriteString("RUN apt update\n") } @@ -145,12 +152,24 @@ func (d *Docker) createDockerfile(tool *Tool, prmConfig Config) string { dockerfile.WriteString("RUN /opt/puppetlabs/puppet/bin/gem install bundler --no-document\n") for _, gem := range tool.Cfg.Gem.Name { + // is there a compatibility matrix? + if len(tool.Cfg.Gem.Compatibility) > 0 { + // is our current version of ruby in the matrix? + if val, ok := tool.Cfg.Gem.Compatibility[rubyVersion]; ok { + // is the gem we want to install listed in the matrix? + if compat, ok := val[gem]; ok { + dockerfile.WriteString(fmt.Sprintf("RUN /opt/puppetlabs/puppet/bin/gem install %s -f --conservative --minimal-deps -v '%s' --no-document\n", gem, compat)) + continue + } + } + } + // just install the latest gem dockerfile.WriteString(fmt.Sprintf("RUN /opt/puppetlabs/puppet/bin/gem install %s -f --conservative --minimal-deps --no-document\n", gem)) } } for key, val := range tool.Cfg.Common.Env { - dockerfile.WriteString(fmt.Sprintf("ENV %s=%s\n", key, val)) + dockerfile.WriteString(fmt.Sprintf("ENV %s=\"%s\"\n", key, val)) } // Copy the tools content into the image @@ -208,11 +227,17 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo log.Info().Msgf("Additional Args: %v", args) // stand up a container - resp, err := d.OrigClient.ContainerCreate(d.Context, &container.Config{ + containerConf := container.Config{ Image: d.ImageName(tool, prmConfig), - Cmd: args, Tty: false, - }, + } + // args can override the default CMD + if len(args) > 0 { + containerConf.Cmd = args + } + + resp, err := d.OrigClient.ContainerCreate(d.Context, + &containerConf, &container.HostConfig{ Mounts: []mount.Mount{ { @@ -315,3 +340,16 @@ func (d *Docker) Status() BackendStatus { StatusMsg: status, } } + +func getRubyVersion(puppet *semver.Version) float32 { + switch puppet.Major() { + case 7: + return 2.7 + case 6: + return 2.5 + case 5: + return 2.4 + default: + return 2.5 + } +} diff --git a/pkg/prm/tool.go b/pkg/prm/tool.go index 6ff8000..df8304d 100644 --- a/pkg/prm/tool.go +++ b/pkg/prm/tool.go @@ -62,10 +62,10 @@ type ContainerConfig struct { } type GemConfig struct { - Name []string `mapstructure:"name"` - Executable string `mapstructure:"executable"` - BuildTools bool `mapstructure:"build_tools"` - Compatibility map[float32][]string `mapstructure:"compatibility"` + Name []string `mapstructure:"name"` + Executable string `mapstructure:"executable"` + BuildTools bool `mapstructure:"build_tools"` + Compatibility map[float32]map[string]string `mapstructure:"compatibility"` } type PuppetConfig struct { From 5875802ab7fd7c7f3f725279ecc3e9549eea9eb9 Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Thu, 9 Dec 2021 13:43:58 +0000 Subject: [PATCH 09/12] (GH-35) Add toolArgs flag This flag allows arguments to be passed to the tool. We are unable to rely on cobra to pass unknown args and flags. --- cmd/exec/exec.go | 14 ++++++++++++-- pkg/prm/exec.go | 4 +++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cmd/exec/exec.go b/cmd/exec/exec.go index e8eb008..ce64901 100644 --- a/cmd/exec/exec.go +++ b/cmd/exec/exec.go @@ -22,6 +22,7 @@ var ( // selectedToolInfo string listTools bool prmApi *prm.Prm + toolArgs string ) func CreateCommand(parent *prm.Prm) *cobra.Command { @@ -66,6 +67,10 @@ func CreateCommand(parent *prm.Prm) *cobra.Command { err = viper.BindPFlag("cachedir", tmp.Flags().Lookup("cachedir")) cobra.CheckErr(err) + tmp.Flags().StringVar(&toolArgs, "toolArgs", "", "Additional arguments to pass to the tool") + err = viper.BindPFlag("toolArgs", tmp.Flags().Lookup("toolArgs")) + cobra.CheckErr(err) + return tmp } @@ -151,6 +156,11 @@ func execute(cmd *cobra.Command, args []string) error { return nil } + var additionalToolArgs []string + if toolArgs != "" { + additionalToolArgs = strings.Split(toolArgs, " ") + } + if selectedTool != "" { // get the tool from the cache cachedTool, ok := prmApi.IsToolAvailable(selectedTool) @@ -158,7 +168,7 @@ func execute(cmd *cobra.Command, args []string) error { return fmt.Errorf("Tool %s not found in cache", selectedTool) } // execute! - err := prmApi.Exec(cachedTool, args[1:]) + err := prmApi.Exec(cachedTool, additionalToolArgs) if err != nil { return err } @@ -177,7 +187,7 @@ func execute(cmd *cobra.Command, args []string) error { if !ok { return fmt.Errorf("Tool %s not found in cache", tool) } - err := prmApi.Exec(cachedTool, args) // todo: do we want to allow folk to specify args from validate.yml? + err := prmApi.Exec(cachedTool, additionalToolArgs) // todo: do we want to allow folk to specify args from validate.yml? if err != nil { return err } diff --git a/pkg/prm/exec.go b/pkg/prm/exec.go index ed05458..405cfe5 100644 --- a/pkg/prm/exec.go +++ b/pkg/prm/exec.go @@ -1,7 +1,9 @@ //nolint:structcheck,unused package prm -import "github.com/rs/zerolog/log" +import ( + "github.com/rs/zerolog/log" +) type ExecExitCode int64 From 77542ee371f903a82aeddc29344dc052bed7a8d6 Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Wed, 8 Dec 2021 16:02:36 +0000 Subject: [PATCH 10/12] (GH-35) Add non-blocking log output --- internal/pkg/mock/backend.go | 4 +-- pkg/prm/docker.go | 49 +++++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/internal/pkg/mock/backend.go b/internal/pkg/mock/backend.go index 6b44da0..fd180b4 100644 --- a/internal/pkg/mock/backend.go +++ b/internal/pkg/mock/backend.go @@ -14,8 +14,8 @@ func (m *MockBackend) Status() prm.BackendStatus { } // Implement when needed -func (m *MockBackend) GetTool(tool *prm.Tool, prmConfig prm.Config) error{ - return nil +func (m *MockBackend) GetTool(tool *prm.Tool, prmConfig prm.Config) error { + return nil } // Implement when needed diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index e8236c7..189850d 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -256,8 +256,12 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo if err != nil { return FAILURE, err } + // the autoremove functionality is too aggressive + // it fires before we can get at the logs defer func() { - err := d.OrigClient.ContainerRemove(d.Context, resp.ID, types.ContainerRemoveOptions{}) + err := d.OrigClient.ContainerRemove(d.Context, resp.ID, types.ContainerRemoveOptions{ + RemoveVolumes: true, + }) if err != nil { log.Error().Msgf("Error removing container: %s", err) } @@ -267,35 +271,44 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo return FAILURE, err } - statusCh, errCh := d.OrigClient.ContainerWait(d.Context, resp.ID, container.WaitConditionNotRunning) - select { - case err := <-errCh: + isDone := make(chan bool) + // Move this wait into a goroutine + // when its finished it will return and post to the isDone channel + go func(d *Docker, isDone chan bool) { + statusCh, errCh := d.OrigClient.ContainerWait(d.Context, resp.ID, container.WaitConditionNotRunning) + select { + case <-errCh: + isDone <- true + case <-statusCh: + isDone <- true + } + }(d, isDone) + + // parse out the containers logs while we wait for the container to finish + for { + out, err := d.OrigClient.ContainerLogs(d.Context, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "2", Follow: true}) if err != nil { return FAILURE, err } - case <-statusCh: - } - out, err := d.OrigClient.ContainerLogs(d.Context, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}) - if err != nil { - return FAILURE, err - } + _, err = stdcopy.StdCopy(os.Stdout, os.Stderr, out) + if err != nil { + return FAILURE, err + } - _, err = stdcopy.StdCopy(os.Stdout, os.Stderr, out) - if err != nil { - return FAILURE, err + if done := <-isDone; done { + return SUCCESS, nil + } } - - return SUCCESS, nil } func (d *Docker) initClient() (err error) { if d.Client == nil { cli, err := dockerClient.NewClientWithOpts(dockerClient.FromEnv) - if err != nil { - return err - } + if err != nil { + return err + } d.Client = cli d.OrigClient = cli // TODO: remove this when we know all the functions that need added to the interface From ce15f1f75e5736208e0f429bc456e1388f9ad6df Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Thu, 9 Dec 2021 13:25:11 +0000 Subject: [PATCH 11/12] (GH-35) Use DockerClientI interface Declare within the interface the used Client function --- go.mod | 1 + internal/pkg/mock/docker.go | 66 +++++++++++++++++++++++++++++++++++++ pkg/prm/docker.go | 29 +++++++++------- pkg/prm/docker_test.go | 41 ++++++----------------- 4 files changed, 95 insertions(+), 42 deletions(-) create mode 100644 internal/pkg/mock/docker.go diff --git a/go.mod b/go.mod index c764f9c..3ff5fae 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/olekukonko/tablewriter v0.0.5 + github.com/opencontainers/image-spec v1.0.1 github.com/puppetlabs/pdkgo v0.0.0-20211208200151-2414a05b08bf github.com/rs/zerolog v1.26.0 github.com/spf13/afero v1.6.0 diff --git a/internal/pkg/mock/docker.go b/internal/pkg/mock/docker.go new file mode 100644 index 0000000..2a92e2c --- /dev/null +++ b/internal/pkg/mock/docker.go @@ -0,0 +1,66 @@ +package mock + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +type DockerClient struct { + Platform string + Version string + ApiVersion string + ErrorString string +} + +func (m *DockerClient) ServerVersion(ctx context.Context) (types.Version, error) { + if m.ErrorString != "" { + return types.Version{}, fmt.Errorf(m.ErrorString) + } + versionInfo := &types.Version{ + Platform: struct{ Name string }{m.Platform}, + Version: m.Version, + APIVersion: m.ApiVersion, + } + return *versionInfo, nil +} + +func (m *DockerClient) ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.ContainerCreateCreatedBody, error) { + return container.ContainerCreateCreatedBody{}, nil +} + +func (m *DockerClient) ContainerLogs(ctx context.Context, container string, options types.ContainerLogsOptions) (io.ReadCloser, error) { + + mockReader := strings.NewReader("FAKE LOG MESSAGES!") + mockReadCloser := io.NopCloser(mockReader) + + return mockReadCloser, nil +} + +func (m *DockerClient) ContainerRemove(ctx context.Context, containerID string, options types.ContainerRemoveOptions) error { + return nil +} + +func (m *DockerClient) ContainerStart(ctx context.Context, containerID string, options types.ContainerStartOptions) error { + return nil +} + +func (m *DockerClient) ContainerWait(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.ContainerWaitOKBody, <-chan error) { + waitChan := make(chan container.ContainerWaitOKBody) + errChan := make(chan error) + return waitChan, errChan +} + +func (m *DockerClient) ImageBuild(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) { + return types.ImageBuildResponse{}, nil +} + +func (m *DockerClient) ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error) { + return []types.ImageSummary{}, nil +} diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index 189850d..d215b56 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -14,21 +14,29 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/network" dockerClient "github.com/docker/docker/client" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/stdcopy" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog/log" ) type Docker struct { // We need to be able to mock the docker client in testing - Client DockerClientI - OrigClient *dockerClient.Client - Context context.Context + Client DockerClientI + Context context.Context } type DockerClientI interface { // All docker client functions must be noted here so they can be mocked + ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.ContainerCreateCreatedBody, error) + ContainerLogs(ctx context.Context, container string, options types.ContainerLogsOptions) (io.ReadCloser, error) + ContainerRemove(ctx context.Context, containerID string, options types.ContainerRemoveOptions) error + ContainerStart(ctx context.Context, containerID string, options types.ContainerStartOptions) error + ContainerWait(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.ContainerWaitOKBody, <-chan error) + ImageBuild(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) + ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error) ServerVersion(context.Context) (types.Version, error) } @@ -44,7 +52,7 @@ func (d *Docker) GetTool(tool *Tool, prmConfig Config) error { toolImageName := d.ImageName(tool, prmConfig) // find out if docker knows about our tool - list, err := d.OrigClient.ImageList(d.Context, types.ImageListOptions{}) + list, err := d.Client.ImageList(d.Context, types.ImageListOptions{}) if err != nil { log.Debug().Msgf("Error listing images: %v", err) @@ -95,7 +103,7 @@ func (d *Docker) GetTool(tool *Tool, prmConfig Config) error { } // build the image - imageBuildResponse, err := d.OrigClient.ImageBuild( + imageBuildResponse, err := d.Client.ImageBuild( d.Context, tar, types.ImageBuildOptions{ @@ -236,7 +244,7 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo containerConf.Cmd = args } - resp, err := d.OrigClient.ContainerCreate(d.Context, + resp, err := d.Client.ContainerCreate(d.Context, &containerConf, &container.HostConfig{ Mounts: []mount.Mount{ @@ -259,7 +267,7 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo // the autoremove functionality is too aggressive // it fires before we can get at the logs defer func() { - err := d.OrigClient.ContainerRemove(d.Context, resp.ID, types.ContainerRemoveOptions{ + err := d.Client.ContainerRemove(d.Context, resp.ID, types.ContainerRemoveOptions{ RemoveVolumes: true, }) if err != nil { @@ -267,7 +275,7 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo } }() - if err := d.OrigClient.ContainerStart(d.Context, resp.ID, types.ContainerStartOptions{}); err != nil { + if err := d.Client.ContainerStart(d.Context, resp.ID, types.ContainerStartOptions{}); err != nil { return FAILURE, err } @@ -275,7 +283,7 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo // Move this wait into a goroutine // when its finished it will return and post to the isDone channel go func(d *Docker, isDone chan bool) { - statusCh, errCh := d.OrigClient.ContainerWait(d.Context, resp.ID, container.WaitConditionNotRunning) + statusCh, errCh := d.Client.ContainerWait(d.Context, resp.ID, container.WaitConditionNotRunning) select { case <-errCh: isDone <- true @@ -286,7 +294,7 @@ func (d *Docker) Exec(tool *Tool, args []string, prmConfig Config, paths Directo // parse out the containers logs while we wait for the container to finish for { - out, err := d.OrigClient.ContainerLogs(d.Context, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "2", Follow: true}) + out, err := d.Client.ContainerLogs(d.Context, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "2", Follow: true}) if err != nil { return FAILURE, err } @@ -311,7 +319,6 @@ func (d *Docker) initClient() (err error) { } d.Client = cli - d.OrigClient = cli // TODO: remove this when we know all the functions that need added to the interface d.Context = context.Background() } return nil diff --git a/pkg/prm/docker_test.go b/pkg/prm/docker_test.go index cad1049..91351cd 100644 --- a/pkg/prm/docker_test.go +++ b/pkg/prm/docker_test.go @@ -1,44 +1,23 @@ package prm_test import ( - "context" - "fmt" "reflect" "testing" - "github.com/docker/docker/api/types" + "github.com/puppetlabs/prm/internal/pkg/mock" "github.com/puppetlabs/prm/pkg/prm" ) -type MockDockerClient struct { - platform string - version string - apiVersion string - errorString string -} - -func (m *MockDockerClient) ServerVersion(ctx context.Context) (types.Version, error) { - if m.errorString != "" { - return types.Version{}, fmt.Errorf(m.errorString) - } - versionInfo := &types.Version{ - Platform: struct{ Name string }{m.platform}, - Version: m.version, - APIVersion: m.apiVersion, - } - return *versionInfo, nil -} - func TestDocker_Status(t *testing.T) { tests := []struct { name string - mockClient MockDockerClient + mockClient mock.DockerClient want prm.BackendStatus }{ { name: "When connection unavailable", - mockClient: MockDockerClient{ - errorString: "error during connect: This error may indicate that the docker daemon is not running.: Get \"http://%2F%2F.%2Fpipe%2Fdocker_engine/v1.41/version\": open //./pipe/docker_engine: The system cannot find the file specified.", + mockClient: mock.DockerClient{ + ErrorString: "error during connect: This error may indicate that the docker daemon is not running.: Get \"http://%2F%2F.%2Fpipe%2Fdocker_engine/v1.41/version\": open //./pipe/docker_engine: The system cannot find the file specified.", }, want: prm.BackendStatus{ IsAvailable: false, @@ -47,8 +26,8 @@ func TestDocker_Status(t *testing.T) { }, { name: "When an edge case failure occurs", - mockClient: MockDockerClient{ - errorString: "Something has gone terribly wrong!", + mockClient: mock.DockerClient{ + ErrorString: "Something has gone terribly wrong!", }, want: prm.BackendStatus{ IsAvailable: false, @@ -57,10 +36,10 @@ func TestDocker_Status(t *testing.T) { }, { name: "When everything is working", - mockClient: MockDockerClient{ - platform: "Docker", - version: "1.2.3", - apiVersion: "3.2.1", + mockClient: mock.DockerClient{ + Platform: "Docker", + Version: "1.2.3", + ApiVersion: "3.2.1", }, want: prm.BackendStatus{ IsAvailable: true, From 7fa418017c45580bf18beaf0d92d5b859a3dda5d Mon Sep 17 00:00:00 2001 From: Dave Armstrong Date: Thu, 9 Dec 2021 13:09:14 +0000 Subject: [PATCH 12/12] (GH-35) Add timeout duration to context used by Docker client --- pkg/prm/docker.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/prm/docker.go b/pkg/prm/docker.go index d215b56..07eac75 100644 --- a/pkg/prm/docker.go +++ b/pkg/prm/docker.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/Masterminds/semver" "github.com/docker/docker/api/types" @@ -24,8 +25,9 @@ import ( type Docker struct { // We need to be able to mock the docker client in testing - Client DockerClientI - Context context.Context + Client DockerClientI + Context context.Context + ContextCancel func() } type DockerClientI interface { @@ -318,8 +320,11 @@ func (d *Docker) initClient() (err error) { return err } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + d.Client = cli - d.Context = context.Background() + d.Context = ctx + d.ContextCancel = cancel } return nil }