diff --git a/Gopkg.lock b/Gopkg.lock index 489b75b77..77514fa60 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -266,7 +266,8 @@ version = "v1.1.1" [[projects]] - digest = "1:397a47c688355450dd1ea9aea8795b3774d5278c8b37199ad4068758dd075956" + branch = "fix/custom-validators" + digest = "1:82dc259bd94f991a4908898f18c419ea83618f3e1d0d1e844cd744058c1fb7e5" name = "github.com/deislabs/cnab-go" packages = [ "action", @@ -283,8 +284,8 @@ "utils/crud", ] pruneopts = "NT" - revision = "93515c713a91d6da48b5a9c68e4b0502d8d39963" - version = "v0.4.0-beta1" + revision = "f51437c7d0babe80355d38e3e103a3a720ecaa97" + source = "github.com/vdice/cnab-go" [[projects]] digest = "1:7a6852b35eb5bbc184561443762d225116ae630c26a7c4d90546619f1e7d2ad2" diff --git a/Gopkg.toml b/Gopkg.toml index cd6011ee8..68b61f745 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,10 +1,12 @@ [[constraint]] name = "github.com/deislabs/cnab-go" - version = "v0.4.0-beta1" + source = "github.com/vdice/cnab-go" + branch = "fix/custom-validators" [[override]] name = "github.com/deislabs/cnab-go" - version = "v0.4.0-beta1" + source = "github.com/vdice/cnab-go" + branch = "fix/custom-validators" # Using master until there is a release of cnab-to-oci [[constraint]] diff --git a/pkg/cnab/config_adapter/adapter.go b/pkg/cnab/config_adapter/adapter.go index ba3364731..ad9d840ad 100644 --- a/pkg/cnab/config_adapter/adapter.go +++ b/pkg/cnab/config_adapter/adapter.go @@ -163,6 +163,10 @@ func (c *ManifestConverter) generateBundleParameters(defs *definition.Definition // (Both Params and Outputs may reference same Definition) if _, exists := (*defs)[param.Name]; !exists { def := param.Schema + if def.Type == "file" { + def.Type = "string" + def.ContentEncoding = "base64" + } (*defs)[param.Name] = &def } params[param.Name] = p diff --git a/pkg/cnab/config_adapter/adapter_test.go b/pkg/cnab/config_adapter/adapter_test.go index ad88ff08d..85d1cb2db 100644 --- a/pkg/cnab/config_adapter/adapter_test.go +++ b/pkg/cnab/config_adapter/adapter_test.go @@ -172,6 +172,20 @@ func TestManifestConverter_generateBundleParametersSchema(t *testing.T) { Default: `"myobject": { "foo": "true", "bar": [ 1, 2, 3 ] }`, }, }, + { + "afile", + bundle.Parameter{ + Definition: "afile", + Destination: &bundle.Location{ + Path: "/root/.kube/config", + }, + Required: true, + }, + definition.Schema{ + Type: "string", + ContentEncoding: "base64", + }, + }, } for _, tc := range testcases { diff --git a/pkg/cnab/config_adapter/testdata/porter-with-parameters.yaml b/pkg/cnab/config_adapter/testdata/porter-with-parameters.yaml index acb947e99..86579080b 100644 --- a/pkg/cnab/config_adapter/testdata/porter-with-parameters.yaml +++ b/pkg/cnab/config_adapter/testdata/porter-with-parameters.yaml @@ -47,6 +47,9 @@ parameters: 3 ] }' + - name: afile + type: file + path: /root/.kube/config mixins: - exec diff --git a/pkg/cnab/provider/parameters.go b/pkg/cnab/provider/parameters.go index fd46f8873..0a5f72f52 100644 --- a/pkg/cnab/provider/parameters.go +++ b/pkg/cnab/provider/parameters.go @@ -1,9 +1,11 @@ package cnabprovider import ( + "encoding/base64" "fmt" "github.com/deislabs/cnab-go/bundle" + "github.com/deislabs/cnab-go/bundle/definition" "github.com/deislabs/cnab-go/claim" "github.com/pkg/errors" ) @@ -25,7 +27,12 @@ func (d *Duffle) loadParameters(claim *claim.Claim, rawOverrides map[string]stri return nil, fmt.Errorf("definition %s not defined in bundle", param.Definition) } - value, err := def.ConvertValue(rawValue) + unconverted, err := d.getUnconvertedValueFromRaw(def, key, rawValue) + if err != nil { + return nil, err + } + + value, err := def.ConvertValue(unconverted) if err != nil { return nil, errors.Wrapf(err, "unable to convert parameter's %s value %s to the destination parameter type %s", key, rawValue, def.Type) } @@ -63,6 +70,20 @@ func (d *Duffle) loadParameters(claim *claim.Claim, rawOverrides map[string]stri return bundle.ValuesOrDefaults(overrides, bun) } +func (d *Duffle) getUnconvertedValueFromRaw(def *definition.Schema, key, rawValue string) (string, error) { + // the parameter value (via rawValue) may represent a file on the local filesystem + if def.Type == "string" && def.ContentEncoding == "base64" { + if _, err := d.FileSystem.Stat(rawValue); err == nil { + bytes, err := d.FileSystem.ReadFile(rawValue) + if err != nil { + return "", errors.Wrapf(err, "unable to read file parameter %s", key) + } + return base64.StdEncoding.EncodeToString(bytes), nil + } + } + return rawValue, nil +} + // TODO: remove in favor of cnab-go logic: https://github.com/deislabs/cnab-go/pull/99 func appliesToAction(action string, parameter bundle.Parameter) bool { if len(parameter.ApplyTo) == 0 { diff --git a/pkg/cnab/provider/parameters_test.go b/pkg/cnab/provider/parameters_test.go index cb16d50c3..95c3db2d9 100644 --- a/pkg/cnab/provider/parameters_test.go +++ b/pkg/cnab/provider/parameters_test.go @@ -189,3 +189,40 @@ func Test_loadParameters_requiredButDoesNotApply(t *testing.T) { require.Equal(t, "foo-claim-value", params["foo"], "expected param 'foo' to be the bundle default") } + +func Test_loadParameters_fileParameter(t *testing.T) { + c := config.NewTestConfig(t) + d := NewDuffle(c.Config) + + c.TestContext.AddTestFile("testdata/file-param", "/path/to/file") + + claim, err := claim.New("test") + require.NoError(t, err) + + claim.Bundle = &bundle.Bundle{ + Definitions: definition.Definitions{ + "foo": &definition.Schema{ + Type: "string", + ContentEncoding: "base64", + }, + }, + Parameters: map[string]bundle.Parameter{ + "foo": bundle.Parameter{ + Definition: "foo", + Required: true, + Destination: &bundle.Location{ + Path: "/tmp/foo", + }, + }, + }, + } + + overrides := map[string]string{ + "foo": "/path/to/file", + } + + params, err := d.loadParameters(claim, overrides, "action") + require.NoError(t, err) + + require.Equal(t, "SGVsbG8gV29ybGQh", params["foo"], "expected param 'foo' to be the base64-encoded file contents") +} diff --git a/pkg/cnab/provider/testdata/file-param b/pkg/cnab/provider/testdata/file-param new file mode 100644 index 000000000..c57eff55e --- /dev/null +++ b/pkg/cnab/provider/testdata/file-param @@ -0,0 +1 @@ +Hello World! \ No newline at end of file diff --git a/pkg/config/manifest.go b/pkg/config/manifest.go index 87afcf499..782206822 100644 --- a/pkg/config/manifest.go +++ b/pkg/config/manifest.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "io/ioutil" "net/http" "reflect" @@ -88,6 +89,13 @@ func (m *Manifest) Validate() error { } } + for _, parameter := range m.Parameters { + err = parameter.Validate() + if err != nil { + result = multierror.Append(result, err) + } + } + return result } @@ -104,6 +112,45 @@ type ParameterDefinition struct { definition.Schema `yaml:",inline"` } +func (pd *ParameterDefinition) Validate() error { + var result *multierror.Error + + if pd.Name == "" { + result = multierror.Append(result, errors.New("parameter name is required")) + } + + // Porter supports declaring a parameter of type: "file", + // which we will convert to the appropriate bundle.Parameter type in adapter.go + // Here, we copy the ParameterDefinition and make the same modification before validation + pdCopy := pd.DeepCopy() + if pdCopy.Type == "file" { + if pd.Destination.Path == "" { + result = multierror.Append(result, fmt.Errorf("no destination path supplied for parameter %s", pd.Name)) + } + pdCopy.Type = "string" + pdCopy.ContentEncoding = "base64" + } + + schemaValidationErrs, err := pdCopy.Schema.Validate(pdCopy) + if err != nil { + result = multierror.Append(result, errors.Wrapf(err, "encountered error while validating parameter %s", pdCopy.Name)) + } + for _, schemaValidationErr := range schemaValidationErrs { + result = multierror.Append(result, errors.Wrapf(err, "encountered validation error(s) for parameter %s: %v", pdCopy.Name, schemaValidationErr)) + } + + return result.ErrorOrNil() +} + +// DeepCopy copies a ParameterDefinition and returns the copy +func (pd *ParameterDefinition) DeepCopy() *ParameterDefinition { + var p2 ParameterDefinition + p2 = *pd + p2.ApplyTo = make([]string, len(pd.ApplyTo)) + copy(p2.ApplyTo, pd.ApplyTo) + return &p2 +} + type CredentialDefinition struct { Name string `yaml:"name"` Description string `yaml:"description,omitempty"` @@ -239,7 +286,13 @@ func (od *OutputDefinition) Validate() error { return errors.New("output name is required") } - // TODO: Validate inline Schema + schemaValidationErrs, err := od.Schema.Validate(od) + if err != nil { + return errors.Wrapf(err, "encountered error while validating output %s", od.Name) + } + if len(schemaValidationErrs) != 0 { + return errors.Wrapf(err, "encountered validation error(s) for output %s: %v", od.Name, schemaValidationErrs) + } return nil } diff --git a/pkg/config/manifest_test.go b/pkg/config/manifest_test.go index 6376b6daf..ab915a907 100644 --- a/pkg/config/manifest_test.go +++ b/pkg/config/manifest_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "testing" + "github.com/deislabs/cnab-go/bundle/definition" "gopkg.in/yaml.v2" "github.com/stretchr/testify/assert" @@ -82,3 +83,25 @@ func TestMixinDeclaration_MarshalYAML(t *testing.T) { assert.Equal(t, string(wantYaml), string(gotYaml)) } + +func TestValidateParameterDefinition(t *testing.T) { + pd := ParameterDefinition{ + Name: "myparam", + Schema: definition.Schema{ + Type: "file", + }, + } + + pd.Destination = Location{} + + err := pd.Validate() + assert.EqualError(t, err, `1 error occurred: + * no destination path supplied for parameter myparam + +`) + + pd.Destination.Path = "/path/to/file" + + err = pd.Validate() + assert.NoError(t, err) +} diff --git a/pkg/config/runtime-manifest.go b/pkg/config/runtime-manifest.go index 7b1761d31..c336abd5d 100644 --- a/pkg/config/runtime-manifest.go +++ b/pkg/config/runtime-manifest.go @@ -1,6 +1,7 @@ package config import ( + "encoding/base64" "fmt" "os" "reflect" @@ -85,7 +86,6 @@ func resolveParameter(pd ParameterDefinition) (string, error) { return pd.Destination.Path, nil } return "", fmt.Errorf("parameter: %s is malformed", pd.Name) - } func resolveCredential(cd CredentialDefinition) (string, error) { @@ -275,3 +275,33 @@ func (m *RuntimeManifest) ResolveStep(step *Step) error { return nil } + +// Prepare prepares the runtime environment prior to step execution +func (m *RuntimeManifest) Prepare() error { + // For parameters of type "file", we may need to decode files on the filesystem + // before execution of the step/action + for _, param := range m.Parameters { + if param.Type == "file" { + if param.Destination.Path == "" { + return fmt.Errorf("destination path is not supplied for parameter %s", param.Name) + } + + // Porter by default places parameter value into file determined by Destination.Path + bytes, err := m.FileSystem.ReadFile(param.Destination.Path) + if err != nil { + return fmt.Errorf("unable to acquire value for parameter %s", param.Name) + } + + decoded, err := base64.StdEncoding.DecodeString(string(bytes)) + if err != nil { + return errors.Wrapf(err, "unable to decode parameter %s", param.Name) + } + + err = m.FileSystem.WriteFile(param.Destination.Path, decoded, os.ModePerm) + if err != nil { + return errors.Wrapf(err, "unable to write decoded parameter %s", param.Name) + } + } + } + return nil +} diff --git a/pkg/config/runtime-manifest_test.go b/pkg/config/runtime-manifest_test.go index 17edfef3f..cf73d1a2a 100644 --- a/pkg/config/runtime-manifest_test.go +++ b/pkg/config/runtime-manifest_test.go @@ -167,6 +167,9 @@ func TestResolveMapParam(t *testing.T) { val, ok := pms["Thing"].(string) assert.True(t, ok) assert.Equal(t, "Ralpha", val) + + err = rm.Prepare() + assert.NoError(t, err) } func TestResolvePathParam(t *testing.T) { @@ -232,6 +235,61 @@ func TestResolveMapParamUnknown(t *testing.T) { assert.Equal(t, "unable to resolve step: unable to render template Parameters:\n Thing: '{{bundle.parameters.person}}'\ndescription: a test step\n: Missing variable \"person\"", err.Error()) } +func TestPrepare_fileParam(t *testing.T) { + c := NewTestConfig(t) + + c.TestContext.AddTestFile("testdata/file-param", "/path/to/file") + + m := &Manifest{ + Parameters: []ParameterDefinition{ + { + Name: "file-param", + Destination: Location{ + Path: "/path/to/file", + }, + Schema: definition.Schema{ + Type: "file", + Default: "/path/to/file", + }, + }, + }, + } + rm := NewRuntimeManifest(c.Context, ActionInstall, m) + s := &Step{ + Data: map[string]interface{}{ + "description": "a test step", + "Parameters": map[string]interface{}{ + "file-param": "{{bundle.parameters.file-param}}", + }, + }, + } + + before, _ := yaml.Marshal(s) + t.Logf("Before:\n %s", before) + err := rm.ResolveStep(s) + require.NoError(t, err) + after, _ := yaml.Marshal(s) + t.Logf("After:\n %s", after) + assert.NotNil(t, s.Data) + t.Logf("Length of data:%d", len(s.Data)) + assert.NotEmpty(t, s.Data["Parameters"]) + for k, v := range s.Data { + t.Logf("Key %s, value: %s, type: %T", k, v, v) + } + pms, ok := s.Data["Parameters"].(map[interface{}]interface{}) + assert.True(t, ok) + val, ok := pms["file-param"].(string) + assert.True(t, ok) + assert.Equal(t, "/path/to/file", val) + + err = rm.Prepare() + assert.NoError(t, err) + + bytes, err := c.FileSystem.ReadFile("/path/to/file") + assert.NoError(t, err) + assert.Equal(t, "Hello World!", string(bytes), "expected file contents to equal the decoded parameter value") +} + func TestResolveArrayUnknown(t *testing.T) { c := NewTestConfig(t) m := &Manifest{ diff --git a/pkg/config/testdata/file-param b/pkg/config/testdata/file-param new file mode 100644 index 000000000..3d948d13a --- /dev/null +++ b/pkg/config/testdata/file-param @@ -0,0 +1 @@ +SGVsbG8gV29ybGQh \ No newline at end of file diff --git a/pkg/porter/run.go b/pkg/porter/run.go index 96290d5c1..4d0048cce 100644 --- a/pkg/porter/run.go +++ b/pkg/porter/run.go @@ -93,6 +93,14 @@ func (p *Porter) Run(opts RunOptions) error { return err } + // Prepare prepares the runtime environment prior to step execution. + // As an example, for parameters of type "file", we may need to decode file contents + // on the filesystem before execution of the step/action + err = runtimeManifest.Prepare() + if err != nil { + return err + } + err = p.FileSystem.MkdirAll(context.MixinOutputsDir, 0755) if err != nil { return errors.Wrapf(err, "could not create outputs directory %s", context.MixinOutputsDir) diff --git a/pkg/porter/testdata/porter.yaml b/pkg/porter/testdata/porter.yaml index db40d93e9..477d83bf0 100644 --- a/pkg/porter/testdata/porter.yaml +++ b/pkg/porter/testdata/porter.yaml @@ -16,7 +16,7 @@ credentials: parameters: - name: my-first-param - type: int + type: integer default: 9 env: MY_FIRST_PARAM - name: my-second-param diff --git a/tests/install_test.go b/tests/install_test.go index 9f484d6f7..bec6c0f05 100644 --- a/tests/install_test.go +++ b/tests/install_test.go @@ -41,3 +41,31 @@ func TestInstall_relativePathPorterHome(t *testing.T) { err = p.InstallBundle(installOpts) require.NoError(t, err) } + +func TestInstall_fileParam(t *testing.T) { + p := porter.NewTestPorter(t) + p.SetupIntegrationTest() + defer p.CleanupIntegrationTest() + p.Debug = false + + p.TestConfig.TestContext.AddTestFile(filepath.Join(p.TestDir, "testdata/bundle-with-file-params.yaml"), "porter.yaml") + p.TestConfig.TestContext.AddTestFile(filepath.Join(p.TestDir, "testdata/myfile"), "./myfile") + + installOpts := porter.InstallOptions{} + installOpts.Insecure = true + installOpts.Params = []string{"myfile=./myfile"} + + err := installOpts.Validate([]string{}, p.Context) + require.NoError(t, err) + + err = p.InstallBundle(installOpts) + require.NoError(t, err) + + // TODO: We can't check this yet because docker driver is printing directly to stdout instead of to the given writer + // output := p.TestConfig.TestContext.GetOutput() + // require.Contains(t, output, "Hello World!", "expected action output to contain provided file contents") + + claim, err := p.CNAB.FetchClaim(p.Manifest.Name) + require.NoError(t, err, "could not fetch claim") + require.Equal(t, "Hello World!", claim.Outputs["myfile"], "expected output to match the decoded file contents") +} \ No newline at end of file diff --git a/tests/testdata/bundle-with-file-params.yaml b/tests/testdata/bundle-with-file-params.yaml new file mode 100644 index 000000000..42bc045fd --- /dev/null +++ b/tests/testdata/bundle-with-file-params.yaml @@ -0,0 +1,43 @@ +name: mybun +version: 0.1.0 +description: "An example Porter configuration" +invocationImage: porter-hello:latest +tag: deislabs/porter-hello-bundle:latest + +mixins: + - exec + +parameters: + - name: myfile + type: file + path: /root/myfile + +outputs: + - name: myfile + type: string + applyTo: + - install + +install: + - exec: + description: "Install Hello World" + command: bash + flags: + c: cat /root/myfile + outputs: + - name: myfile + path: /root/myfile + +upgrade: + - exec: + description: "World 2.0" + command: bash + flags: + c: cat /root/myfile + +uninstall: + - exec: + description: "Uninstall Hello World" + command: bash + flags: + c: cat /root/myfile diff --git a/tests/testdata/myfile b/tests/testdata/myfile new file mode 100644 index 000000000..c57eff55e --- /dev/null +++ b/tests/testdata/myfile @@ -0,0 +1 @@ +Hello World! \ No newline at end of file diff --git a/vendor/github.com/deislabs/cnab-go/bundle/definition/schema.go b/vendor/github.com/deislabs/cnab-go/bundle/definition/schema.go index e756cc34c..25269f7c6 100644 --- a/vendor/github.com/deislabs/cnab-go/bundle/definition/schema.go +++ b/vendor/github.com/deislabs/cnab-go/bundle/definition/schema.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/pkg/errors" - "github.com/qri-io/jsonschema" ) type Definitions map[string]*Schema @@ -96,7 +95,7 @@ func (s *Schema) UnmarshalJSON(data []byte) error { // Before we unmarshal into the cnab-go bundle/definition/Schema type, unmarshal into // the library struct so we can handle any validation errors in the schema. If there // are any errors, return those. - js := new(jsonschema.RootSchema) + js := NewRootSchema() if err := js.UnmarshalJSON(data); err != nil { return err } diff --git a/vendor/github.com/deislabs/cnab-go/bundle/definition/validation.go b/vendor/github.com/deislabs/cnab-go/bundle/definition/validation.go index 4a1511e2d..058d9f92b 100644 --- a/vendor/github.com/deislabs/cnab-go/bundle/definition/validation.go +++ b/vendor/github.com/deislabs/cnab-go/bundle/definition/validation.go @@ -4,7 +4,6 @@ import ( "encoding/json" "github.com/pkg/errors" - "github.com/qri-io/jsonschema" ) // ValidationError error represents a validation error @@ -24,7 +23,7 @@ func (s *Schema) Validate(data interface{}) ([]ValidationError, error) { if err != nil { return nil, errors.Wrap(err, "unable to load schema") } - def := new(jsonschema.RootSchema) + def := NewRootSchema() err = json.Unmarshal([]byte(b), def) if err != nil { return nil, errors.Wrap(err, "unable to build schema") diff --git a/vendor/github.com/deislabs/cnab-go/bundle/definition/validators.go b/vendor/github.com/deislabs/cnab-go/bundle/definition/validators.go new file mode 100644 index 000000000..18d990cea --- /dev/null +++ b/vendor/github.com/deislabs/cnab-go/bundle/definition/validators.go @@ -0,0 +1,29 @@ +package definition + +import ( + "github.com/qri-io/jsonschema" +) + +// ContentEncoding represents a "custom" Schema property +type ContentEncoding string + +// NewContentEncoding allocates a new ContentEncoding validator +func NewContentEncoding() jsonschema.Validator { + return new(ContentEncoding) +} + +// Validate implements the Validator interface for ContentEncoding +// Currently, this is a no-op and is only used to register with the jsonschema library +// that 'contentEncoding' is a valid property (as of writing, it isn't one of the defaults) +func (c ContentEncoding) Validate(propPath string, data interface{}, errs *[]jsonschema.ValError) {} + +// NewRootSchema returns a jsonschema.RootSchema with any needed custom +// jsonschema.Validators pre-registered +func NewRootSchema() *jsonschema.RootSchema { + // Register custom validators here + // Note: as of writing, jsonschema doesn't have a stock validator for intances of type `contentEncoding` + // There may be others missing in the library that exist in http://json-schema.org/draft-07/schema# + // and thus, we'd need to create/register them here (if not included upstream) + jsonschema.RegisterValidator("contentEncoding", NewContentEncoding) + return new(jsonschema.RootSchema) +} diff --git a/vendor/github.com/deislabs/cnab-go/claim/claim.go b/vendor/github.com/deislabs/cnab-go/claim/claim.go index 1355c5122..95c832023 100644 --- a/vendor/github.com/deislabs/cnab-go/claim/claim.go +++ b/vendor/github.com/deislabs/cnab-go/claim/claim.go @@ -48,7 +48,7 @@ type Claim struct { } // ValidName is a regular expression that indicates whether a name is a valid claim name. -var ValidName = regexp.MustCompile("^[a-zA-Z0-9_-]+$") +var ValidName = regexp.MustCompile("^[a-zA-Z0-9._-]+$") // New creates a new Claim initialized for an installation operation. func New(name string) (*Claim, error) {