From 399af6f9d955798230729783c6f7cefb73c67c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Reigota?= Date: Mon, 8 Nov 2021 14:56:28 +0000 Subject: [PATCH] feat(dockerfile): Added Ignore lines by comments to Dockerfile #4420 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: João Reigota --- pkg/model/model.go | 2 +- pkg/parser/docker/comments.go | 111 ++++++++++++ pkg/parser/docker/comments_test.go | 281 +++++++++++++++++++++++++++++ pkg/parser/docker/parser.go | 9 + pkg/parser/docker/parser_test.go | 2 + 5 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 pkg/parser/docker/comments.go create mode 100644 pkg/parser/docker/comments_test.go diff --git a/pkg/model/model.go b/pkg/model/model.go index d74186b1039..d7cab19e637 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -62,7 +62,7 @@ var ( var ( // KICSCommentRgxp is the regexp to identify if a comment is a KICS comment - KICSCommentRgxp = regexp.MustCompile(`^((/{2})|#)\s*kics\s*`) + KICSCommentRgxp = regexp.MustCompile(`^((/{2})|#)*\s*kics\s*`) ) // Version - is the model for the version response diff --git a/pkg/parser/docker/comments.go b/pkg/parser/docker/comments.go new file mode 100644 index 00000000000..72103d3eff4 --- /dev/null +++ b/pkg/parser/docker/comments.go @@ -0,0 +1,111 @@ +package docker + +import ( + "strings" + + "github.com/Checkmarx/kics/pkg/model" + "github.com/moby/buildkit/frontend/dockerfile/parser" +) + +// ignore is a structure that contains information about the lines that are being ignored. +type ignore struct { + from map[string]bool + lines []int +} + +// newIgnore returns a new ignore struct. +func newIgnore() *ignore { + return &ignore{ + from: make(map[string]bool), + lines: make([]int, 0), + } +} + +// setIgnore adds a new entry to the ignore struct for the 'FROM' block to be ignored +func (i *ignore) setIgnore(from string) { + i.from[from] = true +} + +// ignoreBlock adds block lines to be ignored to the ignore struct. +func (i *ignore) ignoreBlock(node *parser.Node, from string) { + if _, ok := i.from[from]; ok { + i.lines = append(i.lines, commentRange(node.StartLine, node.EndLine)...) + } +} + +// getIgnoreLines returns the lines that are being ignored. +func (i *ignore) getIgnoreLines() []int { // nolint + return removeDups(i.lines) +} + +// removeDups removes duplicates from a slice of ints. +func removeDups(lines []int) []int { // nolint + seen := make(map[int]bool) + var result []int + for _, line := range lines { + if !seen[line] { + result = append(result, line) + seen[line] = true + } + } + return result +} + +// getIgnoreComments returns lines to be ignored for each node of the dockerfile +func (i *ignore) getIgnoreComments(node *parser.Node) (ignore bool) { + if len(node.PrevComment) == 0 { + return false + } + + for idx, comment := range node.PrevComment { + switch processComment(comment) { + case model.IgnoreLine: + i.lines = append(i.lines, commentRange(node.StartLine-(idx+1), node.EndLine)...) + case model.IgnoreBlock: + i.lines = append(i.lines, node.StartLine-(idx+1)) + ignore = true + default: + i.lines = append(i.lines, node.StartLine-(idx+1)) + } + } + + return +} + +// processComment returns the type of comment given. +func processComment(comment string) (value model.CommentCommand) { + commentLower := strings.ToLower(comment) + + if model.KICSCommentRgxp.MatchString(commentLower) { + commentLower = model.KICSCommentRgxp.ReplaceAllString(commentLower, "") + commands := strings.Split(strings.Trim(commentLower, "\n"), " ") + value = processCommands(commands) + return + } + return model.CommentCommand(comment) +} + +// processCommands goes over kics commands in a line and returns the type of command given +func processCommands(commands []string) model.CommentCommand { + for _, command := range commands { + switch com := model.CommentCommand(command); com { + case model.IgnoreLine: + return model.IgnoreLine + case model.IgnoreBlock: + return model.IgnoreBlock + default: + continue + } + } + + return model.CommentCommand(commands[0]) +} + +// commentRange returns the range of the comment between the start and end lines. +func commentRange(start, end int) (lines []int) { + lines = make([]int, end-start+1) + for i := range lines { + lines[i] = start + i + } + return +} diff --git a/pkg/parser/docker/comments_test.go b/pkg/parser/docker/comments_test.go new file mode 100644 index 00000000000..98a6af86f85 --- /dev/null +++ b/pkg/parser/docker/comments_test.go @@ -0,0 +1,281 @@ +package docker + +import ( + "reflect" + "testing" + + "github.com/moby/buildkit/frontend/dockerfile/parser" + "github.com/stretchr/testify/require" +) + +func Test_newIgnore(t *testing.T) { + tests := []struct { + name string + want *ignore + }{ + { + name: "new ignore", + want: &ignore{ + from: make(map[string]bool), + lines: make([]int, 0), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := newIgnore(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("newIgnore() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_ignore_setIgnore(t *testing.T) { + type fields struct { + from map[string]bool + lines []int + } + type args struct { + from string + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "set ignore", + fields: fields{ + from: make(map[string]bool), + lines: make([]int, 0), + }, + args: args{ + from: "testing_lines", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &ignore{ + from: tt.fields.from, + lines: tt.fields.lines, + } + i.setIgnore(tt.args.from) + _, ok := i.from[tt.args.from] + require.True(t, ok) + }) + } +} + +func Test_ignore_ignoreBlock(t *testing.T) { + type fields struct { + from map[string]bool + lines []int + } + type args struct { + node *parser.Node + from string + } + tests := []struct { + name string + fields fields + args args + want []int + }{ + { + name: "ignore block: should add", + fields: fields{ + from: map[string]bool{ + "testing_lines": true, + }, + lines: []int{1, 2, 3}, + }, + args: args{ + node: &parser.Node{ + StartLine: 5, + EndLine: 5, + }, + from: "testing_lines", + }, + want: []int{1, 2, 3, 5}, + }, + { + name: "ignore block: should not add", + fields: fields{ + from: map[string]bool{ + "testing_lines": true, + }, + lines: []int{1, 2, 3}, + }, + args: args{ + node: &parser.Node{ + StartLine: 5, + EndLine: 5, + }, + from: "testing_not_lines", + }, + want: []int{1, 2, 3}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &ignore{ + from: tt.fields.from, + lines: tt.fields.lines, + } + i.ignoreBlock(tt.args.node, tt.args.from) + got := i.getIgnoreLines() + require.Equal(t, tt.want, got) + }) + } +} + +func Test_ignore_getIgnoreLines(t *testing.T) { + type fields struct { + from map[string]bool + lines []int + } + tests := []struct { + name string + fields fields + want []int + }{ + { + name: "get ignore lines: with dups", + fields: fields{ + from: make(map[string]bool), + lines: []int{1, 1, 2, 3, 4}, + }, + want: []int{1, 2, 3, 4}, + }, + { + name: "get ignore lines: without dups", + fields: fields{ + from: make(map[string]bool), + lines: []int{1, 2, 3, 4}, + }, + want: []int{1, 2, 3, 4}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &ignore{ + from: tt.fields.from, + lines: tt.fields.lines, + } + if got := i.getIgnoreLines(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ignore.getIgnoreLines() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_ignore_getIgnoreComments(t *testing.T) { + type fields struct { + from map[string]bool + lines []int + } + type args struct { + node *parser.Node + } + tests := []struct { + name string + fields fields + args args + wantIgnore bool + wantLines []int + }{ + { + name: "get ignore comments: should ignore single line", + fields: fields{ + from: make(map[string]bool), + lines: make([]int, 0), + }, + args: args{ + node: &parser.Node{ + PrevComment: []string{"kics ignore-line"}, + StartLine: 3, + EndLine: 3, + }, + }, + wantIgnore: false, + wantLines: []int{2, 3}, + }, + { + name: "get ignore comments: should ignore multi-line", + fields: fields{ + from: make(map[string]bool), + lines: make([]int, 0), + }, + args: args{ + node: &parser.Node{ + PrevComment: []string{"kics ignore-line"}, + StartLine: 3, + EndLine: 6, + }, + }, + wantIgnore: false, + wantLines: []int{2, 3, 4, 5, 6}, + }, + { + name: "get ignore comments: should ignore comment multi-line", + fields: fields{ + from: make(map[string]bool), + lines: make([]int, 0), + }, + args: args{ + node: &parser.Node{ + PrevComment: []string{"kics ignore-line", "kics regular command"}, + StartLine: 3, + EndLine: 6, + }, + }, + wantIgnore: false, + wantLines: []int{2, 3, 4, 5, 6, 1}, + }, + { + name: "get ignore comments: should ignore regular comment", + fields: fields{ + from: make(map[string]bool), + lines: make([]int, 0), + }, + args: args{ + node: &parser.Node{ + PrevComment: []string{"this is a regular comment"}, + StartLine: 3, + EndLine: 6, + }, + }, + wantIgnore: false, + wantLines: []int{2}, + }, + { + name: "get ignore comments: should return true for ignore block", + fields: fields{ + from: make(map[string]bool), + lines: make([]int, 0), + }, + args: args{ + node: &parser.Node{ + PrevComment: []string{"kics ignore-block"}, + StartLine: 3, + EndLine: 6, + }, + }, + wantIgnore: true, + wantLines: []int{2}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &ignore{ + from: tt.fields.from, + lines: tt.fields.lines, + } + if gotIgnore := i.getIgnoreComments(tt.args.node); gotIgnore != tt.wantIgnore { + t.Errorf("ignore.getIgnoreComments() = %v, want %v", gotIgnore, tt.wantIgnore) + } + require.Equal(t, tt.wantLines, i.getIgnoreLines()) + }) + } +} diff --git a/pkg/parser/docker/parser.go b/pkg/parser/docker/parser.go index 046a0e7a946..5ceab093193 100644 --- a/pkg/parser/docker/parser.go +++ b/pkg/parser/docker/parser.go @@ -48,6 +48,7 @@ func (p *Parser) Parse(_ string, fileContent []byte) ([]model.Document, []int, e fromValue := "args" from := make(map[string][]Command) + ignoreStruct := newIgnore() for _, child := range parsed.AST.Children { child.Value = strings.ToLower(child.Value) @@ -55,6 +56,12 @@ func (p *Parser) Parse(_ string, fileContent []byte) ([]model.Document, []int, e fromValue = strings.TrimPrefix(child.Original, "FROM ") } + if ignoreStruct.getIgnoreComments(child) { + ignoreStruct.setIgnore(fromValue) + } + + ignoreStruct.ignoreBlock(child, fromValue) + cmd := Command{ Cmd: child.Value, Original: child.Original, @@ -91,6 +98,8 @@ func (p *Parser) Parse(_ string, fileContent []byte) ([]model.Document, []int, e documents = append(documents, *doc) + // ignoreLines := ignoreStruct.getIgnoreLines() nolint + return documents, []int{}, nil } diff --git a/pkg/parser/docker/parser_test.go b/pkg/parser/docker/parser_test.go index 85d53406f28..dc4ec6dfa88 100644 --- a/pkg/parser/docker/parser_test.go +++ b/pkg/parser/docker/parser_test.go @@ -43,6 +43,7 @@ func TestParser_Parse(t *testing.T) { ` FROM ubuntu:xenial RUN echo hi > /etc/hi.conf + # kics ignore-line CMD ["echo"] HEALTHCHECK --retries=5 CMD echo hi ONBUILD ADD foo bar @@ -53,6 +54,7 @@ func TestParser_Parse(t *testing.T) { CMD ["echo"] `, ` + # kics ignore-block FROM golang:alpine ENV CGO_ENABLED=0 WORKDIR /app