From bd1db3e30a01dae265f2fa97a3bf122d1aa224c4 Mon Sep 17 00:00:00 2001 From: Egor Leonenko Date: Sun, 5 Mar 2017 17:56:56 +1300 Subject: [PATCH 1/2] Add line_mode - event per line of command output --- _meta/beat.full.yml | 3 ++ _meta/beat.yml | 3 ++ _meta/fields.yml | 27 +++++++++++ _meta/kibana/index-pattern/execbeat.json | 2 +- beater/execevent.go | 18 ++++++- beater/executor.go | 61 ++++++++++++++++++------ config/config.go | 1 + docs/fields.asciidoc | 41 ++++++++++++++++ execbeat.full.yml | 3 ++ execbeat.template-es2x.json | 27 +++++++++++ execbeat.template.json | 23 +++++++++ execbeat.yml | 3 ++ 12 files changed, 195 insertions(+), 17 deletions(-) diff --git a/_meta/beat.full.yml b/_meta/beat.full.yml index 1826f78d..c6333bd1 100644 --- a/_meta/beat.full.yml +++ b/_meta/beat.full.yml @@ -20,3 +20,6 @@ execbeat: # the type defines the document type these entries should be stored # in. Default: execbeat document_type: execbeat + + # Send message for each line of output + line_mode: false diff --git a/_meta/beat.yml b/_meta/beat.yml index 9445f53d..42fdb78f 100644 --- a/_meta/beat.yml +++ b/_meta/beat.yml @@ -20,3 +20,6 @@ execbeat: # the type defines the document type these entries should be stored # in. Default: execbeat #document_type: + + # Send message for each line of output + #line_mode: false diff --git a/_meta/fields.yml b/_meta/fields.yml index fbe80c92..e0235a53 100644 --- a/_meta/fields.yml +++ b/_meta/fields.yml @@ -23,3 +23,30 @@ type: keyword description: > Exit code of the command executed by Execbeat. + - name: line + type: group + fields: + - name: command + type: keyword + description: > + The command executed by Execbeat. + + - name: line + type: keyword + description: > + Single Line from output produced by the command executed by Execbeat. + + - name: line_number + type: long + description: > + Line number of the line from output produced by the command executed by Execbeat. + + - name: source + type: keyword + description: > + Source of the line. Either stdout or stderr. + + - name: exitCode + type: keyword + description: > + Exit code of the command executed by Execbeat. diff --git a/_meta/kibana/index-pattern/execbeat.json b/_meta/kibana/index-pattern/execbeat.json index 51659e2a..93e91e71 100644 --- a/_meta/kibana/index-pattern/execbeat.json +++ b/_meta/kibana/index-pattern/execbeat.json @@ -1,5 +1,5 @@ { - "fields": "[{\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"@timestamp\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"fields\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.provider\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.machine_type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.availability_zone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.project_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.region\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"exec.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"exec.stdout\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"exec.stderr\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"exec.exitCode\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}]", + "fields": "[{\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.name\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.hostname\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"beat.version\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"@timestamp\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"date\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"tags\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"fields\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.provider\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.instance_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.machine_type\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.availability_zone\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.project_id\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"meta.cloud.region\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"exec.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"exec.stdout\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"exec.stderr\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"exec.exitCode\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"line.command\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"line.line\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"line.line_number\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"number\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"line.source\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}, {\"count\": 0, \"analyzed\": false, \"aggregatable\": true, \"name\": \"line.exitCode\", \"searchable\": true, \"indexed\": true, \"doc_values\": true, \"type\": \"string\", \"scripted\": false}]", "fieldFormatMap": "{\"@timestamp\": {\"id\": \"date\"}}", "timeFieldName": "@timestamp", "title": "execbeat-*" diff --git a/beater/execevent.go b/beater/execevent.go index 701efcc6..2a3ccd35 100644 --- a/beater/execevent.go +++ b/beater/execevent.go @@ -9,7 +9,8 @@ type ExecEvent struct { ReadTime time.Time DocumentType string Fields map[string]string - Exec Exec + Exec *Exec + Line *Line } type Exec struct { @@ -19,11 +20,24 @@ type Exec struct { ExitCode int `json:"exitCode"` } +type Line struct { + Command string `json:"command,omitempty"` + Source string `json:"source"` + LineNumber int `json:"line_number"` + Line string `json:"line"` + ExitCode int `json:"exitCode"` +} + func (h *ExecEvent) ToMapStr() common.MapStr { event := common.MapStr{ "@timestamp": common.Time(h.ReadTime), "type": h.DocumentType, - "exec": h.Exec, + } + if h.Exec != nil { + event["exec"] = h.Exec + } + if h.Line != nil { + event["line"] = h.Line } if h.Fields != nil { diff --git a/beater/executor.go b/beater/executor.go index 8f201b2f..a6f17bd1 100644 --- a/beater/executor.go +++ b/beater/executor.go @@ -45,10 +45,38 @@ func (e *Executor) Run() { } cron := cron.New() - cron.AddFunc(e.schedule, func() { e.runOneTime() }) + cron.AddFunc(e.schedule, func() { + e.runOneTime() + }) cron.Start() } +func (e *Executor) sendLines(buf bytes.Buffer, source string, cmdName string, exitCode int, now time.Time) { + n := 0 + for _, s := range strings.Split(buf.String(), "\n") { + if len(s) > 0 { + lineEvent := Line{ + Command: cmdName, + Source: source, + LineNumber: n, + Line: s, + ExitCode: exitCode, + } + + event := ExecEvent{ + ReadTime: now, + DocumentType: e.documentType, + Fields: e.config.Fields, + Line: &lineEvent, + } + + e.execbeat.client.PublishEvent(event.ToMapStr()) + + n += 1 + } + } +} + func (e *Executor) runOneTime() error { var cmd *exec.Cmd var cmdArgs []string @@ -92,21 +120,26 @@ func (e *Executor) runOneTime() error { } } - commandEvent := Exec{ - Command: cmdName, - StdOut: stdout.String(), - StdErr: stderr.String(), - ExitCode: exitCode, - } + if e.config.LineMode { + e.sendLines(stdout, "stdout", cmdName, exitCode, now) + e.sendLines(stderr, "stderr", cmdName, exitCode, now) + } else { + commandEvent := Exec{ + Command: cmdName, + StdOut: stdout.String(), + StdErr: stderr.String(), + ExitCode: exitCode, + } - event := ExecEvent{ - ReadTime: now, - DocumentType: e.documentType, - Fields: e.config.Fields, - Exec: commandEvent, - } + event := ExecEvent{ + ReadTime: now, + DocumentType: e.documentType, + Fields: e.config.Fields, + Exec: &commandEvent, + } - e.execbeat.client.PublishEvent(event.ToMapStr()) + e.execbeat.client.PublishEvent(event.ToMapStr()) + } return nil } diff --git a/config/config.go b/config/config.go index 651b39e4..860d66a5 100644 --- a/config/config.go +++ b/config/config.go @@ -14,6 +14,7 @@ type ExecConfig struct { Schedule string Command string Args string + LineMode bool `config:"line_mode"` DocumentType string `config:"document_type"` Fields map[string]string `config:"fields"` } diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc index 5a7513ab..f9d47fb3 100644 --- a/docs/fields.asciidoc +++ b/docs/fields.asciidoc @@ -160,3 +160,44 @@ type: keyword Exit code of the command executed by Execbeat. + +[float] +=== line.command + +type: keyword + +The command executed by Execbeat. + + +[float] +=== line.line + +type: keyword + +Single Line from output produced by the command executed by Execbeat. + + +[float] +=== line.line_number + +type: long + +Line number of the line from output produced by the command executed by Execbeat. + + +[float] +=== line.source + +type: keyword + +Source of the line. Either stdout or stderr. + + +[float] +=== line.exitCode + +type: keyword + +Exit code of the command executed by Execbeat. + + diff --git a/execbeat.full.yml b/execbeat.full.yml index 047665b1..a847ef4d 100644 --- a/execbeat.full.yml +++ b/execbeat.full.yml @@ -21,6 +21,9 @@ execbeat: # in. Default: execbeat document_type: execbeat + # Send message for each line of output + line_mode: false + #================================ General ====================================== # The name of the shipper that publishes the network data. It can be used to group diff --git a/execbeat.template-es2x.json b/execbeat.template-es2x.json index 1bbaa2c9..36bd8f04 100644 --- a/execbeat.template-es2x.json +++ b/execbeat.template-es2x.json @@ -68,6 +68,33 @@ } } }, + "line": { + "properties": { + "command": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "exitCode": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "line": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "line_number": { + "type": "long" + }, + "source": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, "meta": { "properties": { "cloud": { diff --git a/execbeat.template.json b/execbeat.template.json index c33f8363..2ff434c8 100644 --- a/execbeat.template.json +++ b/execbeat.template.json @@ -58,6 +58,29 @@ } } }, + "line": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "exitCode": { + "ignore_above": 1024, + "type": "keyword" + }, + "line": { + "ignore_above": 1024, + "type": "keyword" + }, + "line_number": { + "type": "long" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "meta": { "properties": { "cloud": { diff --git a/execbeat.yml b/execbeat.yml index 1fb1151c..a28d9006 100644 --- a/execbeat.yml +++ b/execbeat.yml @@ -21,6 +21,9 @@ execbeat: # in. Default: execbeat #document_type: + # Send message for each line of output + #line_mode: false + #================================ General ===================================== # The name of the shipper that publishes the network data. It can be used to group From c6e761ce4d84b4973751a33c4fcacd7942710c28 Mon Sep 17 00:00:00 2001 From: Egor Leonenko Date: Sun, 5 Mar 2017 18:11:37 +1300 Subject: [PATCH 2/2] Update tests --- beater/execevent_test.go | 41 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/beater/execevent_test.go b/beater/execevent_test.go index 45adb532..9376d029 100644 --- a/beater/execevent_test.go +++ b/beater/execevent_test.go @@ -7,7 +7,7 @@ import ( "time" ) -func TestHttpEventToMapStr(t *testing.T) { +func TestExecEventToMapStr(t *testing.T) { now := time.Now() fields := make(map[string]string) fields["field1"] = "value1" @@ -20,7 +20,7 @@ func TestHttpEventToMapStr(t *testing.T) { event.Fields = fields event.DocumentType = "test" event.ReadTime = now - event.Exec = command + event.Exec = &command mapStr := event.ToMapStr() _, fieldsExist := mapStr["fields"] assert.True(t, fieldsExist) @@ -28,9 +28,44 @@ func TestHttpEventToMapStr(t *testing.T) { assert.True(t, execExist) assert.Equal(t, "test", mapStr["type"]) assert.Equal(t, common.Time(now), mapStr["@timestamp"]) + + _, lineExist := mapStr["line"] + assert.False(t, lineExist) +} + +func TestExecEventWithLineToMapStr(t *testing.T) { + now := time.Now() + fields := make(map[string]string) + fields["field1"] = "value1" + fields["field2"] = "value2" + command := Line{} + command.Command = "foo" + command.Line = "test" + command.Source = "test" + command.LineNumber = 1 + + event := ExecEvent{} + event.Fields = fields + event.DocumentType = "test" + event.ReadTime = now + event.Line = &command + + mapStr := event.ToMapStr() + + _, fieldsExist := mapStr["fields"] + assert.True(t, fieldsExist) + + _, execExist := mapStr["exec"] + assert.False(t, execExist) + + _, lineExist := mapStr["line"] + assert.True(t, lineExist) + + assert.Equal(t, "test", mapStr["type"]) + assert.Equal(t, common.Time(now), mapStr["@timestamp"]) } -func TestHttpEventToMapStrWIthEmptyFields(t *testing.T) { +func TestExecEventToMapStrWIthEmptyFields(t *testing.T) { event := ExecEvent{} mapStr := event.ToMapStr() _, fieldsExist := mapStr["fields"]