Spec Generation WIP

This is a WIP to generate table spec files.

It pulls out some stuff from `table.go` to `column.go`

To generate specs, we need to expand the options bitmask into the set of booleans. Instead of doing that via the bitmask (as in osquery#77), this explores storing these as booleans and generating the bitmask as needed.

plugin/table/column.go

@@ -0,0 +1,147 @@
+package table
+
+// ColumnDefinition defines the relevant information for a column in a table
+// plugin. Both values are mandatory. Prefer using the *Column helpers to
+// create ColumnDefinition structs.
+type ColumnDefinition struct {
+	Name        string     `json:"name,omitempty"`
+	Type        ColumnType `json:"type,omitempty"`
+	Description string     `json:"description,omitempty"`
+
+	// Options from https://github.com/osquery/osquery/blob/master/osquery/core/sql/column.h#L37
+	Index      bool `json:"index"`
+	Required   bool `json:"required"`
+	Additional bool `json:"additional"`
+	Optimized  bool `json:"optimized"`
+	Hidden     bool `json:"hidden"`
+}
+
+// ColumnType is a strongly typed representation of the data type string for a
+// column definition. The named constants should be used.
+type ColumnType string
+
+// The following column types are defined in osquery tables.h.
+const (
+	ColumnTypeUnknown        ColumnType = "UNKNOWN"
+	ColumnTypeText                      = "TEXT"
+	ColumnTypeInteger                   = "INTEGER"
+	ColumnTypeBigInt                    = "BIGINT"
+	ColumnTypeUnsignedBigInt            = "UNSIGNED BIGINT"
+	ColumnTypeDouble                    = "DOUBLE"
+	ColumnTypeBlob                      = "BLOB"
+)
+
+type ColumnOpt func(*ColumnDefinition)
+
+// TextColumn is a helper for defining columns containing strings.
+func TextColumn(name string, opts ...ColumnOpt) ColumnDefinition {
+	return NewColumn(name, ColumnTypeText, opts...)
+}
+
+// IntegerColumn is a helper for defining columns containing integers.
+func IntegerColumn(name string, opts ...ColumnOpt) ColumnDefinition {
+	return NewColumn(name, ColumnTypeInteger, opts...)
+}
+
+// BigIntColumn is a helper for defining columns containing big integers.
+func BigIntColumn(name string, opts ...ColumnOpt) ColumnDefinition {
+	return NewColumn(name, ColumnTypeBigInt, opts...)
+}
+
+// DoubleColumn is a helper for defining columns containing floating point
+// values.
+func DoubleColumn(name string, opts ...ColumnOpt) ColumnDefinition {
+	return NewColumn(name, ColumnTypeDouble, opts...)
+}
+
+// NewColumn returns a ColumnDefinition for the specified column.
+func NewColumn(name string, ctype ColumnType, opts ...ColumnOpt) ColumnDefinition {
+	cd := ColumnDefinition{
+		Name: name,
+		Type: ctype,
+	}
+
+	for _, opt := range opts {
+		opt(&cd)
+	}
+
+	return cd
+
+}
+
+// IndexColumn is a functional argument to declare this as an indexed
+// column. Depending on impmelentation, this can significantly change
+// performance.  See osquery source code for more information.
+func IndexColumn() ColumnOpt {
+	return func(cd *ColumnDefinition) {
+		cd.Index = true
+	}
+}
+
+// RequiredColumn is a functional argument that sets this as a
+// required column. sqlite will not process queries, if a required
+// column is missing. See osquery source code for more information.
+func RequiredColumn() ColumnOpt {
+	return func(cd *ColumnDefinition) {
+		cd.Required = true
+	}
+
+}
+
+// AdditionalColumn is a functional argument that sets this as an
+// additional column. See osquery source code for more information.
+func AdditionalColumn() ColumnOpt {
+	return func(cd *ColumnDefinition) {
+		cd.Additional = true
+	}
+
+}
+
+// OptimizedColumn is a functional argument that sets this as an
+// optimized column. See osquery source code for more information.
+func OptimizedColumn() ColumnOpt {
+	return func(cd *ColumnDefinition) {
+		cd.Optimized = true
+	}
+
+}
+
+// HiddenColumn is a functional argument that sets this as an
+// hidden column. This omits it from `select *` queries. See osquery source code for more information.
+func HiddenColumn() ColumnOpt {
+	return func(cd *ColumnDefinition) {
+		cd.Hidden = true
+	}
+
+}
+
+// ColumnDescription sets the column description. This is not
+// currently part of the underlying osquery api, it is here for human
+// consumption. It may become part of osquery spec generation.
+func ColumnDescription(d string) ColumnOpt {
+	return func(cd *ColumnDefinition) {
+		cd.Description = d
+	}
+}
+
+// Options returns the bitmask representation of the boolean column
+// options. This uses the values as encoded in
+// https://github.com/osquery/osquery/blob/master/osquery/core/sql/column.h#L37
+func (c *ColumnDefinition) Options() uint8 {
+	optionsBitmask := uint8(0)
+
+	optionValues := map[uint8]bool{
+		1:  c.Index,
+		2:  c.Required,
+		4:  c.Additional,
+		8:  c.Optimized,
+		16: c.Hidden,
+	}
+
+	for v, b := range optionValues {
+		if b {
+			optionsBitmask = optionsBitmask | v
+		}
+	}
+	return optionsBitmask
+}

plugin/table/column_test.go

@@ -0,0 +1,30 @@
+package table
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestColumnDefinition_Options(t *testing.T) {
+	t.Parallel()
+
+	var tests = []struct {
+		in       []ColumnOpt
+		expected uint8
+	}{
+		{
+			in:       []ColumnOpt{},
+			expected: 0,
+		},
+		{
+			in:       []ColumnOpt{IndexColumn(), HiddenColumn()},
+			expected: 17,
+		},
+	}
+
+	for _, tt := range tests {
+		cd := TextColumn("foo", tt.in...)
+		require.Equal(t, tt.expected, cd.Options())
+	}
+}

plugin/table/spec.go

@@ -0,0 +1,30 @@
+package table
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+)
+
+type osqueryTableSpec struct {
+	Cacheable bool               `json:"cacheable"`
+	Evented   bool               `json:"evented"`
+	Name      string             `json:"name,omitempty"`
+	Url       string             `json:"url,omitempty"`
+	Platforms []string           `json:"platforms,omitempty"`
+	Columns   []ColumnDefinition `json:"columns,omitempty"`
+}
+
+func (t *Plugin) Spec() (string, error) {
+	// FIXME: the columndefinition type is upcased, is that an issue?
+	tableSpec := osqueryTableSpec{
+		Name:    t.name,
+		Columns: t.columns,
+		//Platforms: []string{"FIXME"},
+	}
+	specBytes, err := json.MarshalIndent(tableSpec, "", "  ")
+	if err != nil {
+		return "", errors.Wrap(err, "marshalling")
+	}
+	return string(specBytes), nil
+}

plugin/table/spec_test.go

@@ -0,0 +1,60 @@
+package table
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestTable_Spec(t *testing.T) {
+	t.Parallel()
+
+	var tests = []struct {
+		name     string
+		columns  []ColumnDefinition
+		expected string
+	}{
+		{
+			name:    "simple",
+			columns: []ColumnDefinition{TextColumn("simple_text")},
+			expected: `
+{
+  "name": "simple",
+  "cacheable": false,
+  "evented": false,
+  "columns":[
+    { "name": "simple_text", "type": "TEXT", "index": false, "required": false, "additional": false, "optimized": false, "hidden": false }
+  ]
+}`,
+		},
+	}
+
+	mockGenerate := func(_ context.Context, _ QueryContext) ([]map[string]string, error) { return nil, nil }
+
+	for _, tt := range tests {
+		testTable := NewPlugin(tt.name, tt.columns, mockGenerate)
+		generatedSpec, err := testTable.Spec()
+		require.NoError(t, err, "generating spec for %s", tt.name)
+		helperJSONEqVal(t, tt.expected, generatedSpec, "spec for %s", tt.name)
+	}
+}
+
+func helperJSONEqVal(t *testing.T, expected string, actual string, msgAndArgs ...interface{}) {
+	var expectedJSONAsInterface, actualJSONAsInterface interface{}
+
+	if err := json.Unmarshal([]byte(expected), &expectedJSONAsInterface); err != nil {
+		require.Fail(t, fmt.Sprintf("Expected value ('%s') is not valid json.\nJSON parsing error: '%s'", expected, err.Error()), msgAndArgs...)
+		return
+	}
+
+	if err := json.Unmarshal([]byte(actual), &actualJSONAsInterface); err != nil {
+		require.Fail(t, fmt.Sprintf("Input ('%s') needs to be valid json.\nJSON parsing error: '%s'", actual, err.Error()), msgAndArgs...)
+		return
+	}
+
+	require.EqualValues(t, expectedJSONAsInterface, actualJSONAsInterface, msgAndArgs...)
+	return
+}

plugin/table/table.go

@@ -103,59 +103,6 @@ func (t *Plugin) Ping() osquery.ExtensionStatus {
 
 func (t *Plugin) Shutdown() {}
 
-// ColumnDefinition defines the relevant information for a column in a table
-// plugin. Both values are mandatory. Prefer using the *Column helpers to
-// create ColumnDefinition structs.
-type ColumnDefinition struct {
-	Name string
-	Type ColumnType
-}
-
-// TextColumn is a helper for defining columns containing strings.
-func TextColumn(name string) ColumnDefinition {
-	return ColumnDefinition{
-		Name: name,
-		Type: ColumnTypeText,
-	}
-}
-
-// IntegerColumn is a helper for defining columns containing integers.
-func IntegerColumn(name string) ColumnDefinition {
-	return ColumnDefinition{
-		Name: name,
-		Type: ColumnTypeInteger,
-	}
-}
-
-// BigIntColumn is a helper for defining columns containing big integers.
-func BigIntColumn(name string) ColumnDefinition {
-	return ColumnDefinition{
-		Name: name,
-		Type: ColumnTypeBigInt,
-	}
-}
-
-// DoubleColumn is a helper for defining columns containing floating point
-// values.
-func DoubleColumn(name string) ColumnDefinition {
-	return ColumnDefinition{
-		Name: name,
-		Type: ColumnTypeDouble,
-	}
-}
-
-// ColumnType is a strongly typed representation of the data type string for a
-// column definition. The named constants should be used.
-type ColumnType string
-
-// The following column types are defined in osquery tables.h.
-const (
-	ColumnTypeText    ColumnType = "TEXT"
-	ColumnTypeInteger            = "INTEGER"
-	ColumnTypeBigInt             = "BIGINT"
-	ColumnTypeDouble             = "DOUBLE"
-)
-
 // QueryContext contains the constraints from the WHERE clause of the query,
 // that can optionally be used to optimize the table generation. Note that the
 // osquery SQLite engine will perform the filtering with these constraints, so