From 5e5bde74a3912d610194798994441f6c6d580c61 Mon Sep 17 00:00:00 2001 From: Geofrey Ernest Date: Sun, 7 Jan 2024 21:41:31 +0300 Subject: [PATCH] buf format -w (#673) * buf format -w I was working on protobuf files and notices my vscode was formatting the files differently. We already use buf for generating, it is reasonable to use buf for formatting as well. Strictly for consistency. * add workflow for buf format --- .github/workflows/buf.yml | 19 ++ proto/frostdb/schema/v1alpha1/schema.proto | 172 +++++----- proto/frostdb/schema/v1alpha2/schema.proto | 304 +++++++++--------- .../frostdb/snapshot/v1alpha1/snapshot.proto | 2 +- proto/frostdb/table/v1alpha1/config.proto | 26 +- 5 files changed, 271 insertions(+), 252 deletions(-) create mode 100644 .github/workflows/buf.yml diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml new file mode 100644 index 000000000..68bb3e95c --- /dev/null +++ b/.github/workflows/buf.yml @@ -0,0 +1,19 @@ +name: Buf + +on: + push: + branches: [main] + pull_request: + branches: [main] + paths: + - "proto/**/*.proto" + +jobs: + format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: bufbuild/buf-setup-action@v1 + + - name: Format + run: buf format --exit-code diff --git a/proto/frostdb/schema/v1alpha1/schema.proto b/proto/frostdb/schema/v1alpha1/schema.proto index 8f8849698..423b05d0a 100644 --- a/proto/frostdb/schema/v1alpha1/schema.proto +++ b/proto/frostdb/schema/v1alpha1/schema.proto @@ -4,112 +4,112 @@ package frostdb.schema.v1alpha1; // Schema definition for a table. message Schema { - // Name of the schema. - string name = 1; - // Columns in the schema. - repeated Column columns = 2; - // Columns to sort by in the schema. - repeated SortingColumn sorting_columns = 3; - // UniquePrimaryIndex defines whether the primary index is unique. Duplicate - // (according to the sorting column) rows will be dropped on compaction. - bool unique_primary_index = 4; + // Name of the schema. + string name = 1; + // Columns in the schema. + repeated Column columns = 2; + // Columns to sort by in the schema. + repeated SortingColumn sorting_columns = 3; + // UniquePrimaryIndex defines whether the primary index is unique. Duplicate + // (according to the sorting column) rows will be dropped on compaction. + bool unique_primary_index = 4; } // Column definition. message Column { - // Name of the column. - string name = 1; - // Storage layout of the column. - StorageLayout storage_layout = 2; - // Whether the column can dynamically expand. - bool dynamic = 3; - // Prehash the column before storing it. This is an optimization to speed up aggregation queries when this column is often aggregated. - // This will create a separate non-dynamic column with the same name and the prefix "hashed." that contains the prehashed values. - bool prehash = 4; + // Name of the column. + string name = 1; + // Storage layout of the column. + StorageLayout storage_layout = 2; + // Whether the column can dynamically expand. + bool dynamic = 3; + // Prehash the column before storing it. This is an optimization to speed up aggregation queries when this column is often aggregated. + // This will create a separate non-dynamic column with the same name and the prefix "hashed." that contains the prehashed values. + bool prehash = 4; } // Storage layout describes the physical storage properties of a column. message StorageLayout { - // Type enum of a column. - enum Type { - // Unknown type. - TYPE_UNKNOWN_UNSPECIFIED = 0; - // Represents a string type. - TYPE_STRING = 1; - // Represents an int64 type. - TYPE_INT64 = 2; - // Represents a double type. - TYPE_DOUBLE = 3; - // Represents a boolean type. - TYPE_BOOL = 4; - // Represents a int32 type. - TYPE_INT32 = 5; - } + // Type enum of a column. + enum Type { + // Unknown type. + TYPE_UNKNOWN_UNSPECIFIED = 0; + // Represents a string type. + TYPE_STRING = 1; + // Represents an int64 type. + TYPE_INT64 = 2; + // Represents a double type. + TYPE_DOUBLE = 3; + // Represents a boolean type. + TYPE_BOOL = 4; + // Represents a int32 type. + TYPE_INT32 = 5; + } - // Type of the column. - Type type = 1; + // Type of the column. + Type type = 1; - // Encoding enum of a column. - enum Encoding { - // Plain encoding. - ENCODING_PLAIN_UNSPECIFIED = 0; - // Dictionary run-length encoding. - ENCODING_RLE_DICTIONARY = 1; - // Delta binary packed encoding. - ENCODING_DELTA_BINARY_PACKED = 2; - // Delta Byte Array encoding. - ENCODING_DELTA_BYTE_ARRAY = 3; - // Delta Length Byte Array encoding. - ENCODING_DELTA_LENGTH_BYTE_ARRAY = 4; - } + // Encoding enum of a column. + enum Encoding { + // Plain encoding. + ENCODING_PLAIN_UNSPECIFIED = 0; + // Dictionary run-length encoding. + ENCODING_RLE_DICTIONARY = 1; + // Delta binary packed encoding. + ENCODING_DELTA_BINARY_PACKED = 2; + // Delta Byte Array encoding. + ENCODING_DELTA_BYTE_ARRAY = 3; + // Delta Length Byte Array encoding. + ENCODING_DELTA_LENGTH_BYTE_ARRAY = 4; + } - // Encoding of the column. - Encoding encoding = 2; + // Encoding of the column. + Encoding encoding = 2; - // Compression enum of a column. - enum Compression { - // No compression. - COMPRESSION_NONE_UNSPECIFIED = 0; - // Snappy compression. - COMPRESSION_SNAPPY = 1; - // GZIP compression. - COMPRESSION_GZIP = 2; - // Brotli compression. - COMPRESSION_BROTLI = 3; - // LZ4_RAW compression. - COMPRESSION_LZ4_RAW = 4; - // ZSTD compression. - COMPRESSION_ZSTD = 5; - } + // Compression enum of a column. + enum Compression { + // No compression. + COMPRESSION_NONE_UNSPECIFIED = 0; + // Snappy compression. + COMPRESSION_SNAPPY = 1; + // GZIP compression. + COMPRESSION_GZIP = 2; + // Brotli compression. + COMPRESSION_BROTLI = 3; + // LZ4_RAW compression. + COMPRESSION_LZ4_RAW = 4; + // ZSTD compression. + COMPRESSION_ZSTD = 5; + } - // Compression of the column. - Compression compression = 3; + // Compression of the column. + Compression compression = 3; - // Wether values in the column are allowed to be null. - bool nullable = 4; + // Wether values in the column are allowed to be null. + bool nullable = 4; - // Whether the column is repeated. - bool repeated = 5; + // Whether the column is repeated. + bool repeated = 5; } // SortingColumn definition. message SortingColumn { - // Name of the column to sort by. - string name = 1; + // Name of the column to sort by. + string name = 1; - // Enum of possible sorting directions. - enum Direction { - // Unknown direction. - DIRECTION_UNKNOWN_UNSPECIFIED = 0; - // Sort in ascending order. - DIRECTION_ASCENDING = 1; - // Sort in descending order. - DIRECTION_DESCENDING = 2; - } + // Enum of possible sorting directions. + enum Direction { + // Unknown direction. + DIRECTION_UNKNOWN_UNSPECIFIED = 0; + // Sort in ascending order. + DIRECTION_ASCENDING = 1; + // Sort in descending order. + DIRECTION_DESCENDING = 2; + } - // Direction of the sorting. - Direction direction = 2; + // Direction of the sorting. + Direction direction = 2; - // Whether nulls are the smallest or largest values. - bool nulls_first = 3; + // Whether nulls are the smallest or largest values. + bool nulls_first = 3; } diff --git a/proto/frostdb/schema/v1alpha2/schema.proto b/proto/frostdb/schema/v1alpha2/schema.proto index be21e0c7d..854b49172 100644 --- a/proto/frostdb/schema/v1alpha2/schema.proto +++ b/proto/frostdb/schema/v1alpha2/schema.proto @@ -3,180 +3,180 @@ syntax = "proto3"; package frostdb.schema.v1alpha2; /* This schema version differs from the previous one in that it supports nested schema definitions. - For example the following struct - labels: { - label1: value1 - label2: value2 - } - timestamps: [1,2,3] - values: [2,2,2] - - Could be represented by this schema as: - - { - "root": { - "name": "example_schema", - "nodes": [ - { - "group": { - "name": "labels", - "nodes": [ - { - "leaf": { - "name": "label1", - "storage_layout": { ... } - } - }, - { - "leaf": { - "name": "label2", - "storage_layout": { ... } - } - } - ] - } - }, - { - "leaf": { - "name": "timestamps", - "storage_layout": { ... } - } - }, - { - "leaf": { - "name": "values", - "storage_layout": { ... } - } - } - ] - } - } - + For example the following struct + labels: { + label1: value1 + label2: value2 + } + timestamps: [1,2,3] + values: [2,2,2] + + Could be represented by this schema as: + + { + "root": { + "name": "example_schema", + "nodes": [ + { + "group": { + "name": "labels", + "nodes": [ + { + "leaf": { + "name": "label1", + "storage_layout": { ... } + } + }, + { + "leaf": { + "name": "label2", + "storage_layout": { ... } + } + } + ] + } + }, + { + "leaf": { + "name": "timestamps", + "storage_layout": { ... } + } + }, + { + "leaf": { + "name": "values", + "storage_layout": { ... } + } + } + ] + } + } + */ // Schema definition for a table. message Schema { - // Root node of the schema. - Group root = 2; - // Columns to sort by in the schema. - repeated SortingColumn sorting_columns = 3; - // UniquePrimaryIndex defines whether the primary index is unique. Duplicate - // (according to the sorting column) rows will be dropped on compaction. - bool unique_primary_index = 4; + // Root node of the schema. + Group root = 2; + // Columns to sort by in the schema. + repeated SortingColumn sorting_columns = 3; + // UniquePrimaryIndex defines whether the primary index is unique. Duplicate + // (according to the sorting column) rows will be dropped on compaction. + bool unique_primary_index = 4; } // Node is a node in a schema tree. message Node { - oneof type { - // Leaf node of the schema. - Leaf leaf = 1; - // Group of other nodes for nested schemas. - Group group = 2; - } + oneof type { + // Leaf node of the schema. + Leaf leaf = 1; + // Group of other nodes for nested schemas. + Group group = 2; + } } // Leaf definition. message Leaf { - // Name of the column. - string name = 1; - // Storage layout of the column. - StorageLayout storage_layout = 2; + // Name of the column. + string name = 1; + // Storage layout of the column. + StorageLayout storage_layout = 2; } // Group is a grouping of nodes. message Group { - // Name of the group. - string name = 1; - // Wether the group is allowed to be null. - bool nullable = 2; - // Indicates whether the group is repeated. - bool repeated = 3; - // Nodes that this group is composed of. - repeated Node nodes = 4; + // Name of the group. + string name = 1; + // Wether the group is allowed to be null. + bool nullable = 2; + // Indicates whether the group is repeated. + bool repeated = 3; + // Nodes that this group is composed of. + repeated Node nodes = 4; } // Storage layout describes the physical storage properties of a column. message StorageLayout { - // Type enum of a column. - enum Type { - // Unknown type. - TYPE_UNKNOWN_UNSPECIFIED = 0; - // Represents a string type. - TYPE_STRING = 1; - // Represents an int64 type. - TYPE_INT64 = 2; - // Represents a double type. - TYPE_DOUBLE = 3; - // Represents a boolean type. - TYPE_BOOL = 4; - // Represents a int32 type. - TYPE_INT32 = 5; - } - - // Type of the column. - Type type = 1; - - // Encoding enum of a column. - enum Encoding { - // Plain encoding. - ENCODING_PLAIN_UNSPECIFIED = 0; - // Dictionary run-length encoding. - ENCODING_RLE_DICTIONARY = 1; - // Delta binary packed encoding. - ENCODING_DELTA_BINARY_PACKED = 2; - // Delta Byte Array encoding. - ENCODING_DELTA_BYTE_ARRAY = 3; - // Delta Length Byte Array encoding. - ENCODING_DELTA_LENGTH_BYTE_ARRAY = 4; - } - - // Encoding of the column. - Encoding encoding = 2; - - // Compression enum of a column. - enum Compression { - // No compression. - COMPRESSION_NONE_UNSPECIFIED = 0; - // Snappy compression. - COMPRESSION_SNAPPY = 1; - // GZIP compression. - COMPRESSION_GZIP = 2; - // Brotli compression. - COMPRESSION_BROTLI = 3; - // LZ4_RAW compression. - COMPRESSION_LZ4_RAW = 4; - // ZSTD compression. - COMPRESSION_ZSTD = 5; - } - - // Compression of the column. - Compression compression = 3; - - // Wether values in the column are allowed to be null. - bool nullable = 4; - - // Indicates whether the parquet column is repeated. - bool repeated = 5; + // Type enum of a column. + enum Type { + // Unknown type. + TYPE_UNKNOWN_UNSPECIFIED = 0; + // Represents a string type. + TYPE_STRING = 1; + // Represents an int64 type. + TYPE_INT64 = 2; + // Represents a double type. + TYPE_DOUBLE = 3; + // Represents a boolean type. + TYPE_BOOL = 4; + // Represents a int32 type. + TYPE_INT32 = 5; + } + + // Type of the column. + Type type = 1; + + // Encoding enum of a column. + enum Encoding { + // Plain encoding. + ENCODING_PLAIN_UNSPECIFIED = 0; + // Dictionary run-length encoding. + ENCODING_RLE_DICTIONARY = 1; + // Delta binary packed encoding. + ENCODING_DELTA_BINARY_PACKED = 2; + // Delta Byte Array encoding. + ENCODING_DELTA_BYTE_ARRAY = 3; + // Delta Length Byte Array encoding. + ENCODING_DELTA_LENGTH_BYTE_ARRAY = 4; + } + + // Encoding of the column. + Encoding encoding = 2; + + // Compression enum of a column. + enum Compression { + // No compression. + COMPRESSION_NONE_UNSPECIFIED = 0; + // Snappy compression. + COMPRESSION_SNAPPY = 1; + // GZIP compression. + COMPRESSION_GZIP = 2; + // Brotli compression. + COMPRESSION_BROTLI = 3; + // LZ4_RAW compression. + COMPRESSION_LZ4_RAW = 4; + // ZSTD compression. + COMPRESSION_ZSTD = 5; + } + + // Compression of the column. + Compression compression = 3; + + // Wether values in the column are allowed to be null. + bool nullable = 4; + + // Indicates whether the parquet column is repeated. + bool repeated = 5; } // SortingColumn definition. message SortingColumn { - // Path to the leaf column to sort by. - string path = 1; - - // Enum of possible sorting directions. - enum Direction { - // Unknown direction. - DIRECTION_UNKNOWN_UNSPECIFIED = 0; - // Sort in ascending order. - DIRECTION_ASCENDING = 1; - // Sort in descending order. - DIRECTION_DESCENDING = 2; - } - - // Direction of the sorting. - Direction direction = 2; - - // Whether nulls are the smallest or largest values. - bool nulls_first = 3; + // Path to the leaf column to sort by. + string path = 1; + + // Enum of possible sorting directions. + enum Direction { + // Unknown direction. + DIRECTION_UNKNOWN_UNSPECIFIED = 0; + // Sort in ascending order. + DIRECTION_ASCENDING = 1; + // Sort in descending order. + DIRECTION_DESCENDING = 2; + } + + // Direction of the sorting. + Direction direction = 2; + + // Whether nulls are the smallest or largest values. + bool nulls_first = 3; } diff --git a/proto/frostdb/snapshot/v1alpha1/snapshot.proto b/proto/frostdb/snapshot/v1alpha1/snapshot.proto index 9f96dc277..6f7ec9d11 100644 --- a/proto/frostdb/snapshot/v1alpha1/snapshot.proto +++ b/proto/frostdb/snapshot/v1alpha1/snapshot.proto @@ -37,6 +37,6 @@ message Part { ENCODING_UNKNOWN = 0; ENCODING_PARQUET = 1; ENCODING_ARROW = 2; - }; + } Encoding encoding = 5; } diff --git a/proto/frostdb/table/v1alpha1/config.proto b/proto/frostdb/table/v1alpha1/config.proto index c834810a4..fefb7d47b 100644 --- a/proto/frostdb/table/v1alpha1/config.proto +++ b/proto/frostdb/table/v1alpha1/config.proto @@ -7,17 +7,17 @@ import "frostdb/schema/v1alpha2/schema.proto"; // TableConfig is the configuration information for a table. message TableConfig { - // Schema of the table. - oneof schema { - // Deprecated schema definition. Used for backwards compatibility. - frostdb.schema.v1alpha1.Schema deprecated_schema = 1; - // Schema of the table. Use this field. - frostdb.schema.v1alpha2.Schema schema_v2 = 2; - } - // RowGroupSize is the size in rows of row groups that are written to Parquet files. - uint64 row_group_size = 3; - // BlockReaderLimit is the concurrency limit of the number of Go routines spawned when reading storage blocks. - uint64 block_reader_limit = 4; - // DisableWal disables the write ahead log for this table. - bool disable_wal = 5; + // Schema of the table. + oneof schema { + // Deprecated schema definition. Used for backwards compatibility. + frostdb.schema.v1alpha1.Schema deprecated_schema = 1; + // Schema of the table. Use this field. + frostdb.schema.v1alpha2.Schema schema_v2 = 2; + } + // RowGroupSize is the size in rows of row groups that are written to Parquet files. + uint64 row_group_size = 3; + // BlockReaderLimit is the concurrency limit of the number of Go routines spawned when reading storage blocks. + uint64 block_reader_limit = 4; + // DisableWal disables the write ahead log for this table. + bool disable_wal = 5; }