From fcd703dab67ff07e7f09070804dfcee2d69c38b0 Mon Sep 17 00:00:00 2001 From: Matthias Loibl Date: Thu, 4 Apr 2024 14:44:43 +0200 Subject: [PATCH] Add proto CI runs (#807) * Add proto CI runs Mostly copied from Parca. * Add comments to proto messages where buf lint complains * buf generate --- .github/workflows/proto-gen.yaml | 52 +++++++++ .github/workflows/proto-pr.yaml | 64 ++++++++++ .github/workflows/proto-push.yaml | 72 ++++++++++++ .../go/frostdb/schema/v1alpha2/schema.pb.go | 2 + .../frostdb/snapshot/v1alpha1/snapshot.pb.go | 109 +++++++++++------- gen/proto/go/frostdb/wal/v1alpha1/wal.pb.go | 1 + proto/frostdb/schema/v1alpha2/schema.proto | 1 + .../frostdb/snapshot/v1alpha1/snapshot.proto | 25 +++- proto/frostdb/wal/v1alpha1/wal.proto | 1 + 9 files changed, 283 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/proto-gen.yaml create mode 100644 .github/workflows/proto-pr.yaml create mode 100644 .github/workflows/proto-push.yaml diff --git a/.github/workflows/proto-gen.yaml b/.github/workflows/proto-gen.yaml new file mode 100644 index 000000000..6574218f7 --- /dev/null +++ b/.github/workflows/proto-gen.yaml @@ -0,0 +1,52 @@ +name: proto-gen + +on: + pull_request: + merge_group: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +jobs: + skip-check: + name: Skip check + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + should_skip: ${{ steps.skip-check.outputs.should_skip }} + permissions: + actions: write + contents: read + steps: + - id: skip-check + uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 + with: + do_not_skip: '["schedule", "workflow_dispatch"]' + paths: |- + [ + ".github/workflows/proto-gen.yaml", + ".go-version", + "buf.gen.yaml", + "buf.work.yaml", + "proto/**" + ] + skip_after_successful_duplicate: false + + build: + name: Proto Generate + needs: skip-check + if: ${{ needs.skip-check.outputs.should_skip != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa # v1.28.1 + + - name: Generate + run: + buf generate && git diff --exit-code diff --git a/.github/workflows/proto-pr.yaml b/.github/workflows/proto-pr.yaml new file mode 100644 index 000000000..c6bb05602 --- /dev/null +++ b/.github/workflows/proto-pr.yaml @@ -0,0 +1,64 @@ +name: proto-pr + +on: + pull_request: + merge_group: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +jobs: + skip-check: + name: Skip check + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + should_skip: ${{ steps.skip-check.outputs.should_skip }} + permissions: + actions: write + contents: read + steps: + - id: skip-check + uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 + with: + do_not_skip: '["schedule", "workflow_dispatch"]' + paths: |- + [ + ".github/workflows/proto-pr.yaml", + ".go-version", + "buf.gen.yaml", + "buf.work.yaml", + "proto/**" + ] + skip_after_successful_duplicate: false + + build: + name: Proto PR Checks + needs: skip-check + if: ${{ needs.skip-check.outputs.should_skip != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa # v1.28.1 + + - name: version + run: buf --version + + - name: Format + run: buf format --diff --exit-code + + - uses: bufbuild/buf-lint-action@044d13acb1f155179c606aaa2e53aea304d22058 # v1.1.0 + with: + input: 'proto' + + - uses: bufbuild/buf-breaking-action@a074e988ee34efcd4927079e79c611f428354c01 # v1.1.3 + with: + input: 'proto' + # The 'main' branch of the GitHub repository that defines the module. + against: 'https://github.com/${GITHUB_REPOSITORY}.git#branch=main,subdir=proto' diff --git a/.github/workflows/proto-push.yaml b/.github/workflows/proto-push.yaml new file mode 100644 index 000000000..cfe5f78bc --- /dev/null +++ b/.github/workflows/proto-push.yaml @@ -0,0 +1,72 @@ +name: proto-push + +on: + push: + branches: + - main + - release-* + merge_group: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +jobs: + skip-check: + name: Skip check + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + should_skip: ${{ steps.skip-check.outputs.should_skip }} + permissions: + actions: write + contents: read + steps: + - id: skip-check + uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 + with: + do_not_skip: '["schedule", "workflow_dispatch"]' + paths: |- + [ + ".github/workflows/proto-push.yaml", + ".go-version", + "buf.gen.yaml", + "buf.work.yaml", + "proto/**" + ] + skip_after_successful_duplicate: false + + build: + name: Proto Push + needs: skip-check + if: ${{ needs.skip-check.outputs.should_skip != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - uses: bufbuild/buf-setup-action@382440cdb8ec7bc25a68d7b4711163d95f7cc3aa # v1.28.1 + + - name: version + run: buf --version + + - name: Format + run: buf format --diff --exit-code + + - uses: bufbuild/buf-lint-action@044d13acb1f155179c606aaa2e53aea304d22058 # v1.1.0 + with: + input: 'proto' + + - uses: bufbuild/buf-breaking-action@a074e988ee34efcd4927079e79c611f428354c01 # v1.1.3 + with: + input: 'proto' + # The 'main' branch of the GitHub repository that defines the module. + against: 'https://github.com/${GITHUB_REPOSITORY}.git#branch=main,ref=HEAD~1,subdir=proto' + + - uses: bufbuild/buf-push-action@a654ff18effe4641ebea4a4ce242c49800728459 # v1.2.0 + with: + input: 'proto' + buf_token: ${{ secrets.BUF_TOKEN }} diff --git a/gen/proto/go/frostdb/schema/v1alpha2/schema.pb.go b/gen/proto/go/frostdb/schema/v1alpha2/schema.pb.go index 6d7a25ee4..e667c5964 100644 --- a/gen/proto/go/frostdb/schema/v1alpha2/schema.pb.go +++ b/gen/proto/go/frostdb/schema/v1alpha2/schema.pb.go @@ -342,6 +342,8 @@ type Node struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // type can either be a leaf or a group. + // // Types that are assignable to Type: // // *Node_Leaf diff --git a/gen/proto/go/frostdb/snapshot/v1alpha1/snapshot.pb.go b/gen/proto/go/frostdb/snapshot/v1alpha1/snapshot.pb.go index 71e751d63..4cf9aedf4 100644 --- a/gen/proto/go/frostdb/snapshot/v1alpha1/snapshot.pb.go +++ b/gen/proto/go/frostdb/snapshot/v1alpha1/snapshot.pb.go @@ -21,25 +21,29 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Encoding is a enum of possible encodings of the part. type Part_Encoding int32 const ( - Part_ENCODING_UNKNOWN Part_Encoding = 0 + // ENCODING_UNSPECIFIED is an unknown encoding. + Part_ENCODING_UNSPECIFIED Part_Encoding = 0 + // ENCODING_PARQUET is a parquet encoding. Part_ENCODING_PARQUET Part_Encoding = 1 - Part_ENCODING_ARROW Part_Encoding = 2 + // ENCODING_ARROW is an arrow encoding. + Part_ENCODING_ARROW Part_Encoding = 2 ) // Enum value maps for Part_Encoding. var ( Part_Encoding_name = map[int32]string{ - 0: "ENCODING_UNKNOWN", + 0: "ENCODING_UNSPECIFIED", 1: "ENCODING_PARQUET", 2: "ENCODING_ARROW", } Part_Encoding_value = map[string]int32{ - "ENCODING_UNKNOWN": 0, - "ENCODING_PARQUET": 1, - "ENCODING_ARROW": 2, + "ENCODING_UNSPECIFIED": 0, + "ENCODING_PARQUET": 1, + "ENCODING_ARROW": 2, } ) @@ -77,6 +81,7 @@ type FooterData struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // table_metadata is a list of all tables in the snapshot. TableMetadata []*Table `protobuf:"bytes,1,rep,name=table_metadata,json=tableMetadata,proto3" json:"table_metadata,omitempty"` } @@ -119,15 +124,20 @@ func (x *FooterData) GetTableMetadata() []*Table { return nil } +// Table encodes metadata about a table in a snapshot. type Table struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Config *v1alpha1.TableConfig `protobuf:"bytes,2,opt,name=config,proto3" json:"config,omitempty"` - ActiveBlock *Table_TableBlock `protobuf:"bytes,3,opt,name=active_block,json=activeBlock,proto3" json:"active_block,omitempty"` - GranuleMetadata []*Granule `protobuf:"bytes,4,rep,name=granule_metadata,json=granuleMetadata,proto3" json:"granule_metadata,omitempty"` + // name of the table. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // config is the configuration of the table. + Config *v1alpha1.TableConfig `protobuf:"bytes,2,opt,name=config,proto3" json:"config,omitempty"` + // active_block is the active block of the table. + ActiveBlock *Table_TableBlock `protobuf:"bytes,3,opt,name=active_block,json=activeBlock,proto3" json:"active_block,omitempty"` + // granule_metadata is a list of all granules in the table. + GranuleMetadata []*Granule `protobuf:"bytes,4,rep,name=granule_metadata,json=granuleMetadata,proto3" json:"granule_metadata,omitempty"` } func (x *Table) Reset() { @@ -190,11 +200,13 @@ func (x *Table) GetGranuleMetadata() []*Granule { return nil } +// Granule encodes metadata about a granule in a table. type Granule struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // part_metadata is a list of all parts in the granule. PartMetadata []*Part `protobuf:"bytes,1,rep,name=part_metadata,json=partMetadata,proto3" json:"part_metadata,omitempty"` } @@ -237,16 +249,22 @@ func (x *Granule) GetPartMetadata() []*Part { return nil } +// Part encodes metadata about a part in a granule. type Part struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - StartOffset int64 `protobuf:"varint,1,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"` - EndOffset int64 `protobuf:"varint,2,opt,name=end_offset,json=endOffset,proto3" json:"end_offset,omitempty"` - Tx uint64 `protobuf:"varint,3,opt,name=tx,proto3" json:"tx,omitempty"` - CompactionLevel uint64 `protobuf:"varint,4,opt,name=compaction_level,json=compactionLevel,proto3" json:"compaction_level,omitempty"` - Encoding Part_Encoding `protobuf:"varint,5,opt,name=encoding,proto3,enum=frostdb.snapshot.v1alpha1.Part_Encoding" json:"encoding,omitempty"` + // start_offset is the start offset of the part. + StartOffset int64 `protobuf:"varint,1,opt,name=start_offset,json=startOffset,proto3" json:"start_offset,omitempty"` + // end_offset is the end offset of the part. + EndOffset int64 `protobuf:"varint,2,opt,name=end_offset,json=endOffset,proto3" json:"end_offset,omitempty"` + // tx is the transaction id of the part. + Tx uint64 `protobuf:"varint,3,opt,name=tx,proto3" json:"tx,omitempty"` + // compaction_level is the compaction level of the part. + CompactionLevel uint64 `protobuf:"varint,4,opt,name=compaction_level,json=compactionLevel,proto3" json:"compaction_level,omitempty"` + // encoding is the actual encoding of the part. + Encoding Part_Encoding `protobuf:"varint,5,opt,name=encoding,proto3,enum=frostdb.snapshot.v1alpha1.Part_Encoding" json:"encoding,omitempty"` } func (x *Part) Reset() { @@ -313,17 +331,22 @@ func (x *Part) GetEncoding() Part_Encoding { if x != nil { return x.Encoding } - return Part_ENCODING_UNKNOWN + return Part_ENCODING_UNSPECIFIED } +// TableBlock type Table_TableBlock struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Ulid []byte `protobuf:"bytes,1,opt,name=ulid,proto3" json:"ulid,omitempty"` - Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` - MinTx uint64 `protobuf:"varint,3,opt,name=min_tx,json=minTx,proto3" json:"min_tx,omitempty"` + // ulid is the unique identifier of the block. + Ulid []byte `protobuf:"bytes,1,opt,name=ulid,proto3" json:"ulid,omitempty"` + // size is the size of the block in bytes. + Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + // min_tx is the minimum transaction id in the block. + MinTx uint64 `protobuf:"varint,3,opt,name=min_tx,json=minTx,proto3" json:"min_tx,omitempty"` + // prev_tx is the transaction id of the previous block. PrevTx uint64 `protobuf:"varint,4,opt,name=prev_tx,json=prevTx,proto3" json:"prev_tx,omitempty"` } @@ -429,7 +452,7 @@ var file_frostdb_snapshot_v1alpha1_snapshot_proto_rawDesc = []byte{ 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x2e, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x52, 0x0c, 0x70, 0x61, 0x72, 0x74, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x95, 0x02, 0x0a, 0x04, 0x50, 0x61, 0x72, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x99, 0x02, 0x0a, 0x04, 0x50, 0x61, 0x72, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x6e, 0x64, 0x5f, 0x6f, 0x66, 0x66, 0x73, @@ -442,29 +465,29 @@ var file_frostdb_snapshot_v1alpha1_snapshot_proto_rawDesc = []byte{ 0x32, 0x28, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x2e, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x65, 0x6e, 0x63, 0x6f, - 0x64, 0x69, 0x6e, 0x67, 0x22, 0x4a, 0x0a, 0x08, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, - 0x12, 0x14, 0x0a, 0x10, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x4e, 0x4b, - 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, - 0x4e, 0x47, 0x5f, 0x50, 0x41, 0x52, 0x51, 0x55, 0x45, 0x54, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, - 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x41, 0x52, 0x52, 0x4f, 0x57, 0x10, 0x02, - 0x42, 0x8d, 0x02, 0x0a, 0x1d, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, - 0x2e, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, - 0x61, 0x31, 0x42, 0x0d, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x01, 0x5a, 0x57, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x73, 0x2f, 0x66, 0x72, 0x6f, - 0x73, 0x74, 0x64, 0x62, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, - 0x6f, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x2f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, - 0x6f, 0x74, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b, 0x73, 0x6e, 0x61, 0x70, - 0x73, 0x68, 0x6f, 0x74, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xa2, 0x02, 0x03, 0x46, - 0x53, 0x58, 0xaa, 0x02, 0x19, 0x46, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x2e, 0x53, 0x6e, 0x61, - 0x70, 0x73, 0x68, 0x6f, 0x74, 0x2e, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xca, 0x02, - 0x19, 0x46, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x5c, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, - 0x74, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xe2, 0x02, 0x25, 0x46, 0x72, 0x6f, - 0x73, 0x74, 0x64, 0x62, 0x5c, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5c, 0x56, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0xea, 0x02, 0x1b, 0x46, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x3a, 0x3a, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x69, 0x6e, 0x67, 0x22, 0x4e, 0x0a, 0x08, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, + 0x12, 0x18, 0x0a, 0x14, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x55, 0x4e, 0x53, + 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x45, 0x4e, + 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x41, 0x52, 0x51, 0x55, 0x45, 0x54, 0x10, 0x01, + 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x4e, 0x43, 0x4f, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x41, 0x52, 0x52, + 0x4f, 0x57, 0x10, 0x02, 0x42, 0x8d, 0x02, 0x0a, 0x1d, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x72, 0x6f, + 0x73, 0x74, 0x64, 0x62, 0x2e, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x42, 0x0d, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x57, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x73, + 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x66, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x2f, 0x73, 0x6e, + 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b, + 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0xa2, 0x02, 0x03, 0x46, 0x53, 0x58, 0xaa, 0x02, 0x19, 0x46, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, + 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x2e, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0xca, 0x02, 0x19, 0x46, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x5c, 0x53, 0x6e, 0x61, + 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0xe2, 0x02, + 0x25, 0x46, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, 0x5c, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, + 0x74, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x1b, 0x46, 0x72, 0x6f, 0x73, 0x74, 0x64, 0x62, + 0x3a, 0x3a, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/gen/proto/go/frostdb/wal/v1alpha1/wal.pb.go b/gen/proto/go/frostdb/wal/v1alpha1/wal.pb.go index 658bae9e0..2472b2c31 100644 --- a/gen/proto/go/frostdb/wal/v1alpha1/wal.pb.go +++ b/gen/proto/go/frostdb/wal/v1alpha1/wal.pb.go @@ -380,6 +380,7 @@ func (x *Entry_TableBlockPersisted) GetBlockId() []byte { return nil } +// The snapshot entry. type Entry_Snapshot struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/proto/frostdb/schema/v1alpha2/schema.proto b/proto/frostdb/schema/v1alpha2/schema.proto index 1bea2e4e2..17788b04b 100644 --- a/proto/frostdb/schema/v1alpha2/schema.proto +++ b/proto/frostdb/schema/v1alpha2/schema.proto @@ -67,6 +67,7 @@ message Schema { // Node is a node in a schema tree. message Node { + // type can either be a leaf or a group. oneof type { // Leaf node of the schema. Leaf leaf = 1; diff --git a/proto/frostdb/snapshot/v1alpha1/snapshot.proto b/proto/frostdb/snapshot/v1alpha1/snapshot.proto index 6f7ec9d11..76cf90754 100644 --- a/proto/frostdb/snapshot/v1alpha1/snapshot.proto +++ b/proto/frostdb/snapshot/v1alpha1/snapshot.proto @@ -8,35 +8,58 @@ import "frostdb/table/v1alpha1/config.proto"; // data about the rest of the file. message FooterData { reserved 2; // Previously used for transaction metadata. + // table_metadata is a list of all tables in the snapshot. repeated Table table_metadata = 1; } +// Table encodes metadata about a table in a snapshot. message Table { + // name of the table. string name = 1; + // config is the configuration of the table. frostdb.table.v1alpha1.TableConfig config = 2; + // TableBlock message TableBlock { + // ulid is the unique identifier of the block. bytes ulid = 1; + // size is the size of the block in bytes. int64 size = 2; + // min_tx is the minimum transaction id in the block. uint64 min_tx = 3; + // prev_tx is the transaction id of the previous block. uint64 prev_tx = 4; } + // active_block is the active block of the table. TableBlock active_block = 3; + // granule_metadata is a list of all granules in the table. repeated Granule granule_metadata = 4; } +// Granule encodes metadata about a granule in a table. message Granule { + // part_metadata is a list of all parts in the granule. repeated Part part_metadata = 1; } +// Part encodes metadata about a part in a granule. message Part { + // start_offset is the start offset of the part. int64 start_offset = 1; + // end_offset is the end offset of the part. int64 end_offset = 2; + // tx is the transaction id of the part. uint64 tx = 3; + // compaction_level is the compaction level of the part. uint64 compaction_level = 4; + // Encoding is a enum of possible encodings of the part. enum Encoding { - ENCODING_UNKNOWN = 0; + // ENCODING_UNSPECIFIED is an unknown encoding. + ENCODING_UNSPECIFIED = 0; + // ENCODING_PARQUET is a parquet encoding. ENCODING_PARQUET = 1; + // ENCODING_ARROW is an arrow encoding. ENCODING_ARROW = 2; } + // encoding is the actual encoding of the part. Encoding encoding = 5; } diff --git a/proto/frostdb/wal/v1alpha1/wal.proto b/proto/frostdb/wal/v1alpha1/wal.proto index 329c606af..c3c7bf863 100644 --- a/proto/frostdb/wal/v1alpha1/wal.proto +++ b/proto/frostdb/wal/v1alpha1/wal.proto @@ -44,6 +44,7 @@ message Entry { bytes block_id = 2; } + // The snapshot entry. message Snapshot { // The tx the snapshot was taken at. uint64 tx = 1;