Skip to content

Commit

Permalink
Update parquet-go (grafana#3231)
Browse files Browse the repository at this point in the history 
* upgrayyed

Signed-off-by: Joe Elliott <[email protected]>

* fixed dict ordering

Signed-off-by: Joe Elliott <[email protected]>

* update serverless

Signed-off-by: Joe Elliott <[email protected]>

* updated stoewer/parquet-cli

Signed-off-by: Joe Elliott <[email protected]>

* fix cli

Signed-off-by: Joe Elliott <[email protected]>

---------

Signed-off-by: Joe Elliott <[email protected]>
  • Loading branch information
@joe-elliott
joe-elliott authored Dec 14, 2023
1 parent dd0992a commit b29c941
Show file tree
Hide file tree
Showing 28 changed files with 339 additions and 194 deletions.
10 changes: 7 additions & 3 deletions cmd/tempo-cli/cmd-list-column.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,13 @@ func (cmd *listColumnCmd) Run(ctx *globalOptions) error {
fmt.Printf("\n*************** rowgroup %d ********************\n\n\n", i)

pages := cc.Pages()
numPages := cc.ColumnIndex().NumPages()
fmt.Println("Min Value of rowgroup", cc.ColumnIndex().MinValue(0).Bytes())
fmt.Println("Max Value of rowgroup", cc.ColumnIndex().MaxValue(numPages-1).Bytes())
idx, err := cc.ColumnIndex()
if err != nil {
return err
}
numPages := idx.NumPages()
fmt.Println("Min Value of rowgroup", idx.MinValue(0).Bytes())
fmt.Println("Max Value of rowgroup", idx.MaxValue(numPages-1).Bytes())

buffer := make([]parquet.Value, 10000)
for {
Expand Down
8 changes: 6 additions & 2 deletions cmd/tempo-cli/cmd-view-pq-schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,12 @@ func (cmd *viewSchemaCmd) Run(ctx *globalOptions) error {
path, _ := getNodePathByIndex(pf.Root(), "", cc.Column())

var size int64
for pg := 0; pg < cc.OffsetIndex().NumPages(); pg++ {
size += cc.OffsetIndex().CompressedPageSize(pg)
idx, err := cc.OffsetIndex()
if err != nil {
return err
}
for pg := 0; pg < idx.NumPages(); pg++ {
size += idx.CompressedPageSize(pg)
}

columnSizes[path] = columnSizes[path] + size
Expand Down
2 changes: 1 addition & 1 deletion cmd/tempo-serverless/cloud-run/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ require (
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04 // indirect
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
Expand Down
4 changes: 2 additions & 2 deletions cmd/tempo-serverless/cloud-run/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -376,8 +376,8 @@ github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NH
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04 h1:v5xe/XduDhXXcsbidemHvFrfkdDUt5gi5JzsC04lZ2w=
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04/go.mod h1:6pu/Ca02WRyWyF6jbY1KceESGBZMsRMSijjLbajXaG8=
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06 h1:IERPFFXg0xUx3eX/IEtWkLUh8GtPSLS6gVF7ho7LmwU=
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06/go.mod h1:4YfUo8TkoGoqwzhA/joZKZ8f77wSMShOLHESY4Ys0bY=
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
Expand Down
2 changes: 1 addition & 1 deletion cmd/tempo-serverless/lambda/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ require (
github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e // indirect
github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04 // indirect
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
Expand Down
4 changes: 2 additions & 2 deletions cmd/tempo-serverless/lambda/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,8 +380,8 @@ github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NH
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04 h1:v5xe/XduDhXXcsbidemHvFrfkdDUt5gi5JzsC04lZ2w=
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04/go.mod h1:6pu/Ca02WRyWyF6jbY1KceESGBZMsRMSijjLbajXaG8=
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06 h1:IERPFFXg0xUx3eX/IEtWkLUh8GtPSLS6gVF7ho7LmwU=
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06/go.mod h1:4YfUo8TkoGoqwzhA/joZKZ8f77wSMShOLHESY4Ys0bY=
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ require (
github.com/prometheus/client_model v0.5.0
github.com/prometheus/common v0.45.0
github.com/prometheus/prometheus v0.47.2
github.com/prometheus/statsd_exporter v0.22.7 // indirect
github.com/prometheus/statsd_exporter v0.22.7
github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sony/gobreaker v0.4.1
Expand Down Expand Up @@ -109,8 +109,8 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.89.0
github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.89.0
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04
github.com/stoewer/parquet-cli v0.0.5
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06
github.com/stoewer/parquet-cli v0.0.7
go.opentelemetry.io/collector/config/configgrpc v0.89.0
go.opentelemetry.io/collector/config/confighttp v0.89.0
go.opentelemetry.io/collector/config/configtls v0.89.0
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -838,8 +838,8 @@ github.com/openzipkin/zipkin-go v0.4.2 h1:zjqfqHjUpPmB3c1GlCvvgsM1G4LkvqQbBDueDO
github.com/openzipkin/zipkin-go v0.4.2/go.mod h1:ZeVkFjuuBiSy13y8vpSDCjMi9GoI3hPpCJSBx/EYFhY=
github.com/ovh/go-ovh v1.4.1 h1:VBGa5wMyQtTP7Zb+w97zRCh9sLtM/2YKRyy+MEJmWaM=
github.com/ovh/go-ovh v1.4.1/go.mod h1:6bL6pPyUT7tBfI0pqOegJgRjgjuO+mOo+MyXd1EEC0M=
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04 h1:v5xe/XduDhXXcsbidemHvFrfkdDUt5gi5JzsC04lZ2w=
github.com/parquet-go/parquet-go v0.18.1-0.20231023162157-1748e4b3ff04/go.mod h1:6pu/Ca02WRyWyF6jbY1KceESGBZMsRMSijjLbajXaG8=
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06 h1:IERPFFXg0xUx3eX/IEtWkLUh8GtPSLS6gVF7ho7LmwU=
github.com/parquet-go/parquet-go v0.20.1-0.20231211122939-bec192ac8b06/go.mod h1:4YfUo8TkoGoqwzhA/joZKZ8f77wSMShOLHESY4Ys0bY=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
Expand Down Expand Up @@ -976,8 +976,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc=
github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg=
github.com/stoewer/parquet-cli v0.0.5 h1:DxcMD61f08/cl0jbthKRXsDEbAOGrK+yvdJG/sIJ1do=
github.com/stoewer/parquet-cli v0.0.5/go.mod h1:VETNTgL1SWVqH8Vq04C1Hd0esgQwZF78gSuACSpFgb8=
github.com/stoewer/parquet-cli v0.0.7 h1:rhdZODIbyMS3twr4OM3am8BPPT5pbfMcHLH93whDM5o=
github.com/stoewer/parquet-cli v0.0.7/go.mod h1:bskxHdj8q3H1EmfuCqjViFoeO3NEvs5lzZAQvI8Nfjk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
Expand Down
28 changes: 16 additions & 12 deletions pkg/parquetquery/predicates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,12 @@ func (p *StringInPredicate) String() string {
}

func (p *StringInPredicate) KeepColumnChunk(cc *ColumnChunkHelper) bool {
if ci := cc.ColumnIndex(); ci != nil {
if d := cc.Dictionary(); d != nil {
return keepDictionary(d, p.KeepValue)
}

ci, err := cc.ColumnIndex()
if err != nil && ci != nil {
for _, subs := range p.ss {
for i := 0; i < ci.NumPages(); i++ {
ok := bytes.Compare(ci.MinValue(i).ByteArray(), subs) <= 0 && bytes.Compare(ci.MaxValue(i).ByteArray(), subs) >= 0
Expand All @@ -62,10 +67,6 @@ func (p *StringInPredicate) KeepColumnChunk(cc *ColumnChunkHelper) bool {
return false
}

if d := cc.Dictionary(); d != nil {
return keepDictionary(d, p.KeepValue)
}

return true
}

Expand Down Expand Up @@ -246,7 +247,8 @@ func (p *IntBetweenPredicate) String() string {
}

func (p *IntBetweenPredicate) KeepColumnChunk(c *ColumnChunkHelper) bool {
if ci := c.ColumnIndex(); ci != nil {
ci, err := c.ColumnIndex()
if err != nil && ci != nil {
for i := 0; i < ci.NumPages(); i++ {
min := ci.MinValue(i).Int64()
max := ci.MaxValue(i).Int64()
Expand Down Expand Up @@ -294,11 +296,16 @@ func (p *GenericPredicate[T]) String() string {
}

func (p *GenericPredicate[T]) KeepColumnChunk(c *ColumnChunkHelper) bool {
if d := c.Dictionary(); d != nil {
return keepDictionary(d, p.KeepValue)
}

if p.RangeFn == nil {
return true
}

if ci := c.ColumnIndex(); ci != nil {
ci, err := c.ColumnIndex()
if err != nil && ci != nil {
for i := 0; i < ci.NumPages(); i++ {
min := p.Extract(ci.MinValue(i))
max := p.Extract(ci.MaxValue(i))
Expand All @@ -309,10 +316,6 @@ func (p *GenericPredicate[T]) KeepColumnChunk(c *ColumnChunkHelper) bool {
return false
}

if d := c.Dictionary(); d != nil {
return keepDictionary(d, p.KeepValue)
}

return true
}

Expand Down Expand Up @@ -367,7 +370,8 @@ func (p *FloatBetweenPredicate) String() string {
}

func (p *FloatBetweenPredicate) KeepColumnChunk(c *ColumnChunkHelper) bool {
if ci := c.ColumnIndex(); ci != nil {
ci, err := c.ColumnIndex()
if err != nil && ci != nil {
for i := 0; i < ci.NumPages(); i++ {
min := ci.MinValue(i).Double()
max := ci.MaxValue(i).Double()
Expand Down
4 changes: 3 additions & 1 deletion vendor/github.com/parquet-go/parquet-go/column.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit b29c941

Please sign in to comment.