From ed596de476520f8c1d0f895ea556511d9908cc0b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 06:18:14 +0000 Subject: [PATCH] Update github.com/thanos-io/objstore digest to 62b28d1 --- go.mod | 2 +- go.sum | 4 +-- .../thanos-io/objstore/CHANGELOG.md | 6 ++++ .../github.com/thanos-io/objstore/objstore.go | 30 ++++++++++++++----- .../thanos-io/objstore/providers/gcs/gcs.go | 10 +++++-- .../thanos-io/objstore/providers/s3/s3.go | 14 ++++++++- vendor/modules.txt | 2 +- 7 files changed, 54 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 7a82419ec5d6a..32621b2d13a3d 100644 --- a/go.mod +++ b/go.mod @@ -130,7 +130,7 @@ require ( github.com/prometheus/alertmanager v0.26.0 github.com/prometheus/common/sigv4 v0.1.0 github.com/richardartoul/molecule v1.0.0 - github.com/thanos-io/objstore v0.0.0-20230829152104-1b257a36f9a3 + github.com/thanos-io/objstore v0.0.0-20231110045419-62b28d13e408 github.com/willf/bloom v2.0.3+incompatible go.opentelemetry.io/collector/pdata v1.0.0-rcv0015 go4.org/netipx v0.0.0-20230125063823-8449b0a6169f diff --git a/go.sum b/go.sum index 4bdb7d2237462..c2c1c10983652 100644 --- a/go.sum +++ b/go.sum @@ -1678,8 +1678,8 @@ github.com/tbrandon/mbserver v0.0.0-20170611213546-993e1772cc62/go.mod h1:qUzPVl github.com/tedsuo/ifrit v0.0.0-20191009134036-9a97d0632f00/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= github.com/tencentcloud/tencentcloud-sdk-go v1.0.162/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= -github.com/thanos-io/objstore v0.0.0-20230829152104-1b257a36f9a3 h1:avZFY25vRM35FggTBQj2WXq45yEvIKbDLUcNDrJLfKU= -github.com/thanos-io/objstore v0.0.0-20230829152104-1b257a36f9a3/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE= +github.com/thanos-io/objstore v0.0.0-20231110045419-62b28d13e408 h1:pV6OoF/JgabYHWTU8xijCoX3LLVDj1QphrYwImPuwLA= +github.com/thanos-io/objstore v0.0.0-20231110045419-62b28d13e408/go.mod h1:q369VBtseI5OQbK9IsGDfQCfcVu1fsur7ynUcojxnDA= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= diff --git a/vendor/github.com/thanos-io/objstore/CHANGELOG.md b/vendor/github.com/thanos-io/objstore/CHANGELOG.md index 32d0f74268b4c..60d9fa90d2052 100644 --- a/vendor/github.com/thanos-io/objstore/CHANGELOG.md +++ b/vendor/github.com/thanos-io/objstore/CHANGELOG.md @@ -14,6 +14,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#33](https://github.com/thanos-io/objstore/pull/33) Tracing: Add `ContextWithTracer()` to inject the tracer into the context. - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. - [#62](https://github.com/thanos-io/objstore/pull/62) S3: Fix ignored context cancellation in `Iter` method. +- [#77](https://github.com/thanos-io/objstore/pull/77) Fix buckets wrapped with metrics from being unable to determine object sizes in `Upload`. +- [#78](https://github.com/thanos-io/objstore/pull/78) S3: Fix possible concurrent modification of the PutUserMetadata map. +- [#79](https://github.com/thanos-io/objstore/pull/79) Metrics: Fix `objstore_bucket_operation_duration_seconds` for `iter` operations. ### Added - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. @@ -27,9 +30,12 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. +- [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes written to the bucket operation Upload. - [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. - [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile - [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. +- [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects. +- [#85](https://github.com/thanos-io/objstore/pull/85) S3: Allow checksum algorithm to be configured ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. diff --git a/vendor/github.com/thanos-io/objstore/objstore.go b/vendor/github.com/thanos-io/objstore/objstore.go index 84eafce3df4a8..219a9572faacd 100644 --- a/vendor/github.com/thanos-io/objstore/objstore.go +++ b/vendor/github.com/thanos-io/objstore/objstore.go @@ -433,7 +433,7 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu opsDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Name: "objstore_bucket_operation_duration_seconds", - Help: "Duration of successful operations against the bucket", + Help: "Duration of successful operations against the bucket per operation - iter operations include time spent on each callback.", ConstLabels: prometheus.Labels{"bucket": name}, Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }, []string{"operation"}), @@ -458,11 +458,12 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) } - // fetched bytes only relevant for get and getrange + + // fetched bytes only relevant for get, getrange and upload for _, op := range []string{ OpGet, OpGetRange, - // TODO: Add uploads + OpUpload, } { bkt.opsTransferredBytes.WithLabelValues(op) } @@ -503,12 +504,14 @@ func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string) const op = OpIter b.ops.WithLabelValues(op).Inc() + start := time.Now() err := b.bkt.Iter(ctx, dir, f, options...) if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } } + b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) return err } @@ -592,15 +595,25 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.ops.WithLabelValues(op).Inc() - start := time.Now() - if err := b.bkt.Upload(ctx, name, r); err != nil { + trc := newTimingReadCloser( + NopCloserWithSize(r), + op, + b.opsDuration, + b.opsFailures, + b.isOpFailureExpected, + nil, + b.opsTransferredBytes, + ) + defer trc.Close() + err := b.bkt.Upload(ctx, name, trc) + if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } return err } b.lastSuccessfulUploadTime.SetToCurrentTime() - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + return nil } @@ -692,7 +705,10 @@ func (rc *timingReadCloser) Close() error { func (rc *timingReadCloser) Read(b []byte) (n int, err error) { n, err = rc.ReadCloser.Read(b) - rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + if rc.fetchedBytes != nil { + rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + } + rc.readBytes += int64(n) // Report metric just once. if !rc.alreadyGotErr && err != nil && err != io.EOF { diff --git a/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go b/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go index 5ea45c7e97bda..ad305d6e15089 100644 --- a/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go +++ b/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go @@ -108,10 +108,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt delimiter = "" } - it := b.bkt.Objects(ctx, &storage.Query{ + query := &storage.Query{ Prefix: dir, Delimiter: delimiter, - }) + } + err := query.SetAttrSelection([]string{"Name"}) + if err != nil { + return err + } + + it := b.bkt.Objects(ctx, query) for { select { case <-ctx.Done(): diff --git a/vendor/github.com/thanos-io/objstore/providers/s3/s3.go b/vendor/github.com/thanos-io/objstore/providers/s3/s3.go index f92d397398738..507b550bf7beb 100644 --- a/vendor/github.com/thanos-io/objstore/providers/s3/s3.go +++ b/vendor/github.com/thanos-io/objstore/providers/s3/s3.go @@ -113,6 +113,7 @@ var DefaultConfig = Config{ }, PartSize: 1024 * 1024 * 64, // 64MB. BucketLookupType: AutoLookup, + SendContentMd5: true, // Default to using MD5. } // HTTPConfig exists here only because Cortex depends on it, and we depend on Cortex. @@ -136,6 +137,7 @@ type Config struct { TraceConfig TraceConfig `yaml:"trace"` ListObjectsVersion string `yaml:"list_objects_version"` BucketLookupType BucketLookupType `yaml:"bucket_lookup_type"` + SendContentMd5 bool `yaml:"send_content_md5"` // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. // NOTE we need to make sure this number does not produce more parts than 10 000. PartSize uint64 `yaml:"part_size"` @@ -166,6 +168,7 @@ type Bucket struct { storageClass string partSize uint64 listObjectsV1 bool + sendContentMd5 bool } // parseConfig unmarshals a buffer into a Config with default values. @@ -334,6 +337,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B storageClass: storageClass, partSize: config.PartSize, listObjectsV1: config.ListObjectsVersion == "v1", + sendContentMd5: config.SendContentMd5, } return bkt, nil } @@ -492,6 +496,13 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { if size < int64(partSize) { partSize = 0 } + + // Cloning map since minio may modify it + userMetadata := make(map[string]string, len(b.putUserMetadata)) + for k, v := range b.putUserMetadata { + userMetadata[k] = v + } + if _, err := b.client.PutObject( ctx, b.name, @@ -501,8 +512,9 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { minio.PutObjectOptions{ PartSize: partSize, ServerSideEncryption: sse, - UserMetadata: b.putUserMetadata, + UserMetadata: userMetadata, StorageClass: b.storageClass, + SendContentMd5: b.sendContentMd5, // 4 is what minio-go have as the default. To be certain we do micro benchmark before any changes we // ensure we pin this number to four. // TODO(bwplotka): Consider adjusting this number to GOMAXPROCS or to expose this in config if it becomes bottleneck. diff --git a/vendor/modules.txt b/vendor/modules.txt index ae84c85f97224..cff496a72a002 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1385,7 +1385,7 @@ github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require github.com/stretchr/testify/suite -# github.com/thanos-io/objstore v0.0.0-20230829152104-1b257a36f9a3 +# github.com/thanos-io/objstore v0.0.0-20231110045419-62b28d13e408 ## explicit; go 1.18 github.com/thanos-io/objstore github.com/thanos-io/objstore/exthttp