diff --git a/CHANGELOG.md b/CHANGELOG.md index c64facebec8..0db02baaef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,38 @@ ## main / unreleased -+ [BUGFIX] Change exit code if config is successfully verified [#3174](https://github.com/grafana/tempo/pull/3174) (@am3o @agrib-01) +* [BUGFIX] Change exit code if config is successfully verified [#3174](https://github.com/grafana/tempo/pull/3174) (@am3o @agrib-01) +* [BUGFIX] The tempo-cli analyse blocks command no longer fails on compacted blocks [#3183](https://github.com/grafana/tempo/pull/3183) (@stoewer) * [ENHANCEMENT] Introduced `AttributePolicyMatch` & `IntrinsicPolicyMatch` structures to match span attributes based on strongly typed values & precompiled regexp [#3025](https://github.com/grafana/tempo/pull/3025) (@andriusluk) * [CHANGE] TraceQL/Structural operators performance improvement. [#3088](https://github.com/grafana/tempo/pull/3088) (@joe-elliott) * [CHANGE] Merge the processors overrides set through runtime overrides and user-configurable overrides [#3125](https://github.com/grafana/tempo/pull/3125) (@kvrhdn) * [CHANGE] Make vParquet3 the default block encoding [#2526](https://github.com/grafana/tempo/pull/3134) (@stoewer) +* [CHANGE] Major cache refactor to allow multiple role based caches to be configured [#3166](https://github.com/grafana/tempo/pull/3166). + **BREAKING CHANGE** Deprecate the following fields. These have all been migrated to a top level "cache:" field. + ``` + storage: + trace: + cache: + search: + cache_control: + background_cache: + memcached: + redis: + ``` * [FEATURE] Introduce list_blocks_concurrency on GCS and S3 backends to control backend load and performance. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala) * [FEATURE] Add per-tenant compaction window [#3129](https://github.com/grafana/tempo/pull/3129) (@zalegrala) -* [BUGFIX] Include statusMessage intrinsic attribute in tag search. [#3084](https://github.com/grafana/tempo/pull/3084) (@rcrowe) * [ENHANCEMENT] Make the trace ID label name configurable for remote written exemplars [#3074](https://github.com/grafana/tempo/pull/3074) * [ENHANCEMENT] Update poller to make use of previous results and reduce backend load. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala) * [ENHANCEMENT] Improve TraceQL regex performance in certain queries. [#3139](https://github.com/grafana/tempo/pull/3139) (@joe-elliott) * [ENHANCEMENT] Improve TraceQL performance in complex queries. [#3113](https://github.com/grafana/tempo/pull/3113) (@joe-elliott) -* [BUGFIX] Fix compactor ignore configured S3 headers [#3149](https://github.com/grafana/tempo/pull/3154) (@Batkilin) * [BUGFIX] Prevent building parquet iterators that would loop forever. [#3159](https://github.com/grafana/tempo/pull/3159) (@mapno) * [BUGFIX] Sanitize name in mapped dimensions in span-metrics processor [#3171](https://github.com/grafana/tempo/pull/3171) (@mapno) +## v2.3.1 / 2023-11-28 + +* [BUGFIX] Include statusMessage intrinsic attribute in tag search. [#3084](https://github.com/grafana/tempo/pull/3084) (@rcrowe) +* [BUGFIX] Fix compactor ignore configured S3 headers [#3149](https://github.com/grafana/tempo/pull/3154) (@Batkilin) +* [BUGFIX] Readd session token to s3 credentials. [#3144](https://github.com/grafana/tempo/pull/3144) (@farodin91) + ## v2.3.0 / 2023-10-30 * [CHANGE] Update Go to 1.21 [#2486](https://github.com/grafana/tempo/pull/2829) (@zalegrala) diff --git a/cmd/tempo-cli/cmd-analyse-block.go b/cmd/tempo-cli/cmd-analyse-block.go index 49ef29d7c4c..885efe1f802 100644 --- a/cmd/tempo-cli/cmd-analyse-block.go +++ b/cmd/tempo-cli/cmd-analyse-block.go @@ -104,6 +104,9 @@ func (cmd *analyseBlockCmd) Run(ctx *globalOptions) error { blockSum, err := processBlock(r, c, cmd.TenantID, cmd.BlockID, time.Hour, 0) if err != nil { + if errors.Is(err, backend.ErrDoesNotExist) { + return fmt.Errorf("unable to analyze block: block has no block.meta because it was compacted") + } return err } @@ -118,15 +121,9 @@ func processBlock(r backend.Reader, _ backend.Compactor, tenantID, blockID strin id := uuid.MustParse(blockID) meta, err := r.BlockMeta(context.TODO(), id, tenantID) - if err != nil && !errors.Is(err, backend.ErrDoesNotExist) { + if err != nil { return nil, err } - - if meta == nil { - fmt.Println("Unable to load any meta for block", blockID) - return nil, nil - } - if meta.CompactionLevel < minCompactionLvl { return nil, nil } @@ -134,11 +131,11 @@ func processBlock(r backend.Reader, _ backend.Compactor, tenantID, blockID strin var reader io.ReaderAt switch meta.Version { case vparquet.VersionString: - reader = vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta.BlockID, meta.TenantID) + reader = vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta) case vparquet2.VersionString: - reader = vparquet2.NewBackendReaderAt(context.Background(), r, vparquet2.DataFileName, meta.BlockID, meta.TenantID) + reader = vparquet2.NewBackendReaderAt(context.Background(), r, vparquet2.DataFileName, meta) case vparquet3.VersionString: - reader = vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta.BlockID, meta.TenantID) + reader = vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta) default: fmt.Println("Unsupported block version:", meta.Version) return nil, nil @@ -344,7 +341,7 @@ func printSummary(scope string, max int, summary genericAttrSummary) error { return w.Flush() } -func printDedicatedColumnOverridesJsonnet(spanSummary genericAttrSummary, resourceSummary genericAttrSummary) { +func printDedicatedColumnOverridesJsonnet(spanSummary, resourceSummary genericAttrSummary) { fmt.Println("") fmt.Printf("parquet_dedicated_columns: [\n") diff --git a/cmd/tempo-cli/cmd-analyse-blocks.go b/cmd/tempo-cli/cmd-analyse-blocks.go index 0823917d41d..65f6ddcdb2f 100644 --- a/cmd/tempo-cli/cmd-analyse-blocks.go +++ b/cmd/tempo-cli/cmd-analyse-blocks.go @@ -2,7 +2,12 @@ package main import ( "context" + "errors" "time" + + "github.com/google/uuid" + + "github.com/grafana/tempo/tempodb/backend" ) type analyseBlocksCmd struct { @@ -26,17 +31,30 @@ func (cmd *analyseBlocksCmd) Run(ctx *globalOptions) error { return err } - processedBlocks := 0 + processedBlocks := map[uuid.UUID]struct{}{} topSpanAttrs, topResourceAttrs := make(map[string]uint64), make(map[string]uint64) totalSpanBytes, totalResourceBytes := uint64(0), uint64(0) - for _, block := range blocks { - if processedBlocks >= cmd.MaxBlocks { - break + + for i := 0; i < len(blocks) && len(processedBlocks) < cmd.MaxBlocks; i++ { + block := blocks[i] + if _, ok := processedBlocks[block]; ok { + continue } blockSum, err := processBlock(r, c, cmd.TenantID, block.String(), time.Hour, uint8(cmd.MinCompactionLevel)) if err != nil { - return err + if !errors.Is(err, backend.ErrDoesNotExist) { + return err + } + + // the block was already compacted and blocks might be outdated: refreshing blocks + blocks, _, err = r.Blocks(context.Background(), cmd.TenantID) + if err != nil { + return err + } + i = -1 + + continue } if blockSum == nil { @@ -53,8 +71,9 @@ func (cmd *analyseBlocksCmd) Run(ctx *globalOptions) error { } totalResourceBytes += blockSum.resourceSummary.totalBytes - processedBlocks++ + processedBlocks[block] = struct{}{} } + // Get top N attributes from map return (&blockSummary{ spanSummary: genericAttrSummary{ diff --git a/cmd/tempo-cli/cmd-gen-bloom.go b/cmd/tempo-cli/cmd-gen-bloom.go index ad43f6cc635..e3ccf553c93 100644 --- a/cmd/tempo-cli/cmd-gen-bloom.go +++ b/cmd/tempo-cli/cmd-gen-bloom.go @@ -120,7 +120,7 @@ func (cmd *bloomCmd) Run(ctx *globalOptions) error { } for i := 0; i < len(bloomBytes); i++ { - err = w.Write(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, bloomBytes[i], false) + err = w.Write(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, bloomBytes[i], nil) if err != nil { fmt.Println("error writing bloom filter to backend", err) return err @@ -132,7 +132,7 @@ func (cmd *bloomCmd) Run(ctx *globalOptions) error { // verify generated bloom shardedBloomFilter := make([]*willf_bloom.BloomFilter, meta.BloomShardCount) for i := 0; i < int(meta.BloomShardCount); i++ { - bloomBytes, err := r.Read(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, false) + bloomBytes, err := r.Read(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, nil) if err != nil { fmt.Println("error reading bloom from backend") return nil diff --git a/cmd/tempo-cli/cmd-gen-index.go b/cmd/tempo-cli/cmd-gen-index.go index a7687754860..279937ac7b1 100644 --- a/cmd/tempo-cli/cmd-gen-index.go +++ b/cmd/tempo-cli/cmd-gen-index.go @@ -140,7 +140,7 @@ func (cmd *indexCmd) Run(ctx *globalOptions) error { } // write to the local backend - err = w.Write(context.TODO(), "index", blockID, cmd.TenantID, indexBytes, false) + err = w.Write(context.TODO(), "index", blockID, cmd.TenantID, indexBytes, nil) if err != nil { fmt.Println("error writing index to backend", err) return err diff --git a/cmd/tempo-cli/cmd-list-column.go b/cmd/tempo-cli/cmd-list-column.go index 1d6c2f99ef9..9b1b7cef34e 100644 --- a/cmd/tempo-cli/cmd-list-column.go +++ b/cmd/tempo-cli/cmd-list-column.go @@ -32,7 +32,7 @@ func (cmd *listColumnCmd) Run(ctx *globalOptions) error { return err } - rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta.BlockID, meta.TenantID) + rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta) pf, err := parquet.OpenFile(rr, int64(meta.Size)) if err != nil { return err diff --git a/cmd/tempo-cli/cmd-view-pq-schema.go b/cmd/tempo-cli/cmd-view-pq-schema.go index 5d3eac3c7da..72d48fc4cac 100644 --- a/cmd/tempo-cli/cmd-view-pq-schema.go +++ b/cmd/tempo-cli/cmd-view-pq-schema.go @@ -36,7 +36,7 @@ func (cmd *viewSchemaCmd) Run(ctx *globalOptions) error { fmt.Printf("\n*************** block meta *********************\n\n\n") fmt.Printf("%+v\n", meta) - rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta.BlockID, meta.TenantID) + rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta) pf, err := parquet.OpenFile(rr, int64(meta.Size)) if err != nil { return err diff --git a/cmd/tempo/app/app.go b/cmd/tempo/app/app.go index 7a7101c3a9b..a24dcf50255 100644 --- a/cmd/tempo/app/app.go +++ b/cmd/tempo/app/app.go @@ -38,6 +38,7 @@ import ( "github.com/grafana/tempo/modules/querier" "github.com/grafana/tempo/modules/storage" "github.com/grafana/tempo/pkg/api" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/usagestats" "github.com/grafana/tempo/pkg/util" "github.com/grafana/tempo/pkg/util/log" @@ -77,6 +78,7 @@ type App struct { generator *generator.Generator store storage.Store usageReport *usagestats.Reporter + cacheProvider cache.Provider MemberlistKV *memberlist.KVInitService HTTPAuthMiddleware middleware.Interface diff --git a/cmd/tempo/app/config.go b/cmd/tempo/app/config.go index a90d66c6d7f..7e3daef73bd 100644 --- a/cmd/tempo/app/config.go +++ b/cmd/tempo/app/config.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/dskit/server" "github.com/prometheus/client_golang/prometheus" + "github.com/grafana/tempo/modules/cache" "github.com/grafana/tempo/modules/compactor" "github.com/grafana/tempo/modules/distributor" "github.com/grafana/tempo/modules/frontend" @@ -53,6 +54,7 @@ type Config struct { Overrides overrides.Config `yaml:"overrides,omitempty"` MemberlistKV memberlist.KVConfig `yaml:"memberlist,omitempty"` UsageReport usagestats.Config `yaml:"usage_report,omitempty"` + CacheProvider cache.Config `yaml:"cache,omitempty"` } func newDefaultConfig() *Config { @@ -209,6 +211,10 @@ func (c *Config) CheckConfig() []ConfigWarning { warnings = append(warnings, warnNativeAWSAuthEnabled) } + if c.StorageConfig.Trace.Cache != "" { + warnings = append(warnings, warnConfiguredLegacyCache) + } + return warnings } @@ -275,6 +281,11 @@ var ( Message: "c.StorageConfig.Trace.S3.NativeAWSAuthEnabled is deprecated and will be removed in a future release.", Explain: "This setting is no longer necessary and will be ignored.", } + + warnConfiguredLegacyCache = ConfigWarning{ + Message: "c.StorageConfig.Trace.Cache is deprecated and will be removed in a future release.", + Explain: "Please migrate to the top level cache settings config.", + } ) func newV2Warning(setting string) ConfigWarning { diff --git a/cmd/tempo/app/config_test.go b/cmd/tempo/app/config_test.go index 31d26221052..466480bab1c 100644 --- a/cmd/tempo/app/config_test.go +++ b/cmd/tempo/app/config_test.go @@ -33,6 +33,7 @@ func TestConfig_CheckConfig(t *testing.T) { Target: MetricsGenerator, StorageConfig: storage.Config{ Trace: tempodb.Config{ + Cache: "supercache", Backend: backend.S3, BlocklistPoll: time.Minute, Block: &common.BlockConfig{ @@ -57,6 +58,7 @@ func TestConfig_CheckConfig(t *testing.T) { warnBlocklistPollConcurrency, warnLogReceivedTraces, warnNativeAWSAuthEnabled, + warnConfiguredLegacyCache, }, }, { diff --git a/cmd/tempo/app/modules.go b/cmd/tempo/app/modules.go index 18ff9e3aa6f..c53f818b6f2 100644 --- a/cmd/tempo/app/modules.go +++ b/cmd/tempo/app/modules.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/grafana/tempo/modules/cache" "github.com/grafana/tempo/modules/compactor" "github.com/grafana/tempo/modules/distributor" "github.com/grafana/tempo/modules/frontend" @@ -54,6 +55,8 @@ const ( UsageReport string = "usage-report" Overrides string = "overrides" OverridesAPI string = "overrides-api" + CacheProvider string = "cache-provider" + // rings IngesterRing string = "ring" SecondaryIngesterRing string = "secondary-ring" @@ -430,7 +433,7 @@ func (t *App) initCompactor() (services.Service, error) { } func (t *App) initStore() (services.Service, error) { - store, err := tempo_storage.NewStore(t.cfg.StorageConfig, log.Logger) + store, err := tempo_storage.NewStore(t.cfg.StorageConfig, t.cacheProvider, log.Logger) if err != nil { return nil, fmt.Errorf("failed to create store: %w", err) } @@ -510,6 +513,16 @@ func (t *App) initUsageReport() (services.Service, error) { return ur, nil } +func (t *App) initCacheProvider() (services.Service, error) { + c, err := cache.NewProvider(&t.cfg.CacheProvider, util_log.Logger) + if err != nil { + return nil, fmt.Errorf("failed to create cache provider: %w", err) + } + + t.cacheProvider = c + return c, nil +} + func (t *App) setupModuleManager() error { mm := modules.NewManager(log.Logger) @@ -523,6 +536,7 @@ func (t *App) setupModuleManager() error { mm.RegisterModule(Overrides, t.initOverrides, modules.UserInvisibleModule) mm.RegisterModule(OverridesAPI, t.initOverridesAPI) mm.RegisterModule(UsageReport, t.initUsageReport) + mm.RegisterModule(CacheProvider, t.initCacheProvider, modules.UserInvisibleModule) mm.RegisterModule(IngesterRing, t.initIngesterRing, modules.UserInvisibleModule) mm.RegisterModule(MetricsGeneratorRing, t.initGeneratorRing, modules.UserInvisibleModule) mm.RegisterModule(SecondaryIngesterRing, t.initSecondaryIngesterRing, modules.UserInvisibleModule) @@ -540,8 +554,9 @@ func (t *App) setupModuleManager() error { mm.RegisterModule(ScalableSingleBinary, nil) deps := map[string][]string{ - // Store: nil, // InternalServer: nil, + // CacheProvider: nil, + Store: {CacheProvider}, Server: {InternalServer}, Overrides: {Server}, OverridesAPI: {Server, Overrides}, diff --git a/docs/sources/tempo/_index.md b/docs/sources/tempo/_index.md index 94c46884ec9..7ecdfd9693d 100644 --- a/docs/sources/tempo/_index.md +++ b/docs/sources/tempo/_index.md @@ -13,6 +13,8 @@ cascade: Grafana Tempo is an open source, easy-to-use, and high-volume distributed tracing backend. Tempo is cost-efficient, and only requires an object storage to operate. Tempo is deeply integrated with Grafana, Mimir, Prometheus, and Loki. You can use Tempo with open-source tracing protocols, including Jaeger, Zipkin, or OpenTelemetry. +Tempo implements [TraceQL](/docs/tempo/latest/traceql), a traces-first query language inspired by LogQL and PromQL. This query language allows users to very precisely and easily select spans and jump directly to the spans fulfilling the specified conditions. + Tempo integrates well with a number of existing open source tools: - **Grafana** ships with native support for Tempo using the built-in [Tempo data source](/docs/grafana/latest/datasources/tempo/). diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 35549633cf8..4e0afb7a1bb 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -29,6 +29,7 @@ This document explains the configuration options for Tempo as well as the detail - [Standard overrides](#standard-overrides) - [Tenant-specific overrides](#tenant-specific-overrides) - [Override strategies](#override-strategies) + - [Cache](#cache) - [Usage-report](#usage-report) Additionally, you can review [TLS]({{< relref "./tls" >}}) to configure the cluster components to communicate over TLS, or receive traces over TLS. @@ -921,6 +922,7 @@ storage: # Cache type to use. Should be one of "redis", "memcached" # Example: "cache: memcached" + # Deprecated. See [cache](#cache) section below. [cache: ] # Minimum compaction level of block to qualify for bloom filter caching. Default is 0 (disabled), meaning @@ -958,6 +960,7 @@ storage: [read_buffer_count: ] # Granular cache control settings for parquet metadata objects + # Deprecated. See [cache](#cache) section below. cache_control: # Specifies if footer should be cached @@ -969,120 +972,20 @@ storage: # Specifies if offset index should be cached [offset_index: | default = false] - # Cortex Background cache configuration. Requires having a cache configured. - background_cache: - - # at what concurrency to write back to cache. Default is 10. - [writeback_goroutines: ] - # how many key batches to buffer for background write-back. Default is 10000. - [writeback_buffer: ] + # Background cache configuration. Requires having a cache configured. + # Deprecated. See [cache](#cache) section below. + background_cache: # Memcached caching configuration block + # Deprecated. See [cache](#cache) section below. memcached: - # hostname for memcached service to use. If empty and if addresses is unset, no memcached will be used. - # Example: "host: memcached" - [host: ] - - # Optional - # SRV service used to discover memcache servers. (default: memcached) - # Example: "service: memcached-client" - [service: ] - - # Optional - # comma separated addresses list in DNS Service Discovery format. Refer - https://cortexmetrics.io/docs/configuration/arguments/#dns-service-discovery. - # (default: "") - # Example: "addresses: memcached" - [addresses: ] - - # Optional - # Maximum time to wait before giving up on memcached requests. - # (default: 100ms) - [timeout: ] - - # Optional - # Maximum number of idle connections in pool. - # (default: 16) - [max_idle_conns: ] - - # Optional - # period with which to poll DNS for memcache servers. - # (default: 1m) - [update_interval: ] - - # Optional - # use consistent hashing to distribute keys to memcache servers. - # (default: true) - [consistent_hash: ] - - # Optional - # trip circuit-breaker after this number of consecutive dial failures. - # (default: 10) - [circuit_breaker_consecutive_failures: 10] - - # Optional - # duration circuit-breaker remains open after tripping. - # (default: 10s) - [circuit_breaker_timeout: 10s] - - # Optional - # reset circuit-breaker counts after this long. - # (default: 10s) - [circuit_breaker_interval: 10s] - # Redis configuration block # EXPERIMENTAL + # Deprecated. See [cache](#cache) section below. redis: - # redis endpoint to use when caching. - [endpoint: ] - - # optional. - # maximum time to wait before giving up on redis requests. (default 100ms) - [timeout: 500ms] - - # optional. - # redis Sentinel master name. (default "") - # Example: "master-name: redis-master" - [master-name: ] - - # optional. - # database index. (default 0) - [db: ] - - # optional. - # how long keys stay in the redis. (default 0) - [expiration: ] - - # optional. - # enable connecting to redis with TLS. (default false) - [tls-enabled: ] - - # optional. - # skip validating server certificate. (default false) - [tls-insecure-skip-verify: ] - - # optional. - # maximum number of connections in the pool. (default 0) - [pool-size: ] - - # optional. - # password to use when connecting to redis. (default "") - [password: ] - - # optional. - # close connections after remaining idle for this duration. (default 0s) - {idle-timeout: } - - # optional. - # close connections older than this duration. (default 0s) - [max-connection-age: ] - - # optional. - # password to use when connecting to redis sentinel. (default "") - [sentinel_password: ] - # the worker pool is used primarily when finding traces by id, but is also used by other pool: @@ -1572,3 +1475,159 @@ This value is available in the the [tempo-distributed](https://github.com/grafan # -- If true, Tempo will report anonymous usage data about the shape of a deployment to Grafana Labs reportingEnabled: true ``` + +## Cache + +Use this block to configure caches available throughout the application. Multiple caches can be created and assigned roles +which determine how they are used by Tempo. + +```yaml +cache: + # Background cache configuration. Requires having a cache configured. These settings apply + # to all configured caches. + background: + + # At what concurrency to write back to cache. Default is 10. + [writeback_goroutines: ] + + # How many key batches to buffer for background write-back. Default is 10000. + [writeback_buffer: ] + + caches: + + # Roles determine how this cache is used in Tempo. Roles must be unique across all caches and + # every cache must have at least one role. + # Allowed values: + # bloom - Bloom filters for trace id lookup. + # parquet-footer - Parquet footer values. Useful for search and trace by id lookup. + # parquet-column-idx - Parquet column index values. Useful for search and trace by id lookup. + # parquet-offset-idx - Parquet offset index values. Useful for search and trace by id lookup. + + - roles: + - + - + + # Memcached caching configuration block + memcached: + + # Hostname for memcached service to use. If empty and if addresses is unset, no memcached will be used. + # Example: "host: memcached" + [host: ] + + # Optional + # SRV service used to discover memcache servers. (default: memcached) + # Example: "service: memcached-client" + [service: ] + + # Optional + # Comma separated addresses list in DNS Service Discovery format. Refer - https://cortexmetrics.io/docs/configuration/arguments/#dns-service-discovery. + # (default: "") + # Example: "addresses: memcached" + [addresses: ] + + # Optional + # Maximum time to wait before giving up on memcached requests. + # (default: 100ms) + [timeout: ] + + # Optional + # Maximum number of idle connections in pool. + # (default: 16) + [max_idle_conns: ] + + # Optional + # Period with which to poll DNS for memcache servers. + # (default: 1m) + [update_interval: ] + + # Optional + # Use consistent hashing to distribute keys to memcache servers. + # (default: true) + [consistent_hash: ] + + # Optional + # Trip circuit-breaker after this number of consecutive dial failures. + # (default: 10) + [circuit_breaker_consecutive_failures: 10] + + # Optional + # Duration circuit-breaker remains open after tripping. + # (default: 10s) + [circuit_breaker_timeout: 10s] + + # Optional + # Reset circuit-breaker counts after this long. + # (default: 10s) + [circuit_breaker_interval: 10s] + + # Redis configuration block + # EXPERIMENTAL + redis: + + # Redis endpoint to use when caching. + [endpoint: ] + + # optional. + # Maximum time to wait before giving up on redis requests. (default 100ms) + [timeout: 500ms] + + # optional. + # Redis Sentinel master name. (default "") + # Example: "master-name: redis-master" + [master-name: ] + + # optional. + # Database index. (default 0) + [db: ] + + # optional. + # How long keys stay in the redis. (default 0) + [expiration: ] + + # optional. + # Enable connecting to redis with TLS. (default false) + [tls-enabled: ] + + # optional. + # Skip validating server certificate. (default false) + [tls-insecure-skip-verify: ] + + # optional. + # Maximum number of connections in the pool. (default 0) + [pool-size: ] + + # optional. + # Password to use when connecting to redis. (default "") + [password: ] + + # optional. + # Close connections after remaining idle for this duration. (default 0s) + [idle-timeout: ] + + # optional. + # Close connections older than this duration. (default 0s) + [max-connection-age: ] + + # optional. + # Password to use when connecting to redis sentinel. (default "") + [sentinel_password: ] +``` + +Example config: + +```yaml +cache: + background: + writeback_goroutines: 5 + caches: + - roles: + - parquet-footer + - parquet-column-idx + - parquet-offset-idx + memcached: + host: memcached-instance + - roles: + - bloom + redis: + endpoint: redis-instance +``` \ No newline at end of file diff --git a/docs/sources/tempo/configuration/s3.md b/docs/sources/tempo/configuration/s3.md index dd64b47e97a..289ae0fcea3 100644 --- a/docs/sources/tempo/configuration/s3.md +++ b/docs/sources/tempo/configuration/s3.md @@ -29,7 +29,6 @@ The following IAM policy shows minimal permissions required by Tempo, where the "s3:PutObject", "s3:GetObject", "s3:ListBucket", - "s3:ListObjects", "s3:DeleteObject", "s3:GetObjectTagging", "s3:PutObjectTagging" diff --git a/docs/sources/tempo/operations/grafana_datasource.md b/docs/sources/tempo/operations/grafana_datasource.md new file mode 100644 index 00000000000..21cc6b2ec70 --- /dev/null +++ b/docs/sources/tempo/operations/grafana_datasource.md @@ -0,0 +1,77 @@ +--- +title: Grafana Tempo data source +description: Use the Tempo Operator to deploy Tempo and use it as a data source with Grafana +aliases: + - /docs/tempo/latest/gateway + - /docs/tempo/operations/gateway +weight: 15 +--- + +# Grafana Tempo data source + +You can configure the `TempoStack` to send data to Grafana and configure the Tempo data source. + +You can choose to either use Tempo Operator's gateway or not: + +* If the `TempoStack` is deployed using the gateway, you'll need to provide authentication information to Grafana, along with the URL of the tenant from which you expect to see the traces. + +* If the gateway is not used, then you need to make sure Grafana can access the `query-frontend` endpoints. + +For more information, refer to the [Tempo data source for Grafana](/docs/grafana/latest/datasources/tempo/), + +## Use with gateway + +The gateway, an optional component deployed as part of Tempo Operator, provides secure access to Tempo's distributor (for example, for pushing spans) and query-frontend (for example, for querying traces) via consulting an OAuth/OIDC endpoint for the request subject. + +The OIDC configuration expects `clientID` and `clientSecret`. They should be provided via a Kubernetes secret that the `TempoStack` admin provides upfront. + +The gateway exposes all Tempo query endpoints, so you can use the endpoint as a Tempo data source for Grafana. + +If Grafana is configured with some OAuth provider, such as generic OAuth, the `TempoStack` with the gateway should be deployed using the same `clientID` and `clientSecret`: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: oidc-test +stringData: + clientID: + clientSecret: +type: Opaque +``` + +Then deploy `TempoStack` with gateway enabled: + +```yaml +spec: + template: + gateway: + enabled: true + tenants: + mode: static + authentication: + - tenantName: test-oidc + tenantId: test-oidc + oidc: + issuerURL: http://dex:30556/dex + redirectURL: http://tempo-foo-gateway:8080/oidc/test-oidc/callback + usernameClaim: email + secret: + name: oidc-test +``` + +Set the data source URL parameter to `http://:/api/traces/v1/{tenant}/tempo/`, where `{tenant}` is the name of the tenant. + +To use it as a data source, set the Authentication Method to **Forward Oauth Identify** using the same `clientID` and `clientSecret` for gateway and for the OAuth configuration. This will forward the `access_token` to the gateway so it can authenticate the client. + +

Tempo data source configured for the gateway forwarding OAuth access token

+ +If you prefer to set the Bearer token directly and not use the **Forward Oauth Identify**, you can add it to the "Authorization" Header. + +

Tempo data source configured for the gateway using Bearer token

+ +## Without the gateway + +If you are not using the gateway, make sure your Grafana can access to the query-frontend endpoints, you can do this by creating an ingress or a route in OpenShift. + +Once you have the endpoint, you can set it as `URL` when you create the Tempo data source. \ No newline at end of file diff --git a/docs/sources/tempo/operations/grafana_datasource_tempo.png b/docs/sources/tempo/operations/grafana_datasource_tempo.png new file mode 100644 index 00000000000..156fa070d0e Binary files /dev/null and b/docs/sources/tempo/operations/grafana_datasource_tempo.png differ diff --git a/docs/sources/tempo/operations/grafana_datasource_tempo_headers.png b/docs/sources/tempo/operations/grafana_datasource_tempo_headers.png new file mode 100644 index 00000000000..273c1a74a32 Binary files /dev/null and b/docs/sources/tempo/operations/grafana_datasource_tempo_headers.png differ diff --git a/docs/sources/tempo/release-notes/v2-3.md b/docs/sources/tempo/release-notes/v2-3.md index baffab0080a..5a1a6d23665 100644 --- a/docs/sources/tempo/release-notes/v2-3.md +++ b/docs/sources/tempo/release-notes/v2-3.md @@ -183,6 +183,14 @@ The following vulnerabilities have been addressed: For a complete list, refer to the [Tempo changelog](https://github.com/grafana/tempo/releases). +### 2.3.1 + +* Include statusMessage intrinsic attribute in tag search. [PR 3084](https://github.com/grafana/tempo/pull/3084) +* Fix compactor ignore configured S3 headers. [PR 3149](https://github.com/grafana/tempo/pull/3154) +* Readd session token to s3 credentials. [PR 3144](https://github.com/grafana/tempo/pull/3144) + +### 2.3 + * Loaded defaults for the internal server [PR 3041](https://github.com/grafana/tempo/pull/3041) * Fixed pass-through to runtime overrides for FilterPolicies and TargetInfoExcludedDimensions [PR 3012](https://github.com/grafana/tempo/pull/3012) * Fixed a panic in metrics-summary API PR [#2738](https://github.com/grafana/tempo/pull/2738) diff --git a/integration/microservices/tempo.yaml b/integration/microservices/tempo.yaml index 5f768037c4b..c3ce244ba27 100644 --- a/integration/microservices/tempo.yaml +++ b/integration/microservices/tempo.yaml @@ -40,4 +40,5 @@ storage: pool: queue_depth: 2000 wal: - path: /var/tempo/wal \ No newline at end of file + path: /var/tempo/wal + diff --git a/modules/cache/cache.go b/modules/cache/cache.go new file mode 100644 index 00000000000..d1f4ac4da67 --- /dev/null +++ b/modules/cache/cache.go @@ -0,0 +1,97 @@ +package cache + +import ( + "context" + "fmt" + + "github.com/grafana/dskit/services" + "github.com/grafana/tempo/modules/cache/memcached" + "github.com/grafana/tempo/modules/cache/redis" + "github.com/grafana/tempo/pkg/cache" + "github.com/grafana/tempo/pkg/usagestats" + "github.com/prometheus/statsd_exporter/pkg/level" + + "github.com/go-kit/log" +) + +var ( + statMemcached = usagestats.NewInt("cache_memcached") + statRedis = usagestats.NewInt("cache_redis") +) + +type provider struct { + services.Service + + caches map[cache.Role]cache.Cache +} + +// NewProvider creates a new cache provider with the given config. +func NewProvider(cfg *Config, logger log.Logger) (cache.Provider, error) { + p := &provider{ + caches: map[cache.Role]cache.Cache{}, + } + + err := cfg.Validate() + if err != nil { + return nil, fmt.Errorf("invalid cache config: %w", err) + } + + statMemcached.Set(0) + statRedis.Set(0) + + for _, cacheCfg := range cfg.Caches { + var c cache.Cache + + if cacheCfg.MemcachedConfig != nil { + level.Info(logger).Log("msg", "configuring memcached client", "roles", cacheCfg.Name()) + + statMemcached.Add(1) + c = memcached.NewClient(cacheCfg.MemcachedConfig, cfg.Background, cacheCfg.Name(), logger) + } + + if cacheCfg.RedisConfig != nil { + level.Info(logger).Log("msg", "configuring redis client", "roles", cacheCfg.Name()) + + statRedis.Add(1) + c = redis.NewClient(cacheCfg.RedisConfig, cfg.Background, cacheCfg.Name(), logger) + } + + // add this cache for all claimed roles + for _, role := range cacheCfg.Role { + p.caches[role] = c + } + } + + p.Service = services.NewIdleService(p.starting, p.stopping) + return p, nil +} + +// CacheFor is used to retrieve a cache for a given role. +func (p *provider) CacheFor(role cache.Role) cache.Cache { + return p.caches[role] +} + +// AddCache is used to add a cache for a given role. It currently +// only exists to add the legacy cache in tempodb. It should +// likely be removed in the future. +func (p *provider) AddCache(role cache.Role, c cache.Cache) error { + if _, ok := p.caches[role]; ok { + return fmt.Errorf("cache for role %s already exists", role) + } + + p.caches[role] = c + + return nil +} + +func (p *provider) starting(_ context.Context) error { + return nil +} + +func (p *provider) stopping(_ error) error { + for _, c := range p.caches { + c.Stop() + } + + return nil +} diff --git a/modules/cache/config.go b/modules/cache/config.go new file mode 100644 index 00000000000..21d8c82e960 --- /dev/null +++ b/modules/cache/config.go @@ -0,0 +1,83 @@ +package cache + +import ( + "errors" + "fmt" + "strings" + + "github.com/grafana/tempo/modules/cache/memcached" + "github.com/grafana/tempo/modules/cache/redis" + "github.com/grafana/tempo/pkg/cache" +) + +type Config struct { + Background *cache.BackgroundConfig `yaml:"background"` + Caches []CacheConfig `yaml:"caches"` +} + +type CacheConfig struct { // nolint: revive + Role []cache.Role `yaml:"roles"` + MemcachedConfig *memcached.Config `yaml:"memcached"` + RedisConfig *redis.Config `yaml:"redis"` +} + +// Validate validates the config. +func (cfg *Config) Validate() error { + claimedRoles := map[cache.Role]struct{}{} + allRoles := allRoles() + + for _, cacheCfg := range cfg.Caches { + if cacheCfg.MemcachedConfig != nil && cacheCfg.RedisConfig != nil { + return fmt.Errorf("cache config for role %s has both memcached and redis configs", cacheCfg.Role) + } + + if cacheCfg.MemcachedConfig == nil && cacheCfg.RedisConfig == nil { + return fmt.Errorf("cache config for role %s has neither memcached nor redis configs", cacheCfg.Role) + } + + if len(cacheCfg.Role) == 0 { + return errors.New("configured caches require a valid role") + } + + // check that all roles are unique + for _, role := range cacheCfg.Role { + if _, ok := allRoles[role]; !ok { + return fmt.Errorf("role %s is not a valid role", role) + } + + if _, ok := claimedRoles[role]; ok { + return fmt.Errorf("role %s is claimed by more than one cache", role) + } + + claimedRoles[role] = struct{}{} + } + } + + return nil +} + +// Name returns a string representation of the roles claimed by this cache. +func (cfg *CacheConfig) Name() string { + stringRoles := make([]string, len(cfg.Role)) + for i, role := range cfg.Role { + stringRoles[i] = string(role) + } + return strings.Join(stringRoles, "|") +} + +func allRoles() map[cache.Role]struct{} { + all := []cache.Role{ + cache.RoleBloom, + cache.RoleParquetFooter, + cache.RoleParquetColumnIdx, + cache.RoleParquetOffsetIdx, + cache.RoleTraceIDIdx, + } + + roles := map[cache.Role]struct{}{} + for _, role := range all { + roles[role] = struct{}{} + } + + return roles +} diff --git a/modules/cache/config_test.go b/modules/cache/config_test.go new file mode 100644 index 00000000000..d6abbb160e4 --- /dev/null +++ b/modules/cache/config_test.go @@ -0,0 +1,138 @@ +package cache + +import ( + "errors" + "testing" + + "github.com/grafana/tempo/modules/cache/memcached" + "github.com/grafana/tempo/modules/cache/redis" + "github.com/grafana/tempo/pkg/cache" + "github.com/stretchr/testify/require" +) + +func TestConfigValidation(t *testing.T) { + tcs := []struct { + name string + cfg *Config + expected error + }{ + { + name: "no caching is valid", + cfg: &Config{}, + }, + { + name: "valid config", + cfg: &Config{ + Caches: []CacheConfig{ + { + Role: []cache.Role{cache.RoleBloom}, + MemcachedConfig: &memcached.Config{}, + }, + { + Role: []cache.Role{cache.RoleParquetColumnIdx}, + RedisConfig: &redis.Config{}, + }, + }, + }, + }, + { + name: "invalid - duplicate roles", + cfg: &Config{ + Caches: []CacheConfig{ + { + Role: []cache.Role{cache.RoleBloom}, + MemcachedConfig: &memcached.Config{}, + }, + { + Role: []cache.Role{cache.RoleBloom}, + RedisConfig: &redis.Config{}, + }, + }, + }, + expected: errors.New("role bloom is claimed by more than one cache"), + }, + { + name: "invalid - no roles", + cfg: &Config{ + Caches: []CacheConfig{ + { + MemcachedConfig: &memcached.Config{}, + }, + }, + }, + expected: errors.New("configured caches require a valid role"), + }, + { + name: "invalid - both caches configged", + cfg: &Config{ + Caches: []CacheConfig{ + { + Role: []cache.Role{cache.RoleBloom}, + MemcachedConfig: &memcached.Config{}, + RedisConfig: &redis.Config{}, + }, + }, + }, + expected: errors.New("cache config for role [bloom] has both memcached and redis configs"), + }, + { + name: "invalid - no caches configged", + cfg: &Config{ + Caches: []CacheConfig{ + { + Role: []cache.Role{cache.RoleBloom}, + }, + }, + }, + expected: errors.New("cache config for role [bloom] has neither memcached nor redis configs"), + }, + { + name: "invalid - non-existent role", + cfg: &Config{ + Caches: []CacheConfig{ + { + Role: []cache.Role{cache.Role("foo")}, + MemcachedConfig: &memcached.Config{}, + }, + }, + }, + expected: errors.New("role foo is not a valid role"), + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.Validate() + require.Equal(t, tc.expected, err) + }) + } +} + +func TestCacheConfigName(t *testing.T) { + tcs := []struct { + cfg *CacheConfig + expected string + }{ + { + cfg: &CacheConfig{ + Role: []cache.Role{cache.RoleBloom}, + }, + expected: "bloom", + }, + { + cfg: &CacheConfig{ + Role: []cache.Role{cache.RoleBloom, cache.RoleParquetColumnIdx}, + }, + expected: "bloom|parquet-column-idx", + }, + { + cfg: &CacheConfig{}, + expected: "", + }, + } + + for _, tc := range tcs { + actual := tc.cfg.Name() + require.Equal(t, tc.expected, actual) + } +} diff --git a/tempodb/backend/cache/memcached/memcached.go b/modules/cache/memcached/memcached.go similarity index 72% rename from tempodb/backend/cache/memcached/memcached.go rename to modules/cache/memcached/memcached.go index 9c9182ff470..4d1bb87c356 100644 --- a/tempodb/backend/cache/memcached/memcached.go +++ b/modules/cache/memcached/memcached.go @@ -15,7 +15,7 @@ type Config struct { TTL time.Duration `yaml:"ttl"` } -func NewClient(cfg *Config, cfgBackground *cache.BackgroundConfig, logger log.Logger) cache.Cache { +func NewClient(cfg *Config, cfgBackground *cache.BackgroundConfig, name string, logger log.Logger) cache.Cache { if cfg.ClientConfig.MaxIdleConns == 0 { cfg.ClientConfig.MaxIdleConns = 16 } @@ -26,13 +26,13 @@ func NewClient(cfg *Config, cfgBackground *cache.BackgroundConfig, logger log.Lo cfg.ClientConfig.UpdateInterval = time.Minute } - client := cache.NewMemcachedClient(cfg.ClientConfig, "tempo", prometheus.DefaultRegisterer, logger) + client := cache.NewMemcachedClient(cfg.ClientConfig, name, prometheus.DefaultRegisterer, logger) memcachedCfg := cache.MemcachedConfig{ Expiration: cfg.TTL, BatchSize: 0, // we are currently only requesting one key at a time, which is bad. we could restructure Find() to batch request all blooms at once Parallelism: 0, } - c := cache.NewMemcached(memcachedCfg, client, "tempo", prometheus.DefaultRegisterer, logger) + c := cache.NewMemcached(memcachedCfg, client, name, prometheus.DefaultRegisterer, logger) - return cache.NewBackground("tempo", *cfgBackground, c, prometheus.DefaultRegisterer) + return cache.NewBackground(name, *cfgBackground, c, prometheus.DefaultRegisterer) } diff --git a/tempodb/backend/cache/redis/redis.go b/modules/cache/redis/redis.go similarity index 72% rename from tempodb/backend/cache/redis/redis.go rename to modules/cache/redis/redis.go index 14ec94b50b7..005d23627df 100644 --- a/tempodb/backend/cache/redis/redis.go +++ b/modules/cache/redis/redis.go @@ -15,7 +15,7 @@ type Config struct { TTL time.Duration `yaml:"ttl"` } -func NewClient(cfg *Config, cfgBackground *cache.BackgroundConfig, logger log.Logger) cache.Cache { +func NewClient(cfg *Config, cfgBackground *cache.BackgroundConfig, name string, logger log.Logger) cache.Cache { if cfg.ClientConfig.Timeout == 0 { cfg.ClientConfig.Timeout = 100 * time.Millisecond } @@ -24,7 +24,7 @@ func NewClient(cfg *Config, cfgBackground *cache.BackgroundConfig, logger log.Lo } client := cache.NewRedisClient(&cfg.ClientConfig) - c := cache.NewRedisCache("tempo", client, prometheus.DefaultRegisterer, logger) + c := cache.NewRedisCache(name, client, prometheus.DefaultRegisterer, logger) - return cache.NewBackground("tempo", *cfgBackground, c, prometheus.DefaultRegisterer) + return cache.NewBackground(name, *cfgBackground, c, prometheus.DefaultRegisterer) } diff --git a/modules/ingester/ingester_test.go b/modules/ingester/ingester_test.go index 58e253ab1e2..2e030e35dbd 100644 --- a/modules/ingester/ingester_test.go +++ b/modules/ingester/ingester_test.go @@ -428,7 +428,7 @@ func defaultIngesterWithOverrides(t testing.TB, tmpDir string, o overrides.Confi Filepath: tmpDir, }, }, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err, "unexpected error store") ingester, err := New(ingesterConfig, s, limits, prometheus.NewPedanticRegistry()) diff --git a/modules/ingester/local_block.go b/modules/ingester/local_block.go index 04dee83b346..d45e5d7d2ea 100644 --- a/modules/ingester/local_block.go +++ b/modules/ingester/local_block.go @@ -37,7 +37,7 @@ func newLocalBlock(ctx context.Context, existingBlock common.BackendBlock, l *lo writer: backend.NewWriter(l), } - flushedBytes, err := c.reader.Read(ctx, nameFlushed, c.BlockMeta().BlockID, c.BlockMeta().TenantID, false) + flushedBytes, err := c.reader.Read(ctx, nameFlushed, c.BlockMeta().BlockID, c.BlockMeta().TenantID, nil) if err == nil { flushedTime := time.Time{} err = flushedTime.UnmarshalText(flushedBytes) @@ -73,7 +73,7 @@ func (c *localBlock) SetFlushed(ctx context.Context) error { return fmt.Errorf("error marshalling flush time to text: %w", err) } - err = c.writer.Write(ctx, nameFlushed, c.BlockMeta().BlockID, c.BlockMeta().TenantID, flushedBytes, false) + err = c.writer.Write(ctx, nameFlushed, c.BlockMeta().BlockID, c.BlockMeta().TenantID, flushedBytes, nil) if err != nil { return fmt.Errorf("error writing ingester block flushed file: %w", err) } diff --git a/modules/storage/store.go b/modules/storage/store.go index c6b5614cdf8..8bc97855a3a 100644 --- a/modules/storage/store.go +++ b/modules/storage/store.go @@ -6,6 +6,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/dskit/services" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/usagestats" "github.com/grafana/tempo/tempodb" ) @@ -39,7 +40,7 @@ type store struct { } // NewStore creates a new Tempo Store using configuration supplied. -func NewStore(cfg Config, logger log.Logger) (Store, error) { +func NewStore(cfg Config, cacheProvider cache.Provider, logger log.Logger) (Store, error) { statCache.Set(cfg.Trace.Cache) statBackend.Set(cfg.Trace.Backend) statWalEncoding.Set(cfg.Trace.WAL.Encoding.String()) @@ -47,7 +48,7 @@ func NewStore(cfg Config, logger log.Logger) (Store, error) { statBlockEncoding.Set(cfg.Trace.Block.Encoding.String()) statBlockSearchEncoding.Set(cfg.Trace.Block.SearchEncoding.String()) - r, w, c, err := tempodb.New(&cfg.Trace, logger) + r, w, c, err := tempodb.New(&cfg.Trace, cacheProvider, logger) if err != nil { return nil, err } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index ed239d8361b..6db9d3733f1 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -2,8 +2,29 @@ package cache import ( "context" + + "github.com/grafana/dskit/services" +) + +type Role string + +const ( + // individual roles + RoleBloom Role = "bloom" + RoleTraceIDIdx Role = "trace-id-index" + RoleParquetFooter Role = "parquet-footer" + RoleParquetColumnIdx Role = "parquet-column-idx" + RoleParquetOffsetIdx Role = "parquet-offset-idx" ) +// Provider is an object that can return a cache for a requested role +type Provider interface { + services.Service + + CacheFor(role Role) Cache + AddCache(role Role, c Cache) error +} + // Cache byte arrays by key. // // NB we intentionally do not return errors in this interface - caching is best diff --git a/pkg/usagestats/reporter.go b/pkg/usagestats/reporter.go index c6cd3bbff51..843b079b96d 100644 --- a/pkg/usagestats/reporter.go +++ b/pkg/usagestats/reporter.go @@ -200,7 +200,7 @@ func (rep *Reporter) fetchSeed(ctx context.Context, continueFn func(err error) b // readSeedFile reads the cluster seed file from the object store. func (rep *Reporter) readSeedFile(ctx context.Context) (*ClusterSeed, error) { - reader, _, err := rep.reader.Read(ctx, backend.ClusterSeedFileName, backend.KeyPath{}, false) + reader, _, err := rep.reader.Read(ctx, backend.ClusterSeedFileName, backend.KeyPath{}, nil) if err != nil { return nil, err } @@ -226,7 +226,7 @@ func (rep *Reporter) writeSeedFile(ctx context.Context, seed ClusterSeed) error if err != nil { return err } - return rep.writer.Write(ctx, backend.ClusterSeedFileName, []string{}, bytes.NewReader(data), -1, false) + return rep.writer.Write(ctx, backend.ClusterSeedFileName, []string{}, bytes.NewReader(data), -1, nil) } // running inits the reporter seed and start sending report for every interval diff --git a/pkg/usagestats/reporter_test.go b/pkg/usagestats/reporter_test.go index a8fe16c3604..d291c52c56f 100644 --- a/pkg/usagestats/reporter_test.go +++ b/pkg/usagestats/reporter_test.go @@ -81,7 +81,7 @@ func Test_LeaderElectionWithBrokenSeedFile(t *testing.T) { // Ensure that leader election succeeds even when the seed file has been // corrupted. This means that we don't need to extend the interface of the // backend in order to delete a corrupted seed file. - err = objectClient.Write(context.Background(), backend.ClusterSeedFileName, []string{}, bytes.NewReader([]byte("{")), -1, false) + err = objectClient.Write(context.Background(), backend.ClusterSeedFileName, []string{}, bytes.NewReader([]byte("{")), -1, nil) require.NoError(t, err) for i := 0; i < 3; i++ { diff --git a/tempodb/backend/azure/azure_test.go b/tempodb/backend/azure/azure_test.go index 55c9bdea8db..8e028de00af 100644 --- a/tempodb/backend/azure/azure_test.go +++ b/tempodb/backend/azure/azure_test.go @@ -90,12 +90,12 @@ func TestHedge(t *testing.T) { // the first call on each client initiates an extra http request // clearing that here - _, _, _ = r.Read(ctx, "object", backend.KeyPathForBlock(uuid.New(), "tenant"), false) + _, _, _ = r.Read(ctx, "object", backend.KeyPathForBlock(uuid.New(), "tenant"), nil) time.Sleep(tc.returnIn) atomic.StoreInt32(&count, 0) // calls that should hedge - _, _, _ = r.Read(ctx, "object", backend.KeyPathForBlock(uuid.New(), "tenant"), false) + _, _, _ = r.Read(ctx, "object", backend.KeyPathForBlock(uuid.New(), "tenant"), nil) time.Sleep(tc.returnIn) assert.Equal(t, tc.expectedHedgedRequests*2, atomic.LoadInt32(&count)) // *2 b/c reads execute a HEAD and GET atomic.StoreInt32(&count, 0) @@ -111,7 +111,7 @@ func TestHedge(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) - _ = w.Write(ctx, "object", backend.KeyPathForBlock(uuid.New(), "tenant"), bytes.NewReader(make([]byte, 10)), 10, false) + _ = w.Write(ctx, "object", backend.KeyPathForBlock(uuid.New(), "tenant"), bytes.NewReader(make([]byte, 10)), 10, nil) // Write consists of two operations: // - Put Block operation // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block @@ -130,7 +130,7 @@ func TestHedge(t *testing.T) { // blockSize := 2000000 // u, err := uuid.Parse("f97223f3-d60c-4923-b255-bb7b8140b389") // require.NoError(t, err) - // _ = w.Write(ctx, "object", backend.KeyPathForBlock(u, "tenant"), bytes.NewReader(make([]byte, blockSize)), 10, false) + // _ = w.Write(ctx, "object", backend.KeyPathForBlock(u, "tenant"), bytes.NewReader(make([]byte, blockSize)), 10, nil) } else { assert.Equal(t, int32(2), atomic.LoadInt32(&count)) } @@ -246,7 +246,7 @@ func TestObjectWithPrefix(t *testing.T) { require.NoError(t, err) ctx := context.Background() - err = w.Write(ctx, tc.objectName, tc.keyPath, bytes.NewReader([]byte{}), 0, false) + err = w.Write(ctx, tc.objectName, tc.keyPath, bytes.NewReader([]byte{}), 0, nil) assert.NoError(t, err) }) } diff --git a/tempodb/backend/azure/v1/v1.go b/tempodb/backend/azure/v1/v1.go index 79e99ac2b42..d06d0cf82f1 100644 --- a/tempodb/backend/azure/v1/v1.go +++ b/tempodb/backend/azure/v1/v1.go @@ -77,7 +77,7 @@ func New(cfg *config.Config, confirm bool) (*V1, error) { } // Write implements backend.Writer -func (rw *V1) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, _ int64, _ bool) error { +func (rw *V1) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, _ int64, _ *backend.CacheInfo) error { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "azure.Write") @@ -114,7 +114,7 @@ func (rw *V1) CloseAppend(context.Context, backend.AppendTracker) error { return nil } -func (rw *V1) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ bool) error { +func (rw *V1) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) error { blobURL, err := GetBlobURL(ctx, rw.cfg, backend.ObjectFileName(keypath, name)) if err != nil { return fmt.Errorf("cannot get Azure blob URL, name: %s: %w", backend.ObjectFileName(keypath, name), err) @@ -227,7 +227,7 @@ func (rw *V1) ListBlocks(ctx context.Context, tenant string) ([]uuid.UUID, []uui } // Read implements backend.Reader -func (rw *V1) Read(ctx context.Context, name string, keypath backend.KeyPath, _ bool) (io.ReadCloser, int64, error) { +func (rw *V1) Read(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) (io.ReadCloser, int64, error) { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "azure.Read") @@ -243,7 +243,7 @@ func (rw *V1) Read(ctx context.Context, name string, keypath backend.KeyPath, _ } // ReadRange implements backend.Reader -func (rw *V1) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ bool) error { +func (rw *V1) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ *backend.CacheInfo) error { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "azure.ReadRange", opentracing.Tags{ @@ -282,7 +282,7 @@ func (rw *V1) WriteVersioned(ctx context.Context, name string, keypath backend.K return "", backend.ErrVersionDoesNotMatch } - err = rw.Write(ctx, name, keypath, data, -1, false) + err = rw.Write(ctx, name, keypath, data, -1, nil) if err != nil { return "", err } @@ -301,7 +301,7 @@ func (rw *V1) DeleteVersioned(ctx context.Context, name string, keypath backend. return backend.ErrVersionDoesNotMatch } - return rw.Delete(ctx, name, keypath, false) + return rw.Delete(ctx, name, keypath, nil) } func (rw *V1) ReadVersioned(ctx context.Context, name string, keypath backend.KeyPath) (io.ReadCloser, backend.Version, error) { diff --git a/tempodb/backend/azure/v2/compactor.go b/tempodb/backend/azure/v2/compactor.go index 3c57b8c6171..c5b98f1eb3e 100644 --- a/tempodb/backend/azure/v2/compactor.go +++ b/tempodb/backend/azure/v2/compactor.go @@ -45,7 +45,7 @@ func (rw *V2) MarkBlockCompacted(blockID uuid.UUID, tenantID string) error { } // delete the old file - return rw.Delete(ctx, metaFilename, []string{}, false) + return rw.Delete(ctx, metaFilename, []string{}, nil) } func (rw *V2) ClearBlock(blockID uuid.UUID, tenantID string) error { @@ -78,7 +78,7 @@ func (rw *V2) ClearBlock(blockID uuid.UUID, tenantID string) error { return fmt.Errorf("unexpected empty blob name when listing %s: %w", prefix, err) } - err = rw.Delete(ctx, *b.Name, []string{}, false) + err = rw.Delete(ctx, *b.Name, []string{}, nil) if err != nil { warning = err continue diff --git a/tempodb/backend/azure/v2/v2.go b/tempodb/backend/azure/v2/v2.go index 8973a9a91a6..56f7126d60d 100644 --- a/tempodb/backend/azure/v2/v2.go +++ b/tempodb/backend/azure/v2/v2.go @@ -83,7 +83,7 @@ func New(cfg *config.Config, confirm bool) (*V2, error) { } // Write implements backend.Writer -func (rw *V2) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, _ int64, _ bool) error { +func (rw *V2) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, _ int64, _ *backend.CacheInfo) error { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "azure.Write") @@ -120,7 +120,7 @@ func (rw *V2) CloseAppend(context.Context, backend.AppendTracker) error { return nil } -func (rw *V2) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ bool) error { +func (rw *V2) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) error { blobClient, err := getBlobClient(ctx, rw.cfg, backend.ObjectFileName(keypath, name)) if err != nil { return fmt.Errorf("cannot get Azure blob client, name: %s: %w", backend.ObjectFileName(keypath, name), err) @@ -228,7 +228,7 @@ func (rw *V2) ListBlocks(ctx context.Context, tenant string) ([]uuid.UUID, []uui } // Read implements backend.Reader -func (rw *V2) Read(ctx context.Context, name string, keypath backend.KeyPath, _ bool) (io.ReadCloser, int64, error) { +func (rw *V2) Read(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) (io.ReadCloser, int64, error) { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "azure.Read") @@ -244,7 +244,7 @@ func (rw *V2) Read(ctx context.Context, name string, keypath backend.KeyPath, _ } // ReadRange implements backend.Reader -func (rw *V2) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ bool) error { +func (rw *V2) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ *backend.CacheInfo) error { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "azure.ReadRange", opentracing.Tags{ @@ -283,7 +283,7 @@ func (rw *V2) WriteVersioned(ctx context.Context, name string, keypath backend.K return "", backend.ErrVersionDoesNotMatch } - err = rw.Write(ctx, name, keypath, data, -1, false) + err = rw.Write(ctx, name, keypath, data, -1, nil) if err != nil { return "", err } @@ -302,7 +302,7 @@ func (rw *V2) DeleteVersioned(ctx context.Context, name string, keypath backend. return backend.ErrVersionDoesNotMatch } - return rw.Delete(ctx, name, keypath, false) + return rw.Delete(ctx, name, keypath, nil) } func (rw *V2) ReadVersioned(ctx context.Context, name string, keypath backend.KeyPath) (io.ReadCloser, backend.Version, error) { diff --git a/tempodb/backend/backend.go b/tempodb/backend/backend.go index 03e64bdc7d3..7b5c71d4c23 100644 --- a/tempodb/backend/backend.go +++ b/tempodb/backend/backend.go @@ -6,6 +6,7 @@ import ( "io" "github.com/google/uuid" + "github.com/grafana/tempo/pkg/cache" ) const ( @@ -24,13 +25,18 @@ var ( GlobalMaxBlockID = uuid.MustParse("ffffffff-ffff-ffff-ffff-ffffffffffff") ) +type CacheInfo struct { + Meta *BlockMeta + Role cache.Role +} + // AppendTracker is an empty interface usable by the backend to track a long running append operation type AppendTracker interface{} // Writer is a collection of methods to write data to tempodb backends type Writer interface { - // Write is for in memory data. shouldCache specifies whether or not caching should be attempted. - Write(ctx context.Context, name string, blockID uuid.UUID, tenantID string, buffer []byte, shouldCache bool) error + // Write is for in memory data. cacheInfo contains information to make a caching decision. + Write(ctx context.Context, name string, blockID uuid.UUID, tenantID string, buffer []byte, cacheInfo *CacheInfo) error // StreamWriter is for larger data payloads streamed through an io.Reader. It is expected this will _not_ be cached. StreamWriter(ctx context.Context, name string, blockID uuid.UUID, tenantID string, data io.Reader, size int64) error // WriteBlockMeta writes a block meta to its blocks @@ -45,14 +51,12 @@ type Writer interface { // Reader is a collection of methods to read data from tempodb backends type Reader interface { - // Read is for reading entire objects from the backend. There will be an attempt to retrieve this - // from cache if shouldCache is true. - Read(ctx context.Context, name string, blockID uuid.UUID, tenantID string, shouldCache bool) ([]byte, error) + // Read is for reading entire objects from the backend. cacheInfo contains information to make a caching decision + Read(ctx context.Context, name string, blockID uuid.UUID, tenantID string, cacheInfo *CacheInfo) ([]byte, error) // StreamReader is for streaming entire objects from the backend. It is expected this will _not_ be cached. StreamReader(ctx context.Context, name string, blockID uuid.UUID, tenantID string) (io.ReadCloser, int64, error) - // ReadRange is for reading parts of large objects from the backend. - // There will be an attempt to retrieve this from cache if shouldCache is true. Cache key will be tenantID:blockID:offset:bufferLength - ReadRange(ctx context.Context, name string, blockID uuid.UUID, tenantID string, offset uint64, buffer []byte, shouldCache bool) error + // ReadRange is for reading parts of large objects from the backend. cacheInfo contains information to make a caching decision + ReadRange(ctx context.Context, name string, blockID uuid.UUID, tenantID string, offset uint64, buffer []byte, cacheInfo *CacheInfo) error // Tenants returns a list of all tenants in a backend Tenants(ctx context.Context) ([]string, error) // Blocks returns the blockIDs, compactedBlockIDs and an error from the backend. diff --git a/tempodb/backend/cache/cache.go b/tempodb/backend/cache/cache.go index a9f3ebc4a54..2764d653e09 100644 --- a/tempodb/backend/cache/cache.go +++ b/tempodb/backend/cache/cache.go @@ -6,7 +6,10 @@ import ( "io" "strconv" "strings" + "time" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/google/uuid" "github.com/grafana/tempo/pkg/cache" @@ -14,19 +17,46 @@ import ( "github.com/grafana/tempo/tempodb/backend" ) +type BloomConfig struct { + CacheMinCompactionLevel uint8 `yaml:"cache_min_compaction_level"` + CacheMaxBlockAge time.Duration `yaml:"cache_max_block_age"` +} + type readerWriter struct { + cfgBloom *BloomConfig + nextReader backend.RawReader nextWriter backend.RawWriter - cache cache.Cache + + footerCache cache.Cache + bloomCache cache.Cache + columnIdxCache cache.Cache + offsetIdxCache cache.Cache + traceIDIdxCache cache.Cache } -func NewCache(nextReader backend.RawReader, nextWriter backend.RawWriter, cache cache.Cache) (backend.RawReader, backend.RawWriter, error) { +func NewCache(cfgBloom *BloomConfig, nextReader backend.RawReader, nextWriter backend.RawWriter, cacheProvider cache.Provider, logger log.Logger) (backend.RawReader, backend.RawWriter, error) { rw := &readerWriter{ - cache: cache, + cfgBloom: cfgBloom, + + footerCache: cacheProvider.CacheFor(cache.RoleParquetFooter), + bloomCache: cacheProvider.CacheFor(cache.RoleBloom), + offsetIdxCache: cacheProvider.CacheFor(cache.RoleParquetOffsetIdx), + columnIdxCache: cacheProvider.CacheFor(cache.RoleParquetColumnIdx), + traceIDIdxCache: cacheProvider.CacheFor(cache.RoleTraceIDIdx), + nextReader: nextReader, nextWriter: nextWriter, } + level.Info(logger).Log("msg", "caches available to storage backend", + "footer", rw.footerCache != nil, + "bloom", rw.bloomCache != nil, + "offset_idx", rw.offsetIdxCache != nil, + "column_idx", rw.columnIdxCache != nil, + "trace_id_idx", rw.traceIDIdxCache != nil, + ) + return rw, rw, nil } @@ -40,47 +70,53 @@ func (r *readerWriter) ListBlocks(ctx context.Context, tenant string) (blockIDs } // Read implements backend.RawReader -func (r *readerWriter) Read(ctx context.Context, name string, keypath backend.KeyPath, shouldCache bool) (io.ReadCloser, int64, error) { +func (r *readerWriter) Read(ctx context.Context, name string, keypath backend.KeyPath, cacheInfo *backend.CacheInfo) (io.ReadCloser, int64, error) { var k string - if shouldCache { + cache := r.cacheFor(cacheInfo) + if cache != nil { k = key(keypath, name) - found, vals, _ := r.cache.Fetch(ctx, []string{k}) + found, vals, _ := cache.Fetch(ctx, []string{k}) if len(found) > 0 { return io.NopCloser(bytes.NewReader(vals[0])), int64(len(vals[0])), nil } } - object, size, err := r.nextReader.Read(ctx, name, keypath, false) + // previous implemenation always passed false forward for "shouldCache" so we are matching that behavior by passing nil for cacheInfo + // todo: reevaluate. should we pass the cacheInfo forward? + object, size, err := r.nextReader.Read(ctx, name, keypath, nil) if err != nil { return nil, 0, err } defer object.Close() b, err := tempo_io.ReadAllWithEstimate(object, size) - if err == nil && shouldCache { - r.cache.Store(ctx, []string{k}, [][]byte{b}) + if err == nil && cache != nil { + cache.Store(ctx, []string{k}, [][]byte{b}) } return io.NopCloser(bytes.NewReader(b)), size, err } // ReadRange implements backend.RawReader -func (r *readerWriter) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, shouldCache bool) error { +func (r *readerWriter) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, cacheInfo *backend.CacheInfo) error { var k string - if shouldCache { + cache := r.cacheFor(cacheInfo) + if cache != nil { // cache key is tenantID:blockID:offset:length - file name is not needed in key keyGen := keypath keyGen = append(keyGen, strconv.Itoa(int(offset)), strconv.Itoa(len(buffer))) k = strings.Join(keyGen, ":") - found, vals, _ := r.cache.Fetch(ctx, []string{k}) + found, vals, _ := cache.Fetch(ctx, []string{k}) if len(found) > 0 { copy(buffer, vals[0]) } } - err := r.nextReader.ReadRange(ctx, name, keypath, offset, buffer, false) - if err == nil && shouldCache { - r.cache.Store(ctx, []string{k}, [][]byte{buffer}) + // previous implemenation always passed false forward for "shouldCache" so we are matching that behavior by passing nil for cacheInfo + // todo: reevaluate. should we pass the cacheInfo forward? + err := r.nextReader.ReadRange(ctx, name, keypath, offset, buffer, nil) + if err == nil && cache != nil { + cache.Store(ctx, []string{k}, [][]byte{buffer}) } return err @@ -89,20 +125,34 @@ func (r *readerWriter) ReadRange(ctx context.Context, name string, keypath backe // Shutdown implements backend.RawReader func (r *readerWriter) Shutdown() { r.nextReader.Shutdown() - r.cache.Stop() + + stopCache := func(c cache.Cache) { + if c != nil { + c.Stop() + } + } + + stopCache(r.footerCache) + stopCache(r.bloomCache) + stopCache(r.offsetIdxCache) + stopCache(r.columnIdxCache) + stopCache(r.traceIDIdxCache) } // Write implements backend.Writer -func (r *readerWriter) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, size int64, shouldCache bool) error { +func (r *readerWriter) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, size int64, cacheInfo *backend.CacheInfo) error { b, err := tempo_io.ReadAllWithEstimate(data, size) if err != nil { return err } - if shouldCache { - r.cache.Store(ctx, []string{key(keypath, name)}, [][]byte{b}) + if cache := r.cacheFor(cacheInfo); cache != nil { + cache.Store(ctx, []string{key(keypath, name)}, [][]byte{b}) } - return r.nextWriter.Write(ctx, name, keypath, bytes.NewReader(b), int64(len(b)), false) + + // previous implemenation always passed false forward for "shouldCache" so we are matching that behavior by passing nil for cacheInfo + // todo: reevaluate. should we pass the cacheInfo forward? + return r.nextWriter.Write(ctx, name, keypath, bytes.NewReader(b), int64(len(b)), nil) } // Append implements backend.Writer @@ -115,13 +165,55 @@ func (r *readerWriter) CloseAppend(ctx context.Context, tracker backend.AppendTr return r.nextWriter.CloseAppend(ctx, tracker) } -func (r *readerWriter) Delete(ctx context.Context, name string, keypath backend.KeyPath, shouldCache bool) error { - if shouldCache { +func (r *readerWriter) Delete(ctx context.Context, name string, keypath backend.KeyPath, cacheInfo *backend.CacheInfo) error { + if cacheInfo != nil { panic("delete is not supported for cache.Cache backend") } - return r.nextWriter.Delete(ctx, name, keypath, shouldCache) + return r.nextWriter.Delete(ctx, name, keypath, nil) } func key(keypath backend.KeyPath, name string) string { return strings.Join(keypath, ":") + ":" + name } + +// cacheFor evaluates the cacheInfo and returns the appropriate cache. +func (r *readerWriter) cacheFor(cacheInfo *backend.CacheInfo) cache.Cache { + if cacheInfo == nil { + return nil + } + + switch cacheInfo.Role { + case cache.RoleParquetFooter: + return r.footerCache + case cache.RoleParquetColumnIdx: + return r.columnIdxCache + case cache.RoleParquetOffsetIdx: + return r.offsetIdxCache + case cache.RoleTraceIDIdx: + return r.traceIDIdxCache + case cache.RoleBloom: + // if there is no bloom cfg then there are no restrictions on bloom filter caching + if r.cfgBloom == nil { + return r.bloomCache + } + + if cacheInfo.Meta == nil { + return nil + } + + // compaction level is _atleast_ CacheMinCompactionLevel + if r.cfgBloom.CacheMinCompactionLevel > 0 && cacheInfo.Meta.CompactionLevel > r.cfgBloom.CacheMinCompactionLevel { + return nil + } + + curTime := time.Now() + // block is not older than CacheMaxBlockAge + if r.cfgBloom.CacheMaxBlockAge > 0 && curTime.Sub(cacheInfo.Meta.StartTime) > r.cfgBloom.CacheMaxBlockAge { + return nil + } + + return r.bloomCache + } + + return nil +} diff --git a/tempodb/backend/cache/cache_test.go b/tempodb/backend/cache/cache_test.go index 8fa52e34d85..79aaa5a40bd 100644 --- a/tempodb/backend/cache/cache_test.go +++ b/tempodb/backend/cache/cache_test.go @@ -5,11 +5,15 @@ import ( "context" "io" "testing" + "time" + "github.com/go-kit/log" "github.com/google/uuid" + "github.com/grafana/dskit/services" "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/tempodb/backend" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type mockClient struct { @@ -41,6 +45,166 @@ func NewMockClient() cache.Cache { } } +func newMockProvider() *mockProvider { + return &mockProvider{ + c: NewMockClient(), + } +} + +type mockProvider struct { + services.Service + + c cache.Cache +} + +func (p *mockProvider) CacheFor(_ cache.Role) cache.Cache { + return p.c +} + +func (p *mockProvider) AddCache(_ cache.Role, _ cache.Cache) error { + return nil +} + +func TestCacheFor(t *testing.T) { + reader, _, err := NewCache(&BloomConfig{ + CacheMaxBlockAge: time.Hour, + CacheMinCompactionLevel: 1, + }, nil, nil, newMockProvider(), log.NewNopLogger()) + require.NoError(t, err) + + rw := reader.(*readerWriter) + + testCases := []struct { + name string + cacheInfo *backend.CacheInfo + expectedCache cache.Cache + }{ + // first three caches are unconditionally returned + { + name: "footer is always returned", + cacheInfo: &backend.CacheInfo{Role: cache.RoleParquetFooter}, + expectedCache: rw.footerCache, + }, + { + name: "col idx is always returned", + cacheInfo: &backend.CacheInfo{Role: cache.RoleParquetColumnIdx}, + expectedCache: rw.columnIdxCache, + }, + { + name: "offset idx is always returned", + cacheInfo: &backend.CacheInfo{Role: cache.RoleParquetOffsetIdx}, + expectedCache: rw.offsetIdxCache, + }, + { + name: "trace id idx is always returned", + cacheInfo: &backend.CacheInfo{Role: cache.RoleTraceIDIdx}, + expectedCache: rw.traceIDIdxCache, + }, + // bloom cache is returned if the meta is valid given the bloom config + { + name: "bloom - no meta means no cache", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + }, + }, + { + name: "bloom - compaction lvl and start time valid", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: &backend.BlockMeta{CompactionLevel: 1, StartTime: time.Now()}, + }, + expectedCache: rw.bloomCache, + }, + { + name: "bloom - start time invalid", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: &backend.BlockMeta{CompactionLevel: 1, StartTime: time.Now().Add(-2 * time.Hour)}, + }, + }, + { + name: "bloom - compaction lvl invalid", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: &backend.BlockMeta{CompactionLevel: 2, StartTime: time.Now()}, + }, + }, + { + name: "bloom - both invalid", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: &backend.BlockMeta{CompactionLevel: 2, StartTime: time.Now().Add(-2 * time.Hour)}, + }, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expectedCache, rw.cacheFor(tt.cacheInfo)) + }) + } +} + +func TestCacheForReturnsBloomWithNoConfig(t *testing.T) { + reader, _, err := NewCache(nil, nil, nil, newMockProvider(), log.NewNopLogger()) + require.NoError(t, err) + + rw := reader.(*readerWriter) + + testCases := []struct { + name string + cacheInfo *backend.CacheInfo + expectedCache cache.Cache + }{ + // bloom cache is returned if the meta is valid given the bloom config + { + name: "bloom - no meta means no cache", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + }, + expectedCache: rw.bloomCache, + }, + { + name: "bloom - compaction lvl and start time valid", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: &backend.BlockMeta{CompactionLevel: 1, StartTime: time.Now()}, + }, + expectedCache: rw.bloomCache, + }, + { + name: "bloom - start time invalid", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: &backend.BlockMeta{CompactionLevel: 1, StartTime: time.Now().Add(-2 * time.Hour)}, + }, + expectedCache: rw.bloomCache, + }, + { + name: "bloom - compaction lvl invalid", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: &backend.BlockMeta{CompactionLevel: 2, StartTime: time.Now()}, + }, + expectedCache: rw.bloomCache, + }, + { + name: "bloom - both invalid", + cacheInfo: &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: &backend.BlockMeta{CompactionLevel: 2, StartTime: time.Now().Add(-2 * time.Hour)}, + }, + expectedCache: rw.bloomCache, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expectedCache, rw.cacheFor(tt.cacheInfo)) + }) + } +} + func TestReadWrite(t *testing.T) { tenantID := "test" blockID := uuid.New() @@ -49,7 +213,7 @@ func TestReadWrite(t *testing.T) { name string readerRead []byte readerName string - shouldCache bool + cacheInfo *backend.CacheInfo expectedRead []byte expectedCache []byte }{ @@ -57,14 +221,14 @@ func TestReadWrite(t *testing.T) { name: "should cache", readerName: "foo", readerRead: []byte{0x02}, - shouldCache: true, + cacheInfo: &backend.CacheInfo{Role: cache.RoleParquetFooter}, expectedRead: []byte{0x02}, expectedCache: []byte{0x02}, }, { name: "should not cache", readerName: "bar", - shouldCache: false, + cacheInfo: &backend.CacheInfo{Role: cache.Role("foo")}, // fake role name will not find a matching cache readerRead: []byte{0x02}, expectedRead: []byte{0x02}, }, @@ -78,24 +242,27 @@ func TestReadWrite(t *testing.T) { mockW := &backend.MockRawWriter{} // READ - r, _, _ := NewCache(mockR, mockW, NewMockClient()) + r, _, err := NewCache(nil, mockR, mockW, newMockProvider(), log.NewNopLogger()) + require.NoError(t, err) ctx := context.Background() - reader, _, _ := r.Read(ctx, tt.readerName, backend.KeyPathForBlock(blockID, tenantID), tt.shouldCache) + reader, _, _ := r.Read(ctx, tt.readerName, backend.KeyPathForBlock(blockID, tenantID), tt.cacheInfo) read, _ := io.ReadAll(reader) assert.Equal(t, tt.expectedRead, read) // clear reader and re-request mockR.R = nil - reader, _, _ = r.Read(ctx, tt.readerName, backend.KeyPathForBlock(blockID, tenantID), tt.shouldCache) + reader, _, _ = r.Read(ctx, tt.readerName, backend.KeyPathForBlock(blockID, tenantID), tt.cacheInfo) read, _ = io.ReadAll(reader) assert.Equal(t, len(tt.expectedCache), len(read)) // WRITE - _, w, _ := NewCache(mockR, mockW, NewMockClient()) - _ = w.Write(ctx, tt.readerName, backend.KeyPathForBlock(blockID, tenantID), bytes.NewReader(tt.readerRead), int64(len(tt.readerRead)), tt.shouldCache) - reader, _, _ = r.Read(ctx, tt.readerName, backend.KeyPathForBlock(blockID, tenantID), tt.shouldCache) + _, w, err := NewCache(nil, mockR, mockW, newMockProvider(), log.NewNopLogger()) + require.NoError(t, err) + + _ = w.Write(ctx, tt.readerName, backend.KeyPathForBlock(blockID, tenantID), bytes.NewReader(tt.readerRead), int64(len(tt.readerRead)), tt.cacheInfo) + reader, _, _ = r.Read(ctx, tt.readerName, backend.KeyPathForBlock(blockID, tenantID), tt.cacheInfo) read, _ = io.ReadAll(reader) assert.Equal(t, len(tt.expectedCache), len(read)) }) @@ -127,7 +294,7 @@ func TestList(t *testing.T) { } mockW := &backend.MockRawWriter{} - rw, _, _ := NewCache(mockR, mockW, NewMockClient()) + rw, _, _ := NewCache(nil, mockR, mockW, newMockProvider(), log.NewNopLogger()) ctx := context.Background() list, _ := rw.List(ctx, backend.KeyPathForBlock(blockID, tenantID)) diff --git a/tempodb/backend/gcs/gcs.go b/tempodb/backend/gcs/gcs.go index 2a5509e95af..c864daae3be 100644 --- a/tempodb/backend/gcs/gcs.go +++ b/tempodb/backend/gcs/gcs.go @@ -104,7 +104,7 @@ func internalNew(cfg *Config, confirm bool) (*readerWriter, error) { } // Write implements backend.Writer -func (rw *readerWriter) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, _ int64, _ bool) error { +func (rw *readerWriter) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, _ int64, _ *backend.CacheInfo) error { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "gcs.Write") defer span.Finish() @@ -156,7 +156,7 @@ func (rw *readerWriter) CloseAppend(_ context.Context, tracker backend.AppendTra return w.Close() } -func (rw *readerWriter) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ bool) error { +func (rw *readerWriter) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) error { return readError(rw.bucket.Object(backend.ObjectFileName(keypath, name)).Delete(ctx)) } @@ -309,7 +309,7 @@ func (rw *readerWriter) ListBlocks(ctx context.Context, tenant string) ([]uuid.U } // Read implements backend.Reader -func (rw *readerWriter) Read(ctx context.Context, name string, keypath backend.KeyPath, _ bool) (io.ReadCloser, int64, error) { +func (rw *readerWriter) Read(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) (io.ReadCloser, int64, error) { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "gcs.Read") defer span.Finish() @@ -324,7 +324,7 @@ func (rw *readerWriter) Read(ctx context.Context, name string, keypath backend.K } // ReadRange implements backend.Reader -func (rw *readerWriter) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ bool) error { +func (rw *readerWriter) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ *backend.CacheInfo) error { keypath = backend.KeyPathWithPrefix(keypath, rw.cfg.Prefix) span, derivedCtx := opentracing.StartSpanFromContext(ctx, "gcs.ReadRange", opentracing.Tags{ "len": len(buffer), diff --git a/tempodb/backend/gcs/gcs_test.go b/tempodb/backend/gcs/gcs_test.go index 1ab270dca9f..96face1846a 100644 --- a/tempodb/backend/gcs/gcs_test.go +++ b/tempodb/backend/gcs/gcs_test.go @@ -67,17 +67,17 @@ func TestHedge(t *testing.T) { // the first call on each client initiates an extra http request // clearing that here - _, _, _ = r.Read(ctx, "object", []string{"test"}, false) + _, _, _ = r.Read(ctx, "object", []string{"test"}, nil) time.Sleep(tc.returnIn) atomic.StoreInt32(&count, 0) // calls that should hedge - _, _, _ = r.Read(ctx, "object", []string{"test"}, false) + _, _, _ = r.Read(ctx, "object", []string{"test"}, nil) time.Sleep(tc.returnIn) assert.Equal(t, tc.expectedHedgedRequests, atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) - _ = r.ReadRange(ctx, "object", []string{"test"}, 10, []byte{}, false) + _ = r.ReadRange(ctx, "object", []string{"test"}, 10, []byte{}, nil) time.Sleep(tc.returnIn) assert.Equal(t, tc.expectedHedgedRequests, atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) @@ -87,7 +87,7 @@ func TestHedge(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) - _ = w.Write(ctx, "object", []string{"test"}, bytes.NewReader([]byte{}), 0, false) + _ = w.Write(ctx, "object", []string{"test"}, bytes.NewReader([]byte{}), 0, nil) assert.Equal(t, int32(1), atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) }) @@ -139,7 +139,7 @@ func TestObjectConfigAttributes(t *testing.T) { ctx := context.Background() - _ = w.Write(ctx, "object", []string{"test"}, bytes.NewReader([]byte{}), 0, false) + _ = w.Write(ctx, "object", []string{"test"}, bytes.NewReader([]byte{}), 0, nil) assert.Equal(t, tc.expectedObject, rawObject) }) } @@ -293,7 +293,7 @@ func TestObjectWithPrefix(t *testing.T) { require.NoError(t, err) ctx := context.Background() - err = w.Write(ctx, tc.objectName, tc.keyPath, bytes.NewReader([]byte{}), 0, false) + err = w.Write(ctx, tc.objectName, tc.keyPath, bytes.NewReader([]byte{}), 0, nil) assert.NoError(t, err) }) } diff --git a/tempodb/backend/local/local.go b/tempodb/backend/local/local.go index 4daef4a086b..351bdf8eb4e 100644 --- a/tempodb/backend/local/local.go +++ b/tempodb/backend/local/local.go @@ -43,7 +43,7 @@ func New(cfg *Config) (backend.RawReader, backend.RawWriter, backend.Compactor, } // Write implements backend.Writer -func (rw *Backend) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, _ int64, _ bool) error { +func (rw *Backend) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, _ int64, _ *backend.CacheInfo) error { if err := ctx.Err(); err != nil { return err } @@ -118,7 +118,7 @@ func (rw *Backend) CloseAppend(ctx context.Context, tracker backend.AppendTracke return dst.Close() } -func (rw *Backend) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ bool) error { +func (rw *Backend) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) error { if err := ctx.Err(); err != nil { return err } @@ -190,7 +190,7 @@ func (rw *Backend) ListBlocks(_ context.Context, tenant string) (metas []uuid.UU } // Read implements backend.Reader -func (rw *Backend) Read(ctx context.Context, name string, keypath backend.KeyPath, _ bool) (io.ReadCloser, int64, error) { +func (rw *Backend) Read(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) (io.ReadCloser, int64, error) { if err := ctx.Err(); err != nil { return nil, -1, err } @@ -212,7 +212,7 @@ func (rw *Backend) Read(ctx context.Context, name string, keypath backend.KeyPat } // ReadRange implements backend.Reader -func (rw *Backend) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ bool) error { +func (rw *Backend) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ *backend.CacheInfo) error { if err := ctx.Err(); err != nil { return err } diff --git a/tempodb/backend/local/local_test.go b/tempodb/backend/local/local_test.go index 4e623b08160..31c3fe59501 100644 --- a/tempodb/backend/local/local_test.go +++ b/tempodb/backend/local/local_test.go @@ -49,23 +49,23 @@ func TestReadWrite(t *testing.T) { ctx := context.Background() for _, id := range tenantIDs { fakeMeta.TenantID = id - err = w.Write(ctx, objectName, backend.KeyPathForBlock(fakeMeta.BlockID, id), bytes.NewReader(fakeObject), int64(len(fakeObject)), false) + err = w.Write(ctx, objectName, backend.KeyPathForBlock(fakeMeta.BlockID, id), bytes.NewReader(fakeObject), int64(len(fakeObject)), nil) assert.NoError(t, err, "unexpected error writing") - err = w.Write(ctx, backend.MetaName, backend.KeyPathForBlock(fakeMeta.BlockID, id), bytes.NewReader(fakeObject), int64(len(fakeObject)), false) + err = w.Write(ctx, backend.MetaName, backend.KeyPathForBlock(fakeMeta.BlockID, id), bytes.NewReader(fakeObject), int64(len(fakeObject)), nil) assert.NoError(t, err, "unexpected error meta.json") - err = w.Write(ctx, backend.CompactedMetaName, backend.KeyPathForBlock(fakeMeta.BlockID, id), bytes.NewReader(fakeObject), int64(len(fakeObject)), false) + err = w.Write(ctx, backend.CompactedMetaName, backend.KeyPathForBlock(fakeMeta.BlockID, id), bytes.NewReader(fakeObject), int64(len(fakeObject)), nil) assert.NoError(t, err, "unexpected error meta.compacted.json") } - actualObject, size, err := r.Read(ctx, objectName, backend.KeyPathForBlock(blockID, tenantIDs[0]), false) + actualObject, size, err := r.Read(ctx, objectName, backend.KeyPathForBlock(blockID, tenantIDs[0]), nil) assert.NoError(t, err, "unexpected error reading") actualObjectBytes, err := io.ReadAllWithEstimate(actualObject, size) assert.NoError(t, err, "unexpected error reading") assert.Equal(t, fakeObject, actualObjectBytes) actualReadRange := make([]byte, 5) - err = r.ReadRange(ctx, objectName, backend.KeyPathForBlock(blockID, tenantIDs[0]), 5, actualReadRange, false) + err = r.ReadRange(ctx, objectName, backend.KeyPathForBlock(blockID, tenantIDs[0]), 5, actualReadRange, nil) assert.NoError(t, err, "unexpected error range") assert.Equal(t, fakeObject[5:10], actualReadRange) @@ -92,7 +92,7 @@ func TestShutdownLeavesTenantsWithBlocks(t *testing.T) { tenant := "fake" // write a "block" - err = w.Write(ctx, "test", backend.KeyPathForBlock(blockID, tenant), contents, contents.Size(), false) + err = w.Write(ctx, "test", backend.KeyPathForBlock(blockID, tenant), contents, contents.Size(), nil) require.NoError(t, err) tenantExists(t, tenant, r) @@ -117,7 +117,7 @@ func TestShutdownRemovesTenantsWithoutBlocks(t *testing.T) { tenant := "tenant" // write a "block" - err = w.Write(ctx, "test", backend.KeyPathForBlock(blockID, tenant), contents, contents.Size(), false) + err = w.Write(ctx, "test", backend.KeyPathForBlock(blockID, tenant), contents, contents.Size(), nil) require.NoError(t, err) tenantExists(t, tenant, r) diff --git a/tempodb/backend/mocks.go b/tempodb/backend/mocks.go index f539606e354..d4773100fbb 100644 --- a/tempodb/backend/mocks.go +++ b/tempodb/backend/mocks.go @@ -27,7 +27,7 @@ type MockRawReader struct { ListBlocksFn func(ctx context.Context, tenant string) ([]uuid.UUID, []uuid.UUID, error) R []byte // read Range []byte // ReadRange - ReadFn func(ctx context.Context, name string, keypath KeyPath, shouldCache bool) (io.ReadCloser, int64, error) + ReadFn func(ctx context.Context, name string, keypath KeyPath, cacheInfo *CacheInfo) (io.ReadCloser, int64, error) BlockIDs []uuid.UUID CompactedBlockIDs []uuid.UUID @@ -50,15 +50,15 @@ func (m *MockRawReader) ListBlocks(ctx context.Context, tenant string) ([]uuid.U return m.BlockIDs, m.CompactedBlockIDs, nil } -func (m *MockRawReader) Read(ctx context.Context, name string, keypath KeyPath, shouldCache bool) (io.ReadCloser, int64, error) { +func (m *MockRawReader) Read(ctx context.Context, name string, keypath KeyPath, cacheInfo *CacheInfo) (io.ReadCloser, int64, error) { if m.ReadFn != nil { - return m.ReadFn(ctx, name, keypath, shouldCache) + return m.ReadFn(ctx, name, keypath, cacheInfo) } return io.NopCloser(bytes.NewReader(m.R)), int64(len(m.R)), nil } -func (m *MockRawReader) ReadRange(_ context.Context, _ string, _ KeyPath, _ uint64, buffer []byte, _ bool) error { +func (m *MockRawReader) ReadRange(_ context.Context, _ string, _ KeyPath, _ uint64, buffer []byte, _ *CacheInfo) error { copy(buffer, m.Range) return nil @@ -75,7 +75,7 @@ type MockRawWriter struct { err error } -func (m *MockRawWriter) Write(_ context.Context, _ string, _ KeyPath, data io.Reader, size int64, _ bool) error { +func (m *MockRawWriter) Write(_ context.Context, _ string, _ KeyPath, data io.Reader, size int64, _ *CacheInfo) error { var err error m.writeBuffer, err = tempo_io.ReadAllWithEstimate(data, size) return err @@ -91,7 +91,7 @@ func (m *MockRawWriter) CloseAppend(context.Context, AppendTracker) error { return nil } -func (m *MockRawWriter) Delete(_ context.Context, name string, keypath KeyPath, _ bool) error { +func (m *MockRawWriter) Delete(_ context.Context, name string, keypath KeyPath, _ *CacheInfo) error { if m.deleteCalls == nil { m.deleteCalls = make(map[string]map[string]int) } @@ -187,7 +187,7 @@ func (m *MockReader) BlockMeta(ctx context.Context, blockID uuid.UUID, tenantID return m.M, nil } -func (m *MockReader) Read(_ context.Context, name string, blockID uuid.UUID, tenantID string, _ bool) ([]byte, error) { +func (m *MockReader) Read(_ context.Context, name string, blockID uuid.UUID, tenantID string, _ *CacheInfo) ([]byte, error) { if m.ReadFn != nil { return m.ReadFn(name, blockID, tenantID) } @@ -199,7 +199,7 @@ func (m *MockReader) StreamReader(context.Context, string, uuid.UUID, string) (i panic("StreamReader is not yet supported for mock reader") } -func (m *MockReader) ReadRange(_ context.Context, _ string, _ uuid.UUID, _ string, _ uint64, buffer []byte, _ bool) error { +func (m *MockReader) ReadRange(_ context.Context, _ string, _ uuid.UUID, _ string, _ uint64, buffer []byte, _ *CacheInfo) error { copy(buffer, m.Range) return nil @@ -222,7 +222,7 @@ type MockWriter struct { IndexCompactedMeta map[string][]*CompactedBlockMeta } -func (m *MockWriter) Write(context.Context, string, uuid.UUID, string, []byte, bool) error { +func (m *MockWriter) Write(context.Context, string, uuid.UUID, string, []byte, *CacheInfo) error { return nil } diff --git a/tempodb/backend/raw.go b/tempodb/backend/raw.go index 6b62a12a260..d4d332c27e2 100644 --- a/tempodb/backend/raw.go +++ b/tempodb/backend/raw.go @@ -30,13 +30,13 @@ type Feature int // RawWriter is a collection of methods to write data to tempodb backends type RawWriter interface { // Write is for in memory data. shouldCache specifies whether or not caching should be attempted. - Write(ctx context.Context, name string, keypath KeyPath, data io.Reader, size int64, shouldCache bool) error + Write(ctx context.Context, name string, keypath KeyPath, data io.Reader, size int64, cacheInfo *CacheInfo) error // Append starts or continues an Append job. Pass nil to AppendTracker to start a job. Append(ctx context.Context, name string, keypath KeyPath, tracker AppendTracker, buffer []byte) (AppendTracker, error) // CloseAppend closes any resources associated with the AppendTracker. CloseAppend(ctx context.Context, tracker AppendTracker) error // Delete deletes a file. - Delete(ctx context.Context, name string, keypath KeyPath, shouldCache bool) error + Delete(ctx context.Context, name string, keypath KeyPath, cacheInfo *CacheInfo) error } // RawReader is a collection of methods to read data from tempodb backends @@ -46,10 +46,10 @@ type RawReader interface { // ListBlocks returns all blockIDs and compactedBlockIDs for a tenant. ListBlocks(ctx context.Context, tenant string) (blockIDs []uuid.UUID, compactedBlockIDs []uuid.UUID, err error) // Read is for streaming entire objects from the backend. There will be an attempt to retrieve this from cache if shouldCache is true. - Read(ctx context.Context, name string, keyPath KeyPath, shouldCache bool) (io.ReadCloser, int64, error) + Read(ctx context.Context, name string, keyPath KeyPath, cacheInfo *CacheInfo) (io.ReadCloser, int64, error) // ReadRange is for reading parts of large objects from the backend. // There will be an attempt to retrieve this from cache if shouldCache is true. Cache key will be tenantID:blockID:offset:bufferLength - ReadRange(ctx context.Context, name string, keypath KeyPath, offset uint64, buffer []byte, shouldCache bool) error + ReadRange(ctx context.Context, name string, keypath KeyPath, offset uint64, buffer []byte, cacheInfo *CacheInfo) error // Shutdown must be called when the Reader is finished and cleans up any associated resources. Shutdown() } @@ -72,13 +72,13 @@ func NewWriter(w RawWriter) Writer { // ) // Write implements backend.Writer -func (w *writer) Write(ctx context.Context, name string, blockID uuid.UUID, tenantID string, buffer []byte, shouldCache bool) error { - return w.w.Write(ctx, name, KeyPathForBlock(blockID, tenantID), bytes.NewReader(buffer), int64(len(buffer)), shouldCache) +func (w *writer) Write(ctx context.Context, name string, blockID uuid.UUID, tenantID string, buffer []byte, cacheInfo *CacheInfo) error { + return w.w.Write(ctx, name, KeyPathForBlock(blockID, tenantID), bytes.NewReader(buffer), int64(len(buffer)), cacheInfo) } // Write implements backend.Writer func (w *writer) StreamWriter(ctx context.Context, name string, blockID uuid.UUID, tenantID string, data io.Reader, size int64) error { - return w.w.Write(ctx, name, KeyPathForBlock(blockID, tenantID), data, size, false) + return w.w.Write(ctx, name, KeyPathForBlock(blockID, tenantID), data, size, nil) } // Write implements backend.Writer @@ -91,7 +91,7 @@ func (w *writer) WriteBlockMeta(ctx context.Context, meta *BlockMeta) error { return err } - return w.w.Write(ctx, MetaName, KeyPathForBlock(blockID, tenantID), bytes.NewReader(bMeta), int64(len(bMeta)), false) + return w.w.Write(ctx, MetaName, KeyPathForBlock(blockID, tenantID), bytes.NewReader(bMeta), int64(len(bMeta)), nil) } // Write implements backend.Writer @@ -109,7 +109,7 @@ func (w *writer) WriteTenantIndex(ctx context.Context, tenantID string, meta []* // If meta and compactedMeta are empty, call delete the tenant index. if len(meta) == 0 && len(compactedMeta) == 0 { // Skip returning an error when the object is already deleted. - err := w.w.Delete(ctx, TenantIndexName, []string{tenantID}, false) + err := w.w.Delete(ctx, TenantIndexName, []string{tenantID}, nil) if err != nil && !errors.Is(err, ErrDoesNotExist) { return err } @@ -123,7 +123,7 @@ func (w *writer) WriteTenantIndex(ctx context.Context, tenantID string, meta []* return err } - err = w.w.Write(ctx, TenantIndexName, KeyPath([]string{tenantID}), bytes.NewReader(indexBytes), int64(len(indexBytes)), false) + err = w.w.Write(ctx, TenantIndexName, KeyPath([]string{tenantID}), bytes.NewReader(indexBytes), int64(len(indexBytes)), nil) if err != nil { return err } @@ -143,8 +143,8 @@ func NewReader(r RawReader) Reader { } // Read implements backend.Reader -func (r *reader) Read(ctx context.Context, name string, blockID uuid.UUID, tenantID string, shouldCache bool) ([]byte, error) { - objReader, size, err := r.r.Read(ctx, name, KeyPathForBlock(blockID, tenantID), shouldCache) +func (r *reader) Read(ctx context.Context, name string, blockID uuid.UUID, tenantID string, cacheInfo *CacheInfo) ([]byte, error) { + objReader, size, err := r.r.Read(ctx, name, KeyPathForBlock(blockID, tenantID), cacheInfo) if err != nil { return nil, err } @@ -154,12 +154,12 @@ func (r *reader) Read(ctx context.Context, name string, blockID uuid.UUID, tenan // StreamReader implements backend.Reader func (r *reader) StreamReader(ctx context.Context, name string, blockID uuid.UUID, tenantID string) (io.ReadCloser, int64, error) { - return r.r.Read(ctx, name, KeyPathForBlock(blockID, tenantID), false) + return r.r.Read(ctx, name, KeyPathForBlock(blockID, tenantID), nil) } // ReadRange implements backend.Reader -func (r *reader) ReadRange(ctx context.Context, name string, blockID uuid.UUID, tenantID string, offset uint64, buffer []byte, shouldCache bool) error { - return r.r.ReadRange(ctx, name, KeyPathForBlock(blockID, tenantID), offset, buffer, shouldCache) +func (r *reader) ReadRange(ctx context.Context, name string, blockID uuid.UUID, tenantID string, offset uint64, buffer []byte, cacheInfo *CacheInfo) error { + return r.r.ReadRange(ctx, name, KeyPathForBlock(blockID, tenantID), offset, buffer, cacheInfo) } // Tenants implements backend.Reader @@ -184,7 +184,7 @@ func (r *reader) Blocks(ctx context.Context, tenantID string) ([]uuid.UUID, []uu // BlockMeta implements backend.Reader func (r *reader) BlockMeta(ctx context.Context, blockID uuid.UUID, tenantID string) (*BlockMeta, error) { - reader, size, err := r.r.Read(ctx, MetaName, KeyPathForBlock(blockID, tenantID), false) + reader, size, err := r.r.Read(ctx, MetaName, KeyPathForBlock(blockID, tenantID), nil) if err != nil { return nil, err } @@ -206,7 +206,7 @@ func (r *reader) BlockMeta(ctx context.Context, blockID uuid.UUID, tenantID stri // TenantIndex implements backend.Reader func (r *reader) TenantIndex(ctx context.Context, tenantID string) (*TenantIndex, error) { - reader, size, err := r.r.Read(ctx, TenantIndexName, KeyPath([]string{tenantID}), false) + reader, size, err := r.r.Read(ctx, TenantIndexName, KeyPath([]string{tenantID}), nil) if err != nil { return nil, err } diff --git a/tempodb/backend/raw_test.go b/tempodb/backend/raw_test.go index b3b61518fde..960954c54d2 100644 --- a/tempodb/backend/raw_test.go +++ b/tempodb/backend/raw_test.go @@ -23,7 +23,7 @@ func TestWriter(t *testing.T) { expected := []byte{0x01, 0x02, 0x03, 0x04} - err := w.Write(ctx, "test", uuid.New(), "test", expected, false) + err := w.Write(ctx, "test", uuid.New(), "test", expected, nil) assert.NoError(t, err) assert.Equal(t, expected, m.writeBuffer) @@ -74,12 +74,12 @@ func TestReader(t *testing.T) { expected := []byte{0x01, 0x02, 0x03, 0x04} m.R = expected - actual, err := r.Read(ctx, "test", uuid.New(), "test", false) + actual, err := r.Read(ctx, "test", uuid.New(), "test", nil) assert.NoError(t, err) assert.Equal(t, expected, actual) m.Range = expected - err = r.ReadRange(ctx, "test", uuid.New(), "test", 10, actual, false) + err = r.ReadRange(ctx, "test", uuid.New(), "test", 10, actual, nil) assert.NoError(t, err) assert.Equal(t, expected, actual) diff --git a/tempodb/backend/readerat.go b/tempodb/backend/readerat.go index dbb17a82ffa..f2071606d31 100644 --- a/tempodb/backend/readerat.go +++ b/tempodb/backend/readerat.go @@ -19,31 +19,29 @@ type ContextReader interface { // backendReader is a shim that allows a backend.Reader to be used as a ContextReader type backendReader struct { - meta *BlockMeta - name string - r Reader - shouldCache bool + meta *BlockMeta + name string + r Reader } // NewContextReader creates a ReaderAt for the given BlockMeta -func NewContextReader(meta *BlockMeta, name string, r Reader, shouldCache bool) ContextReader { +func NewContextReader(meta *BlockMeta, name string, r Reader) ContextReader { return &backendReader{ - meta: meta, - name: name, - r: r, - shouldCache: shouldCache, + meta: meta, + name: name, + r: r, } } // ReadAt implements ContextReader func (b *backendReader) ReadAt(ctx context.Context, p []byte, off int64) (int, error) { - err := b.r.ReadRange(ctx, b.name, b.meta.BlockID, b.meta.TenantID, uint64(off), p, false) + err := b.r.ReadRange(ctx, b.name, b.meta.BlockID, b.meta.TenantID, uint64(off), p, nil) return len(p), err } // ReadAll implements ContextReader func (b *backendReader) ReadAll(ctx context.Context) ([]byte, error) { - return b.r.Read(ctx, b.name, b.meta.BlockID, b.meta.TenantID, b.shouldCache) + return b.r.Read(ctx, b.name, b.meta.BlockID, b.meta.TenantID, nil) } // Reader implements ContextReader diff --git a/tempodb/backend/s3/s3.go b/tempodb/backend/s3/s3.go index 7435fb23931..709a844660c 100644 --- a/tempodb/backend/s3/s3.go +++ b/tempodb/backend/s3/s3.go @@ -137,7 +137,7 @@ func getPutObjectOptions(rw *readerWriter) minio.PutObjectOptions { } // Write implements backend.Writer -func (rw *readerWriter) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, size int64, _ bool) error { +func (rw *readerWriter) Write(ctx context.Context, name string, keypath backend.KeyPath, data io.Reader, size int64, _ *backend.CacheInfo) error { span, derivedCtx := opentracing.StartSpanFromContext(ctx, "s3.Write") defer span.Finish() @@ -244,7 +244,7 @@ func (rw *readerWriter) CloseAppend(ctx context.Context, tracker backend.AppendT return nil } -func (rw *readerWriter) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ bool) error { +func (rw *readerWriter) Delete(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) error { filename := backend.ObjectFileName(keypath, name) return rw.core.RemoveObject(ctx, rw.cfg.Bucket, filename, minio.RemoveObjectOptions{}) } @@ -392,7 +392,7 @@ func (rw *readerWriter) ListBlocks( } // Read implements backend.Reader -func (rw *readerWriter) Read(ctx context.Context, name string, keypath backend.KeyPath, _ bool) (io.ReadCloser, int64, error) { +func (rw *readerWriter) Read(ctx context.Context, name string, keypath backend.KeyPath, _ *backend.CacheInfo) (io.ReadCloser, int64, error) { span, derivedCtx := opentracing.StartSpanFromContext(ctx, "s3.Read") defer span.Finish() @@ -406,7 +406,7 @@ func (rw *readerWriter) Read(ctx context.Context, name string, keypath backend.K } // ReadRange implements backend.Reader -func (rw *readerWriter) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ bool) error { +func (rw *readerWriter) ReadRange(ctx context.Context, name string, keypath backend.KeyPath, offset uint64, buffer []byte, _ *backend.CacheInfo) error { span, derivedCtx := opentracing.StartSpanFromContext(ctx, "s3.ReadRange", opentracing.Tags{ "len": len(buffer), "offset": offset, @@ -442,7 +442,7 @@ func (rw *readerWriter) WriteVersioned(ctx context.Context, name string, keypath } // TODO extract Write to a separate method which returns minio.UploadInfo, saves us a GetObject request - err = rw.Write(ctx, name, keypath, data, -1, false) + err = rw.Write(ctx, name, keypath, data, -1, nil) if err != nil { return "", err } @@ -464,7 +464,7 @@ func (rw *readerWriter) DeleteVersioned(ctx context.Context, name string, keypat return backend.ErrVersionDoesNotMatch } - return rw.Delete(ctx, name, keypath, false) + return rw.Delete(ctx, name, keypath, nil) } func (rw *readerWriter) ReadVersioned(ctx context.Context, name string, keypath backend.KeyPath) (io.ReadCloser, backend.Version, error) { diff --git a/tempodb/backend/s3/s3_test.go b/tempodb/backend/s3/s3_test.go index fcb06fd4891..f4f460972c0 100644 --- a/tempodb/backend/s3/s3_test.go +++ b/tempodb/backend/s3/s3_test.go @@ -249,17 +249,17 @@ func TestHedge(t *testing.T) { // the first call on each client initiates an extra http request // clearing that here - _, _, _ = r.Read(ctx, "object", backend.KeyPath{"test"}, false) + _, _, _ = r.Read(ctx, "object", backend.KeyPath{"test"}, nil) time.Sleep(tc.returnIn) atomic.StoreInt32(&count, 0) // calls that should hedge - _, _, _ = r.Read(ctx, "object", backend.KeyPath{"test"}, false) + _, _, _ = r.Read(ctx, "object", backend.KeyPath{"test"}, nil) time.Sleep(tc.returnIn) assert.Equal(t, tc.expectedHedgedRequests, atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) - _ = r.ReadRange(ctx, "object", backend.KeyPath{"test"}, 10, []byte{}, false) + _ = r.ReadRange(ctx, "object", backend.KeyPath{"test"}, 10, []byte{}, nil) time.Sleep(tc.returnIn) assert.Equal(t, tc.expectedHedgedRequests, atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) @@ -269,7 +269,7 @@ func TestHedge(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) - _ = w.Write(ctx, "object", backend.KeyPath{"test"}, bytes.NewReader([]byte{}), 0, false) + _ = w.Write(ctx, "object", backend.KeyPath{"test"}, bytes.NewReader([]byte{}), 0, nil) assert.Equal(t, int32(1), atomic.LoadInt32(&count)) atomic.StoreInt32(&count, 0) }) @@ -367,7 +367,7 @@ func TestObjectBlockTags(t *testing.T) { require.NoError(t, err) ctx := context.Background() - _ = w.Write(ctx, "object", backend.KeyPath{"test"}, bytes.NewReader([]byte{}), 0, false) + _ = w.Write(ctx, "object", backend.KeyPath{"test"}, bytes.NewReader([]byte{}), 0, nil) for k, v := range tc.tags { vv := obj.Get(k) @@ -443,7 +443,7 @@ func TestObjectWithPrefix(t *testing.T) { require.NoError(t, err) ctx := context.Background() - err = w.Write(ctx, tc.objectName, tc.keyPath, bytes.NewReader([]byte{}), 0, false) + err = w.Write(ctx, tc.objectName, tc.keyPath, bytes.NewReader([]byte{}), 0, nil) assert.NoError(t, err) }) } @@ -478,7 +478,7 @@ func TestObjectStorageClass(t *testing.T) { require.NoError(t, err) ctx := context.Background() - _ = w.Write(ctx, "object", backend.KeyPath{"test"}, bytes.NewReader([]byte{}), 0, false) + _ = w.Write(ctx, "object", backend.KeyPath{"test"}, bytes.NewReader([]byte{}), 0, nil) require.Equal(t, obj.Has(tc.StorageClass), true) }) } diff --git a/tempodb/backend/versioned.go b/tempodb/backend/versioned.go index 5a7611e653d..f8b966899e8 100644 --- a/tempodb/backend/versioned.go +++ b/tempodb/backend/versioned.go @@ -49,15 +49,15 @@ func NewFakeVersionedReaderWriter(r RawReader, w RawWriter) *FakeVersionedReader } func (f *FakeVersionedReaderWriter) WriteVersioned(ctx context.Context, name string, keypath KeyPath, data io.Reader, _ Version) (Version, error) { - err := f.Write(ctx, name, keypath, data, -1, false) + err := f.Write(ctx, name, keypath, data, -1, nil) return VersionNew, err } func (f *FakeVersionedReaderWriter) ReadVersioned(ctx context.Context, name string, keypath KeyPath) (io.ReadCloser, Version, error) { - readCloser, _, err := f.Read(ctx, name, keypath, false) + readCloser, _, err := f.Read(ctx, name, keypath, nil) return readCloser, VersionNew, err } func (f *FakeVersionedReaderWriter) DeleteVersioned(ctx context.Context, name string, keypath KeyPath, _ Version) error { - return f.Delete(ctx, name, keypath, false) + return f.Delete(ctx, name, keypath, nil) } diff --git a/tempodb/compactor.go b/tempodb/compactor.go index f4e264ac763..b20418a7e12 100644 --- a/tempodb/compactor.go +++ b/tempodb/compactor.go @@ -241,7 +241,7 @@ func (rw *readerWriter) compact(ctx context.Context, blockMetas []*backend.Block compactor := enc.NewCompactor(opts) // Compact selected blocks into a larger one - newCompactedBlocks, err := compactor.Compact(ctx, rw.logger, rw.r, rw.getWriterForBlock, blockMetas) + newCompactedBlocks, err := compactor.Compact(ctx, rw.logger, rw.r, rw.w, blockMetas) if err != nil { return err } diff --git a/tempodb/compactor_test.go b/tempodb/compactor_test.go index e6c8d6c9f66..1206118acfe 100644 --- a/tempodb/compactor_test.go +++ b/tempodb/compactor_test.go @@ -98,7 +98,7 @@ func testCompactionRoundtrip(t *testing.T, targetBlockVersion string) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err) ctx := context.Background() @@ -245,7 +245,7 @@ func testSameIDCompaction(t *testing.T, targetBlockVersion string) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err) ctx := context.Background() @@ -389,7 +389,7 @@ func TestCompactionUpdatesBlocklist(t *testing.T) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err) ctx := context.Background() @@ -460,7 +460,7 @@ func TestCompactionMetrics(t *testing.T) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) assert.NoError(t, err) ctx := context.Background() @@ -534,7 +534,7 @@ func TestCompactionIteratesThroughTenants(t *testing.T) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) assert.NoError(t, err) ctx := context.Background() @@ -607,7 +607,7 @@ func testCompactionHonorsBlockStartEndTimes(t *testing.T, targetBlockVersion str IngestionSlack: time.Since(time.Unix(0, 0)), // Let us use obvious start/end times below }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err) ctx := context.Background() @@ -737,7 +737,7 @@ func benchmarkCompaction(b *testing.B, targetBlockVersion string) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(b, err) rw := c.(*readerWriter) diff --git a/tempodb/config.go b/tempodb/config.go index d6310e12b60..76d0cbbc35d 100644 --- a/tempodb/config.go +++ b/tempodb/config.go @@ -6,10 +6,12 @@ import ( "fmt" "time" + "github.com/grafana/tempo/modules/cache/memcached" + "github.com/grafana/tempo/modules/cache/redis" + "github.com/grafana/tempo/pkg/cache" azure "github.com/grafana/tempo/tempodb/backend/azure/config" - "github.com/grafana/tempo/tempodb/backend/cache/memcached" - "github.com/grafana/tempo/tempodb/backend/cache/redis" + backend_cache "github.com/grafana/tempo/tempodb/backend/cache" "github.com/grafana/tempo/tempodb/backend/gcs" "github.com/grafana/tempo/tempodb/backend/local" "github.com/grafana/tempo/tempodb/backend/s3" @@ -56,13 +58,20 @@ type Config struct { S3 *s3.Config `yaml:"s3"` Azure *azure.Config `yaml:"azure"` - // caches - Cache string `yaml:"cache"` - CacheMinCompactionLevel uint8 `yaml:"cache_min_compaction_level"` - CacheMaxBlockAge time.Duration `yaml:"cache_max_block_age"` - BackgroundCache *cache.BackgroundConfig `yaml:"background_cache"` - Memcached *memcached.Config `yaml:"memcached"` - Redis *redis.Config `yaml:"redis"` + // legacy cache config. this is loaded by tempodb and added to the cache + // provider on construction + Cache string `yaml:"cache"` + BackgroundCache *cache.BackgroundConfig `yaml:"background_cache"` + Memcached *memcached.Config `yaml:"memcached"` + Redis *redis.Config `yaml:"redis"` + + BloomCacheCfg backend_cache.BloomConfig `yaml:",inline"` +} + +type CacheControlConfig struct { + Footer bool `yaml:"footer"` + ColumnIndex bool `yaml:"column_index"` + OffsetIndex bool `yaml:"offset_index"` } type SearchConfig struct { @@ -73,11 +82,8 @@ type SearchConfig struct { // vParquet blocks ReadBufferCount int `yaml:"read_buffer_count"` ReadBufferSizeBytes int `yaml:"read_buffer_size_bytes"` - CacheControl struct { - Footer bool `yaml:"footer"` - ColumnIndex bool `yaml:"column_index"` - OffsetIndex bool `yaml:"offset_index"` - } `yaml:"cache_control"` + // todo: consolidate caching conffig in one spot + CacheControl CacheControlConfig `yaml:"cache_control"` } func (c *SearchConfig) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) { @@ -105,10 +111,6 @@ func (c SearchConfig) ApplyToOptions(o *common.SearchOptions) { if o.ReadBufferCount <= 0 { o.ReadBufferCount = DefaultReadBufferCount } - - o.CacheControl.Footer = c.CacheControl.Footer - o.CacheControl.ColumnIndex = c.CacheControl.ColumnIndex - o.CacheControl.OffsetIndex = c.CacheControl.OffsetIndex } // CompactorConfig contains compaction configuration options diff --git a/tempodb/encoding/common/interfaces.go b/tempodb/encoding/common/interfaces.go index 87130293998..c46b420acda 100644 --- a/tempodb/encoding/common/interfaces.go +++ b/tempodb/encoding/common/interfaces.go @@ -2,7 +2,6 @@ package common import ( "context" - "time" "github.com/go-kit/log" @@ -30,12 +29,6 @@ type Searcher interface { FetchTagValues(context.Context, traceql.AutocompleteRequest, traceql.AutocompleteCallback, SearchOptions) error } -type CacheControl struct { - Footer bool - ColumnIndex bool - OffsetIndex bool -} - type SearchOptions struct { ChunkSizeBytes uint32 // Buffer size to read from backend storage. StartPage int // Controls searching only a subset of the block. Which page to begin searching at. @@ -44,7 +37,6 @@ type SearchOptions struct { PrefetchTraceCount int // How many traces to prefetch async. ReadBufferCount int ReadBufferSize int - CacheControl CacheControl } // DefaultSearchOptions() is used in a lot of places such as local ingester searches. It is important @@ -70,7 +62,7 @@ func DefaultSearchOptionsWithMaxBytes(maxBytes int) SearchOptions { } type Compactor interface { - Compact(ctx context.Context, l log.Logger, r backend.Reader, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, inputs []*backend.BlockMeta) ([]*backend.BlockMeta, error) + Compact(ctx context.Context, l log.Logger, r backend.Reader, w backend.Writer, inputs []*backend.BlockMeta) ([]*backend.BlockMeta, error) } type CompactionOptions struct { diff --git a/tempodb/encoding/v2/backend_block.go b/tempodb/encoding/v2/backend_block.go index 7711bc1d5ee..d43c7dec8ae 100644 --- a/tempodb/encoding/v2/backend_block.go +++ b/tempodb/encoding/v2/backend_block.go @@ -8,6 +8,7 @@ import ( "github.com/opentracing/opentracing-go" willf_bloom "github.com/willf/bloom" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/model" "github.com/grafana/tempo/pkg/tempopb" "github.com/grafana/tempo/pkg/traceql" @@ -52,7 +53,10 @@ func (b *BackendBlock) find(ctx context.Context, id common.ID) ([]byte, error) { tenantID := b.meta.TenantID nameBloom := common.BloomName(shardKey) - bloomBytes, err := b.reader.Read(ctx, nameBloom, blockID, tenantID, true) + bloomBytes, err := b.reader.Read(ctx, nameBloom, blockID, tenantID, &backend.CacheInfo{ + Meta: b.meta, + Role: cache.RoleBloom, + }) if err != nil { return nil, fmt.Errorf("error retrieving bloom %s (%s, %s): %w", nameBloom, b.meta.TenantID, b.meta.BlockID, err) } @@ -67,13 +71,13 @@ func (b *BackendBlock) find(ctx context.Context, id common.ID) ([]byte, error) { return nil, nil } - indexReaderAt := backend.NewContextReader(b.meta, common.NameIndex, b.reader, false) + indexReaderAt := backend.NewContextReader(b.meta, common.NameIndex, b.reader) indexReader, err := NewIndexReader(indexReaderAt, int(b.meta.IndexPageSize), int(b.meta.TotalRecords)) if err != nil { return nil, fmt.Errorf("error building index reader (%s, %s): %w", b.meta.TenantID, b.meta.BlockID, err) } - ra := backend.NewContextReader(b.meta, common.NameObjects, b.reader, false) + ra := backend.NewContextReader(b.meta, common.NameObjects, b.reader) dataReader, err := NewDataReader(ra, b.meta.Encoding) if err != nil { return nil, fmt.Errorf("error building page reader (%s, %s): %w", b.meta.TenantID, b.meta.BlockID, err) @@ -93,7 +97,7 @@ func (b *BackendBlock) find(ctx context.Context, id common.ID) ([]byte, error) { // Iterator returns an Iterator that iterates over the objects in the block from the backend func (b *BackendBlock) Iterator(chunkSizeBytes uint32) (BytesIterator, error) { // read index - ra := backend.NewContextReader(b.meta, common.NameObjects, b.reader, false) + ra := backend.NewContextReader(b.meta, common.NameObjects, b.reader) dataReader, err := NewDataReader(ra, b.meta.Encoding) if err != nil { return nil, fmt.Errorf("failed to create dataReader (%s, %s): %w", b.meta.TenantID, b.meta.BlockID, err) @@ -108,7 +112,7 @@ func (b *BackendBlock) Iterator(chunkSizeBytes uint32) (BytesIterator, error) { } func (b *BackendBlock) NewIndexReader() (IndexReader, error) { - indexReaderAt := backend.NewContextReader(b.meta, common.NameIndex, b.reader, false) + indexReaderAt := backend.NewContextReader(b.meta, common.NameIndex, b.reader) reader, err := NewIndexReader(indexReaderAt, int(b.meta.IndexPageSize), int(b.meta.TotalRecords)) if err != nil { return nil, fmt.Errorf("failed to create index reader (%s, %s): %w", b.meta.TenantID, b.meta.BlockID, err) diff --git a/tempodb/encoding/v2/block.go b/tempodb/encoding/v2/block.go index 6a966e32c27..e2de0c2ff68 100644 --- a/tempodb/encoding/v2/block.go +++ b/tempodb/encoding/v2/block.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/encoding/common" ) @@ -16,15 +17,19 @@ func writeBlockMeta(ctx context.Context, w backend.Writer, meta *backend.BlockMe } // index - err = w.Write(ctx, common.NameIndex, meta.BlockID, meta.TenantID, indexBytes, false) + err = w.Write(ctx, common.NameIndex, meta.BlockID, meta.TenantID, indexBytes, nil) if err != nil { return fmt.Errorf("unexpected error writing index: %w", err) } + cacheInfo := &backend.CacheInfo{ + Meta: meta, + Role: cache.RoleBloom, + } // bloom for i, bloom := range blooms { nameBloom := common.BloomName(i) - err := w.Write(ctx, nameBloom, meta.BlockID, meta.TenantID, bloom, true) + err := w.Write(ctx, nameBloom, meta.BlockID, meta.TenantID, bloom, cacheInfo) if err != nil { return fmt.Errorf("unexpected error writing bloom-%d: %w", i, err) } @@ -57,14 +62,20 @@ func CopyBlock(ctx context.Context, srcMeta, destMeta *backend.BlockMeta, src ba return dest.StreamWriter(ctx, name, destMeta.BlockID, destMeta.TenantID, reader, size) } + cacheInfo := &backend.CacheInfo{ + Role: cache.RoleBloom, + } + // Read entire object and attempt to cache - cpy := func(name string) error { - b, err := src.Read(ctx, name, srcMeta.BlockID, srcMeta.TenantID, true) + cpyBloom := func(name string) error { + cacheInfo.Meta = srcMeta + b, err := src.Read(ctx, name, srcMeta.BlockID, srcMeta.TenantID, cacheInfo) if err != nil { return fmt.Errorf("error reading %s: %w", name, err) } - return dest.Write(ctx, name, destMeta.BlockID, destMeta.TenantID, b, true) + cacheInfo.Meta = destMeta + return dest.Write(ctx, name, destMeta.BlockID, destMeta.TenantID, b, cacheInfo) } // Data @@ -75,7 +86,7 @@ func CopyBlock(ctx context.Context, srcMeta, destMeta *backend.BlockMeta, src ba // Bloom for i := 0; i < common.ValidateShardCount(int(srcMeta.BloomShardCount)); i++ { - err = cpy(common.BloomName(i)) + err = cpyBloom(common.BloomName(i)) if err != nil { return err } diff --git a/tempodb/encoding/v2/compactor.go b/tempodb/encoding/v2/compactor.go index 031c7a5fe82..55230504487 100644 --- a/tempodb/encoding/v2/compactor.go +++ b/tempodb/encoding/v2/compactor.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "runtime" - "time" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -26,7 +25,7 @@ func NewCompactor(opts common.CompactionOptions) *Compactor { return &Compactor{opts} } -func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error) { +func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, w backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error) { tenantID := inputs[0].TenantID dataEncoding := inputs[0].DataEncoding // blocks chosen for compaction always have the same data encoding @@ -106,7 +105,7 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, // write partial block if currentBlock.CurrentBufferLength() >= int(c.opts.FlushSizeBytes) { runtime.GC() - tracker, err = c.appendBlock(ctx, writerCallback, tracker, currentBlock) + tracker, err = c.appendBlock(ctx, w, tracker, currentBlock) if err != nil { return nil, fmt.Errorf("error writing partial block: %w", err) } @@ -114,7 +113,7 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, // ship block to backend if done if currentBlock.Length() >= recordsPerBlock { - err = c.finishBlock(ctx, writerCallback, tracker, currentBlock, l) + err = c.finishBlock(ctx, w, tracker, currentBlock, l) if err != nil { return nil, fmt.Errorf("error shipping block to backend: %w", err) } @@ -125,7 +124,7 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, // ship final block to backend if currentBlock != nil { - err = c.finishBlock(ctx, writerCallback, tracker, currentBlock, l) + err = c.finishBlock(ctx, w, tracker, currentBlock, l) if err != nil { return nil, fmt.Errorf("error shipping block to backend: %w", err) } @@ -134,14 +133,14 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, return newCompactedBlocks, nil } -func (c *Compactor) appendBlock(ctx context.Context, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, tracker backend.AppendTracker, block *StreamingBlock) (backend.AppendTracker, error) { +func (c *Compactor) appendBlock(ctx context.Context, w backend.Writer, tracker backend.AppendTracker, block *StreamingBlock) (backend.AppendTracker, error) { compactionLevel := int(block.BlockMeta().CompactionLevel - 1) if c.opts.ObjectsWritten != nil { c.opts.ObjectsWritten(compactionLevel, block.CurrentBufferedObjects()) } - tracker, bytesFlushed, err := block.FlushBuffer(ctx, tracker, writerCallback(block.BlockMeta(), time.Now())) + tracker, bytesFlushed, err := block.FlushBuffer(ctx, tracker, w) if err != nil { return nil, err } @@ -153,10 +152,10 @@ func (c *Compactor) appendBlock(ctx context.Context, writerCallback func(*backen return tracker, nil } -func (c *Compactor) finishBlock(ctx context.Context, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, tracker backend.AppendTracker, block *StreamingBlock, l log.Logger) error { +func (c *Compactor) finishBlock(ctx context.Context, w backend.Writer, tracker backend.AppendTracker, block *StreamingBlock, l log.Logger) error { level.Info(l).Log("msg", "writing compacted block", "block", fmt.Sprintf("%+v", block.BlockMeta())) - bytesFlushed, err := block.Complete(ctx, tracker, writerCallback(block.BlockMeta(), time.Now())) + bytesFlushed, err := block.Complete(ctx, tracker, w) if err != nil { return err } diff --git a/tempodb/encoding/vparquet/block_findtracebyid.go b/tempodb/encoding/vparquet/block_findtracebyid.go index 09cd529e8ae..ddea9ee5138 100644 --- a/tempodb/encoding/vparquet/block_findtracebyid.go +++ b/tempodb/encoding/vparquet/block_findtracebyid.go @@ -11,6 +11,7 @@ import ( "github.com/parquet-go/parquet-go" "github.com/willf/bloom" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/parquetquery" pq "github.com/grafana/tempo/pkg/parquetquery" "github.com/grafana/tempo/pkg/tempopb" @@ -39,7 +40,10 @@ func (b *backendBlock) checkBloom(ctx context.Context, id common.ID) (found bool nameBloom := common.BloomName(shardKey) span.SetTag("bloom", nameBloom) - bloomBytes, err := b.r.Read(derivedCtx, nameBloom, b.meta.BlockID, b.meta.TenantID, true) + bloomBytes, err := b.r.Read(derivedCtx, nameBloom, b.meta.BlockID, b.meta.TenantID, &backend.CacheInfo{ + Meta: b.meta, + Role: cache.RoleBloom, + }) if err != nil { return false, fmt.Errorf("error retrieving bloom %s (%s, %s): %w", nameBloom, b.meta.TenantID, b.meta.BlockID, err) } diff --git a/tempodb/encoding/vparquet/block_iterator.go b/tempodb/encoding/vparquet/block_iterator.go index a54ebade9fa..351b149c000 100644 --- a/tempodb/encoding/vparquet/block_iterator.go +++ b/tempodb/encoding/vparquet/block_iterator.go @@ -14,7 +14,7 @@ import ( ) func (b *backendBlock) open(ctx context.Context) (*parquet.File, *parquet.Reader, error) { //nolint:all //deprecated - rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta.BlockID, b.meta.TenantID) + rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta) // 128 MB memory buffering br := tempo_io.NewBufferedReaderAt(rr, int64(b.meta.Size), 2*1024*1024, 64) diff --git a/tempodb/encoding/vparquet/block_search.go b/tempodb/encoding/vparquet/block_search.go index 28a5d5b7425..e7c73ea80f4 100644 --- a/tempodb/encoding/vparquet/block_search.go +++ b/tempodb/encoding/vparquet/block_search.go @@ -63,7 +63,7 @@ func (b *backendBlock) openForSearch(ctx context.Context, opts common.SearchOpti defer b.openMtx.Unlock() // TODO: ctx is also cached when we cache backendReaderAt, not ideal but leaving it as is for now - backendReaderAt := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta.BlockID, b.meta.TenantID) + backendReaderAt := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta) // no searches currently require bloom filters or the page index. so just add them statically o := []parquet.FileOption{ @@ -90,9 +90,7 @@ func (b *backendBlock) openForSearch(ctx context.Context, opts common.SearchOpti readerAt = newParquetOptimizedReaderAt(readerAt, int64(b.meta.Size), b.meta.FooterSize) // cached reader - if opts.CacheControl.ColumnIndex || opts.CacheControl.Footer || opts.CacheControl.OffsetIndex { - readerAt = newCachedReaderAt(readerAt, backendReaderAt, opts.CacheControl) - } + readerAt = newCachedReaderAt(readerAt, backendReaderAt) span, _ := opentracing.StartSpanFromContext(ctx, "parquet.OpenFile") defer span.Finish() diff --git a/tempodb/encoding/vparquet/compactor.go b/tempodb/encoding/vparquet/compactor.go index 1a3db6cdf9f..95b52a47b11 100644 --- a/tempodb/encoding/vparquet/compactor.go +++ b/tempodb/encoding/vparquet/compactor.go @@ -29,7 +29,7 @@ type Compactor struct { opts common.CompactionOptions } -func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error) { +func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, w backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error) { var ( compactionLevel uint8 totalRecords int @@ -152,7 +152,6 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, CompactionLevel: nextCompactionLevel, TotalObjects: recordsPerBlock, // Just an estimate } - w := writerCallback(newMeta, time.Now()) currentBlock = newStreamingBlock(ctx, &c.opts.BlockConfig, newMeta, r, w, tempo_io.NewBufferedWriter) currentBlock.meta.CompactionLevel = nextCompactionLevel diff --git a/tempodb/encoding/vparquet/compactor_test.go b/tempodb/encoding/vparquet/compactor_test.go index 8ff9d792c49..ec647a5e054 100644 --- a/tempodb/encoding/vparquet/compactor_test.go +++ b/tempodb/encoding/vparquet/compactor_test.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "math/rand" "testing" - "time" "github.com/go-kit/log" "github.com/google/uuid" @@ -64,7 +63,7 @@ func benchmarkCompactor(b *testing.B, traceCount, batchCount, spanCount int) { MaxBytesPerTrace: 50_000_000, }) - _, err = c.Compact(ctx, l, r, func(*backend.BlockMeta, time.Time) backend.Writer { return w }, inputs) + _, err = c.Compact(ctx, l, r, w, inputs) require.NoError(b, err) } } @@ -102,7 +101,7 @@ func BenchmarkCompactorDupes(b *testing.B) { SpansDiscarded: func(traceID, rootSpanName string, rootServiceName string, spans int) {}, }) - _, err = c.Compact(ctx, l, r, func(*backend.BlockMeta, time.Time) backend.Writer { return w }, inputs) + _, err = c.Compact(ctx, l, r, w, inputs) require.NoError(b, err) } } diff --git a/tempodb/encoding/vparquet/copy.go b/tempodb/encoding/vparquet/copy.go index c4b93cd4037..8d7c6c9f829 100644 --- a/tempodb/encoding/vparquet/copy.go +++ b/tempodb/encoding/vparquet/copy.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/encoding/common" ) @@ -20,14 +21,19 @@ func CopyBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from ba return to.StreamWriter(ctx, name, toMeta.BlockID, toMeta.TenantID, reader, size) } + cacheInfo := &backend.CacheInfo{ + Role: cache.RoleBloom, + } // Read entire object and attempt to cache - cpy := func(name string) error { - b, err := from.Read(ctx, name, fromMeta.BlockID, fromMeta.TenantID, true) + cpyBloom := func(name string) error { + cacheInfo.Meta = fromMeta + b, err := from.Read(ctx, name, fromMeta.BlockID, fromMeta.TenantID, cacheInfo) if err != nil { return fmt.Errorf("error reading %s: %w", name, err) } - return to.Write(ctx, name, toMeta.BlockID, toMeta.TenantID, b, true) + cacheInfo.Meta = toMeta + return to.Write(ctx, name, toMeta.BlockID, toMeta.TenantID, b, cacheInfo) } // Data @@ -38,7 +44,7 @@ func CopyBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from ba // Bloom for i := 0; i < common.ValidateShardCount(int(fromMeta.BloomShardCount)); i++ { - err = cpy(common.BloomName(i)) + err = cpyBloom(common.BloomName(i)) if err != nil { return err } @@ -55,9 +61,13 @@ func writeBlockMeta(ctx context.Context, w backend.Writer, meta *backend.BlockMe if err != nil { return err } + cacheInfo := &backend.CacheInfo{ + Role: cache.RoleBloom, + Meta: meta, + } for i, bloom := range blooms { nameBloom := common.BloomName(i) - err := w.Write(ctx, nameBloom, meta.BlockID, meta.TenantID, bloom, true) + err := w.Write(ctx, nameBloom, meta.BlockID, meta.TenantID, bloom, cacheInfo) if err != nil { return fmt.Errorf("unexpected error writing bloom-%d: %w", i, err) } diff --git a/tempodb/encoding/vparquet/create.go b/tempodb/encoding/vparquet/create.go index e34c254b06a..3bef74f2a9d 100644 --- a/tempodb/encoding/vparquet/create.go +++ b/tempodb/encoding/vparquet/create.go @@ -218,7 +218,7 @@ func (b *streamingBlock) Complete() (int, error) { // Read the footer size out of the parquet footer buf := make([]byte, 8) - err = b.r.ReadRange(b.ctx, DataFileName, b.meta.BlockID, b.meta.TenantID, b.meta.Size-8, buf, false) + err = b.r.ReadRange(b.ctx, DataFileName, b.meta.BlockID, b.meta.TenantID, b.meta.Size-8, buf, nil) if err != nil { return 0, fmt.Errorf("error reading parquet file footer: %w", err) } diff --git a/tempodb/encoding/vparquet/readers.go b/tempodb/encoding/vparquet/readers.go index 757362d9e34..4e852500465 100644 --- a/tempodb/encoding/vparquet/readers.go +++ b/tempodb/encoding/vparquet/readers.go @@ -5,11 +5,10 @@ import ( "encoding/binary" "io" - "github.com/google/uuid" "go.uber.org/atomic" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/tempodb/backend" - "github.com/grafana/tempo/tempodb/encoding/common" ) // This stack of readers is used to bridge the gap between the backend.Reader and the parquet.File. @@ -21,32 +20,34 @@ import ( // BackendReaderAt is used to track backend requests and present a io.ReaderAt interface backed // by a backend.Reader type BackendReaderAt struct { - ctx context.Context - r backend.Reader - name string - blockID uuid.UUID - tenantID string + ctx context.Context + r backend.Reader + name string + meta *backend.BlockMeta bytesRead atomic.Uint64 } var _ io.ReaderAt = (*BackendReaderAt)(nil) -func NewBackendReaderAt(ctx context.Context, r backend.Reader, name string, blockID uuid.UUID, tenantID string) *BackendReaderAt { - return &BackendReaderAt{ctx, r, name, blockID, tenantID, atomic.Uint64{}} +func NewBackendReaderAt(ctx context.Context, r backend.Reader, name string, meta *backend.BlockMeta) *BackendReaderAt { + return &BackendReaderAt{ctx, r, name, meta, atomic.Uint64{}} } func (b *BackendReaderAt) ReadAt(p []byte, off int64) (int, error) { b.bytesRead.Add(uint64(len(p))) - err := b.r.ReadRange(b.ctx, b.name, b.blockID, b.tenantID, uint64(off), p, false) + err := b.r.ReadRange(b.ctx, b.name, b.meta.BlockID, b.meta.TenantID, uint64(off), p, nil) if err != nil { return 0, err } return len(p), err } -func (b *BackendReaderAt) ReadAtWithCache(p []byte, off int64) (int, error) { - err := b.r.ReadRange(b.ctx, b.name, b.blockID, b.tenantID, uint64(off), p, true) +func (b *BackendReaderAt) ReadAtWithCache(p []byte, off int64, role cache.Role) (int, error) { + err := b.r.ReadRange(b.ctx, b.name, b.meta.BlockID, b.meta.TenantID, uint64(off), p, &backend.CacheInfo{ + Role: role, + Meta: b.meta, + }) if err != nil { return 0, err } @@ -88,46 +89,45 @@ func (r *parquetOptimizedReaderAt) ReadAt(p []byte, off int64) (int, error) { return r.r.ReadAt(p, off) } +type cachedObjectRecord struct { + length int64 + role cache.Role +} + // cachedReaderAt is used to route specific reads to the caching layer. this must be passed directly into // the parquet.File so thet Set*Section() methods get called. type cachedReaderAt struct { r io.ReaderAt br *BackendReaderAt - cacheControl common.CacheControl - cachedObjects map[int64]int64 // storing offsets and length of objects we want to cache + cachedObjects map[int64]cachedObjectRecord // storing offsets and length of objects we want to cache } var _ io.ReaderAt = (*cachedReaderAt)(nil) -func newCachedReaderAt(br io.ReaderAt, rr *BackendReaderAt, cc common.CacheControl) *cachedReaderAt { - return &cachedReaderAt{br, rr, cc, map[int64]int64{}} +func newCachedReaderAt(br io.ReaderAt, rr *BackendReaderAt) *cachedReaderAt { + return &cachedReaderAt{br, rr, map[int64]cachedObjectRecord{}} } // called by parquet-go in OpenFile() to set offset and length of footer section func (r *cachedReaderAt) SetFooterSection(offset, length int64) { - if r.cacheControl.Footer { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetFooter} } // called by parquet-go in OpenFile() to set offset and length of column indexes func (r *cachedReaderAt) SetColumnIndexSection(offset, length int64) { - if r.cacheControl.ColumnIndex { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetColumnIdx} } // called by parquet-go in OpenFile() to set offset and length of offset index section func (r *cachedReaderAt) SetOffsetIndexSection(offset, length int64) { - if r.cacheControl.OffsetIndex { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetOffsetIdx} } func (r *cachedReaderAt) ReadAt(p []byte, off int64) (int, error) { // check if the offset and length is stored as a special object - if r.cachedObjects[off] == int64(len(p)) { - return r.br.ReadAtWithCache(p, off) + rec := r.cachedObjects[off] + if rec.length == int64(len(p)) { + return r.br.ReadAtWithCache(p, off, rec.role) } return r.r.ReadAt(p, off) diff --git a/tempodb/encoding/vparquet/readers_test.go b/tempodb/encoding/vparquet/readers_test.go index 4e468bd869a..23b0d5ef7bf 100644 --- a/tempodb/encoding/vparquet/readers_test.go +++ b/tempodb/encoding/vparquet/readers_test.go @@ -10,7 +10,6 @@ import ( "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/local" - "github.com/grafana/tempo/tempodb/encoding/common" ) var tenantID = "single-tenant" @@ -48,7 +47,7 @@ func TestParquetGoSetsMetadataSections(t *testing.T) { meta, err := r.BlockMeta(ctx, blocks[0], tenantID) require.NoError(t, err) - br := NewBackendReaderAt(ctx, r, DataFileName, meta.BlockID, tenantID) + br := NewBackendReaderAt(ctx, r, DataFileName, meta) dr := &dummyReader{r: br} _, err = parquet.OpenFile(dr, int64(meta.Size)) require.NoError(t, err) @@ -104,10 +103,10 @@ func TestCachingReaderAt(t *testing.T) { meta, err := r.BlockMeta(ctx, blocks[0], tenantID) require.NoError(t, err) - br := NewBackendReaderAt(ctx, r, DataFileName, meta.BlockID, tenantID) + br := NewBackendReaderAt(ctx, r, DataFileName, meta) rr := &recordingReaderAt{} - cr := newCachedReaderAt(rr, br, common.CacheControl{Footer: true, ColumnIndex: true, OffsetIndex: true}) + cr := newCachedReaderAt(rr, br) // cached items should not hit rr cr.SetColumnIndexSection(1, 34) diff --git a/tempodb/encoding/vparquet2/block_findtracebyid.go b/tempodb/encoding/vparquet2/block_findtracebyid.go index 10f490b4dae..7442c1320ce 100644 --- a/tempodb/encoding/vparquet2/block_findtracebyid.go +++ b/tempodb/encoding/vparquet2/block_findtracebyid.go @@ -12,6 +12,7 @@ import ( "github.com/parquet-go/parquet-go" "github.com/willf/bloom" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/parquetquery" pq "github.com/grafana/tempo/pkg/parquetquery" "github.com/grafana/tempo/pkg/tempopb" @@ -43,7 +44,10 @@ func (b *backendBlock) checkBloom(ctx context.Context, id common.ID) (found bool nameBloom := common.BloomName(shardKey) span.SetTag("bloom", nameBloom) - bloomBytes, err := b.r.Read(derivedCtx, nameBloom, b.meta.BlockID, b.meta.TenantID, true) + bloomBytes, err := b.r.Read(derivedCtx, nameBloom, b.meta.BlockID, b.meta.TenantID, &backend.CacheInfo{ + Meta: b.meta, + Role: cache.RoleBloom, + }) if err != nil { return false, fmt.Errorf("error retrieving bloom %s (%s, %s): %w", nameBloom, b.meta.TenantID, b.meta.BlockID, err) } @@ -70,7 +74,10 @@ func (b *backendBlock) checkIndex(ctx context.Context, id common.ID) (bool, int, }) defer span.Finish() - indexBytes, err := b.r.Read(derivedCtx, common.NameIndex, b.meta.BlockID, b.meta.TenantID, true) + indexBytes, err := b.r.Read(derivedCtx, common.NameIndex, b.meta.BlockID, b.meta.TenantID, &backend.CacheInfo{ + Meta: b.meta, + Role: cache.RoleTraceIDIdx, + }) if errors.Is(err, backend.ErrDoesNotExist) { return true, -1, nil } diff --git a/tempodb/encoding/vparquet2/block_iterator.go b/tempodb/encoding/vparquet2/block_iterator.go index 5593eb47163..51e54cbb2df 100644 --- a/tempodb/encoding/vparquet2/block_iterator.go +++ b/tempodb/encoding/vparquet2/block_iterator.go @@ -14,7 +14,7 @@ import ( ) func (b *backendBlock) open(ctx context.Context) (*parquet.File, *parquet.Reader, error) { //nolint:all //deprecated - rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta.BlockID, b.meta.TenantID) + rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta) // 128 MB memory buffering br := tempo_io.NewBufferedReaderAt(rr, int64(b.meta.Size), 2*1024*1024, 64) diff --git a/tempodb/encoding/vparquet2/block_search.go b/tempodb/encoding/vparquet2/block_search.go index 58a0cd7fdc3..5a689ef39d9 100644 --- a/tempodb/encoding/vparquet2/block_search.go +++ b/tempodb/encoding/vparquet2/block_search.go @@ -63,7 +63,7 @@ func (b *backendBlock) openForSearch(ctx context.Context, opts common.SearchOpti defer b.openMtx.Unlock() // TODO: ctx is also cached when we cache backendReaderAt, not ideal but leaving it as is for now - backendReaderAt := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta.BlockID, b.meta.TenantID) + backendReaderAt := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta) // no searches currently require bloom filters or the page index. so just add them statically o := []parquet.FileOption{ @@ -90,9 +90,7 @@ func (b *backendBlock) openForSearch(ctx context.Context, opts common.SearchOpti readerAt = newParquetOptimizedReaderAt(readerAt, int64(b.meta.Size), b.meta.FooterSize) // cached reader - if opts.CacheControl.ColumnIndex || opts.CacheControl.Footer || opts.CacheControl.OffsetIndex { - readerAt = newCachedReaderAt(readerAt, backendReaderAt, opts.CacheControl) - } + readerAt = newCachedReaderAt(readerAt, backendReaderAt) span, _ := opentracing.StartSpanFromContext(ctx, "parquet.OpenFile") defer span.Finish() diff --git a/tempodb/encoding/vparquet2/compactor.go b/tempodb/encoding/vparquet2/compactor.go index ffbaf2be223..714faa8c68a 100644 --- a/tempodb/encoding/vparquet2/compactor.go +++ b/tempodb/encoding/vparquet2/compactor.go @@ -29,7 +29,7 @@ type Compactor struct { opts common.CompactionOptions } -func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error) { +func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, w backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error) { var ( compactionLevel uint8 totalRecords int @@ -152,7 +152,6 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, CompactionLevel: nextCompactionLevel, TotalObjects: recordsPerBlock, // Just an estimate } - w := writerCallback(newMeta, time.Now()) currentBlock = newStreamingBlock(ctx, &c.opts.BlockConfig, newMeta, r, w, tempo_io.NewBufferedWriter) currentBlock.meta.CompactionLevel = nextCompactionLevel diff --git a/tempodb/encoding/vparquet2/compactor_test.go b/tempodb/encoding/vparquet2/compactor_test.go index 2821969716d..678ae8087e6 100644 --- a/tempodb/encoding/vparquet2/compactor_test.go +++ b/tempodb/encoding/vparquet2/compactor_test.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "math/rand" "testing" - "time" "github.com/go-kit/log" "github.com/google/uuid" @@ -64,7 +63,7 @@ func benchmarkCompactor(b *testing.B, traceCount, batchCount, spanCount int) { MaxBytesPerTrace: 50_000_000, }) - _, err = c.Compact(ctx, l, r, func(*backend.BlockMeta, time.Time) backend.Writer { return w }, inputs) + _, err = c.Compact(ctx, l, r, w, inputs) require.NoError(b, err) } } @@ -102,7 +101,7 @@ func BenchmarkCompactorDupes(b *testing.B) { SpansDiscarded: func(traceID, rootSpanName string, rootServiceName string, spans int) {}, }) - _, err = c.Compact(ctx, l, r, func(*backend.BlockMeta, time.Time) backend.Writer { return w }, inputs) + _, err = c.Compact(ctx, l, r, w, inputs) require.NoError(b, err) } } diff --git a/tempodb/encoding/vparquet2/copy.go b/tempodb/encoding/vparquet2/copy.go index 7c8088d0e77..c3ee1b304c2 100644 --- a/tempodb/encoding/vparquet2/copy.go +++ b/tempodb/encoding/vparquet2/copy.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/encoding/common" ) @@ -22,13 +23,15 @@ func CopyBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from ba } // Read entire object and attempt to cache - cpy := func(name string) error { - b, err := from.Read(ctx, name, fromMeta.BlockID, fromMeta.TenantID, true) + cpy := func(name string, cacheInfo *backend.CacheInfo) error { + cacheInfo.Meta = fromMeta + b, err := from.Read(ctx, name, fromMeta.BlockID, fromMeta.TenantID, cacheInfo) if err != nil { return fmt.Errorf("error reading %s: %w", name, err) } - return to.Write(ctx, name, toMeta.BlockID, toMeta.TenantID, b, true) + cacheInfo.Meta = toMeta + return to.Write(ctx, name, toMeta.BlockID, toMeta.TenantID, b, cacheInfo) } // Data @@ -39,14 +42,14 @@ func CopyBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from ba // Bloom for i := 0; i < common.ValidateShardCount(int(fromMeta.BloomShardCount)); i++ { - err = cpy(common.BloomName(i)) + err = cpy(common.BloomName(i), &backend.CacheInfo{Role: cache.RoleBloom}) if err != nil { return err } } // Index (may not exist) - err = cpy(common.NameIndex) + err = cpy(common.NameIndex, &backend.CacheInfo{Role: cache.RoleTraceIDIdx}) if err != nil && !errors.Is(err, backend.ErrDoesNotExist) { return err } @@ -62,9 +65,14 @@ func writeBlockMeta(ctx context.Context, w backend.Writer, meta *backend.BlockMe if err != nil { return err } + + cacheInfo := &backend.CacheInfo{ + Meta: meta, + Role: cache.RoleBloom, + } for i, bloom := range blooms { nameBloom := common.BloomName(i) - err := w.Write(ctx, nameBloom, meta.BlockID, meta.TenantID, bloom, true) + err := w.Write(ctx, nameBloom, meta.BlockID, meta.TenantID, bloom, cacheInfo) if err != nil { return fmt.Errorf("unexpected error writing bloom-%d: %w", i, err) } @@ -75,7 +83,10 @@ func writeBlockMeta(ctx context.Context, w backend.Writer, meta *backend.BlockMe if err != nil { return err } - err = w.Write(ctx, common.NameIndex, meta.BlockID, meta.TenantID, i, true) + err = w.Write(ctx, common.NameIndex, meta.BlockID, meta.TenantID, i, &backend.CacheInfo{ + Meta: meta, + Role: cache.RoleTraceIDIdx, + }) if err != nil { return err } diff --git a/tempodb/encoding/vparquet2/create.go b/tempodb/encoding/vparquet2/create.go index 4ff055d11f0..2572a0a1591 100644 --- a/tempodb/encoding/vparquet2/create.go +++ b/tempodb/encoding/vparquet2/create.go @@ -224,7 +224,7 @@ func (b *streamingBlock) Complete() (int, error) { // Read the footer size out of the parquet footer buf := make([]byte, 8) - err = b.r.ReadRange(b.ctx, DataFileName, b.meta.BlockID, b.meta.TenantID, b.meta.Size-8, buf, false) + err = b.r.ReadRange(b.ctx, DataFileName, b.meta.BlockID, b.meta.TenantID, b.meta.Size-8, buf, nil) if err != nil { return 0, fmt.Errorf("error reading parquet file footer: %w", err) } diff --git a/tempodb/encoding/vparquet2/readers.go b/tempodb/encoding/vparquet2/readers.go index 4090c31ba5d..39c56ffdffc 100644 --- a/tempodb/encoding/vparquet2/readers.go +++ b/tempodb/encoding/vparquet2/readers.go @@ -5,11 +5,10 @@ import ( "encoding/binary" "io" - "github.com/google/uuid" "go.uber.org/atomic" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/tempodb/backend" - "github.com/grafana/tempo/tempodb/encoding/common" ) // This stack of readers is used to bridge the gap between the backend.Reader and the parquet.File. @@ -21,32 +20,34 @@ import ( // BackendReaderAt is used to track backend requests and present a io.ReaderAt interface backed // by a backend.Reader type BackendReaderAt struct { - ctx context.Context - r backend.Reader - name string - blockID uuid.UUID - tenantID string + ctx context.Context + r backend.Reader + name string + meta *backend.BlockMeta bytesRead atomic.Uint64 } var _ io.ReaderAt = (*BackendReaderAt)(nil) -func NewBackendReaderAt(ctx context.Context, r backend.Reader, name string, blockID uuid.UUID, tenantID string) *BackendReaderAt { - return &BackendReaderAt{ctx, r, name, blockID, tenantID, atomic.Uint64{}} +func NewBackendReaderAt(ctx context.Context, r backend.Reader, name string, meta *backend.BlockMeta) *BackendReaderAt { + return &BackendReaderAt{ctx, r, name, meta, atomic.Uint64{}} } func (b *BackendReaderAt) ReadAt(p []byte, off int64) (int, error) { b.bytesRead.Add(uint64(len(p))) - err := b.r.ReadRange(b.ctx, b.name, b.blockID, b.tenantID, uint64(off), p, false) + err := b.r.ReadRange(b.ctx, b.name, b.meta.BlockID, b.meta.TenantID, uint64(off), p, nil) if err != nil { return 0, err } return len(p), err } -func (b *BackendReaderAt) ReadAtWithCache(p []byte, off int64) (int, error) { - err := b.r.ReadRange(b.ctx, b.name, b.blockID, b.tenantID, uint64(off), p, true) +func (b *BackendReaderAt) ReadAtWithCache(p []byte, off int64, role cache.Role) (int, error) { + err := b.r.ReadRange(b.ctx, b.name, b.meta.BlockID, b.meta.TenantID, uint64(off), p, &backend.CacheInfo{ + Role: role, + Meta: b.meta, + }) if err != nil { return 0, err } @@ -88,46 +89,45 @@ func (r *parquetOptimizedReaderAt) ReadAt(p []byte, off int64) (int, error) { return r.r.ReadAt(p, off) } +type cachedObjectRecord struct { + length int64 + role cache.Role +} + // cachedReaderAt is used to route specific reads to the caching layer. this must be passed directly into // the parquet.File so thet Set*Section() methods get called. type cachedReaderAt struct { r io.ReaderAt br *BackendReaderAt - cacheControl common.CacheControl - cachedObjects map[int64]int64 // storing offsets and length of objects we want to cache + cachedObjects map[int64]cachedObjectRecord // storing offsets and length of objects we want to cache } var _ io.ReaderAt = (*cachedReaderAt)(nil) -func newCachedReaderAt(br io.ReaderAt, rr *BackendReaderAt, cc common.CacheControl) *cachedReaderAt { - return &cachedReaderAt{br, rr, cc, map[int64]int64{}} +func newCachedReaderAt(br io.ReaderAt, rr *BackendReaderAt) *cachedReaderAt { + return &cachedReaderAt{br, rr, map[int64]cachedObjectRecord{}} } // called by parquet-go in OpenFile() to set offset and length of footer section func (r *cachedReaderAt) SetFooterSection(offset, length int64) { - if r.cacheControl.Footer { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetFooter} } // called by parquet-go in OpenFile() to set offset and length of column indexes func (r *cachedReaderAt) SetColumnIndexSection(offset, length int64) { - if r.cacheControl.ColumnIndex { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetColumnIdx} } // called by parquet-go in OpenFile() to set offset and length of offset index section func (r *cachedReaderAt) SetOffsetIndexSection(offset, length int64) { - if r.cacheControl.OffsetIndex { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetOffsetIdx} } func (r *cachedReaderAt) ReadAt(p []byte, off int64) (int, error) { // check if the offset and length is stored as a special object - if r.cachedObjects[off] == int64(len(p)) { - return r.br.ReadAtWithCache(p, off) + rec := r.cachedObjects[off] + if rec.length == int64(len(p)) { + return r.br.ReadAtWithCache(p, off, rec.role) } return r.r.ReadAt(p, off) diff --git a/tempodb/encoding/vparquet2/readers_test.go b/tempodb/encoding/vparquet2/readers_test.go index 392d756bf09..1a9ded01705 100644 --- a/tempodb/encoding/vparquet2/readers_test.go +++ b/tempodb/encoding/vparquet2/readers_test.go @@ -10,7 +10,6 @@ import ( "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/local" - "github.com/grafana/tempo/tempodb/encoding/common" ) var tenantID = "single-tenant" @@ -48,7 +47,7 @@ func TestParquetGoSetsMetadataSections(t *testing.T) { meta, err := r.BlockMeta(ctx, blocks[0], tenantID) require.NoError(t, err) - br := NewBackendReaderAt(ctx, r, DataFileName, meta.BlockID, tenantID) + br := NewBackendReaderAt(ctx, r, DataFileName, meta) dr := &dummyReader{r: br} _, err = parquet.OpenFile(dr, int64(meta.Size)) require.NoError(t, err) @@ -104,10 +103,10 @@ func TestCachingReaderAt(t *testing.T) { meta, err := r.BlockMeta(ctx, blocks[0], tenantID) require.NoError(t, err) - br := NewBackendReaderAt(ctx, r, DataFileName, meta.BlockID, tenantID) + br := NewBackendReaderAt(ctx, r, DataFileName, meta) rr := &recordingReaderAt{} - cr := newCachedReaderAt(rr, br, common.CacheControl{Footer: true, ColumnIndex: true, OffsetIndex: true}) + cr := newCachedReaderAt(rr, br) // cached items should not hit rr cr.SetColumnIndexSection(1, 34) diff --git a/tempodb/encoding/vparquet3/block_findtracebyid.go b/tempodb/encoding/vparquet3/block_findtracebyid.go index 119680898c7..bbc2e28a872 100644 --- a/tempodb/encoding/vparquet3/block_findtracebyid.go +++ b/tempodb/encoding/vparquet3/block_findtracebyid.go @@ -12,6 +12,7 @@ import ( "github.com/parquet-go/parquet-go" "github.com/willf/bloom" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/parquetquery" pq "github.com/grafana/tempo/pkg/parquetquery" "github.com/grafana/tempo/pkg/tempopb" @@ -43,7 +44,10 @@ func (b *backendBlock) checkBloom(ctx context.Context, id common.ID) (found bool nameBloom := common.BloomName(shardKey) span.SetTag("bloom", nameBloom) - bloomBytes, err := b.r.Read(derivedCtx, nameBloom, b.meta.BlockID, b.meta.TenantID, true) + bloomBytes, err := b.r.Read(derivedCtx, nameBloom, b.meta.BlockID, b.meta.TenantID, &backend.CacheInfo{ + Meta: b.meta, + Role: cache.RoleBloom, + }) if err != nil { return false, fmt.Errorf("error retrieving bloom %s (%s, %s): %w", nameBloom, b.meta.TenantID, b.meta.BlockID, err) } @@ -70,7 +74,10 @@ func (b *backendBlock) checkIndex(ctx context.Context, id common.ID) (bool, int, }) defer span.Finish() - indexBytes, err := b.r.Read(derivedCtx, common.NameIndex, b.meta.BlockID, b.meta.TenantID, true) + indexBytes, err := b.r.Read(derivedCtx, common.NameIndex, b.meta.BlockID, b.meta.TenantID, &backend.CacheInfo{ + Meta: b.meta, + Role: cache.RoleTraceIDIdx, + }) if errors.Is(err, backend.ErrDoesNotExist) { return true, -1, nil } diff --git a/tempodb/encoding/vparquet3/block_iterator.go b/tempodb/encoding/vparquet3/block_iterator.go index 3fc8758fede..71ada4895e1 100644 --- a/tempodb/encoding/vparquet3/block_iterator.go +++ b/tempodb/encoding/vparquet3/block_iterator.go @@ -14,7 +14,7 @@ import ( ) func (b *backendBlock) open(ctx context.Context) (*parquet.File, *parquet.Reader, error) { //nolint:all //deprecated - rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta.BlockID, b.meta.TenantID) + rr := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta) // 128 MB memory buffering br := tempo_io.NewBufferedReaderAt(rr, int64(b.meta.Size), 2*1024*1024, 64) diff --git a/tempodb/encoding/vparquet3/block_search.go b/tempodb/encoding/vparquet3/block_search.go index 10c13b8671e..15631cfd3ee 100644 --- a/tempodb/encoding/vparquet3/block_search.go +++ b/tempodb/encoding/vparquet3/block_search.go @@ -64,7 +64,7 @@ func (b *backendBlock) openForSearch(ctx context.Context, opts common.SearchOpti defer b.openMtx.Unlock() // TODO: ctx is also cached when we cache backendReaderAt, not ideal but leaving it as is for now - backendReaderAt := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta.BlockID, b.meta.TenantID) + backendReaderAt := NewBackendReaderAt(ctx, b.r, DataFileName, b.meta) // no searches currently require bloom filters or the page index. so just add them statically o := []parquet.FileOption{ @@ -91,9 +91,7 @@ func (b *backendBlock) openForSearch(ctx context.Context, opts common.SearchOpti readerAt = newParquetOptimizedReaderAt(readerAt, int64(b.meta.Size), b.meta.FooterSize) // cached reader - if opts.CacheControl.ColumnIndex || opts.CacheControl.Footer || opts.CacheControl.OffsetIndex { - readerAt = newCachedReaderAt(readerAt, backendReaderAt, opts.CacheControl) - } + readerAt = newCachedReaderAt(readerAt, backendReaderAt) span, _ := opentracing.StartSpanFromContext(ctx, "parquet.OpenFile") defer span.Finish() diff --git a/tempodb/encoding/vparquet3/compactor.go b/tempodb/encoding/vparquet3/compactor.go index 84e061c000d..32f6c308ce7 100644 --- a/tempodb/encoding/vparquet3/compactor.go +++ b/tempodb/encoding/vparquet3/compactor.go @@ -29,7 +29,7 @@ type Compactor struct { opts common.CompactionOptions } -func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error) { +func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, w backend.Writer, inputs []*backend.BlockMeta) (newCompactedBlocks []*backend.BlockMeta, err error) { var ( compactionLevel uint8 totalRecords int @@ -156,7 +156,6 @@ func (c *Compactor) Compact(ctx context.Context, l log.Logger, r backend.Reader, TotalObjects: recordsPerBlock, // Just an estimate DedicatedColumns: inputs[0].DedicatedColumns, } - w := writerCallback(newMeta, time.Now()) currentBlock = newStreamingBlock(ctx, &c.opts.BlockConfig, newMeta, r, w, tempo_io.NewBufferedWriter) currentBlock.meta.CompactionLevel = nextCompactionLevel diff --git a/tempodb/encoding/vparquet3/compactor_test.go b/tempodb/encoding/vparquet3/compactor_test.go index 2e54f3dc9e8..a2764fa39b8 100644 --- a/tempodb/encoding/vparquet3/compactor_test.go +++ b/tempodb/encoding/vparquet3/compactor_test.go @@ -6,7 +6,6 @@ import ( "flag" "math/rand" "testing" - "time" "github.com/go-kit/log" "github.com/google/uuid" @@ -64,7 +63,7 @@ func benchmarkCompactor(b *testing.B, traceCount, batchCount, spanCount int) { MaxBytesPerTrace: 50_000_000, }) - _, err = c.Compact(ctx, l, r, func(*backend.BlockMeta, time.Time) backend.Writer { return w }, inputs) + _, err = c.Compact(ctx, l, r, w, inputs) require.NoError(b, err) } } @@ -102,7 +101,7 @@ func BenchmarkCompactorDupes(b *testing.B) { SpansDiscarded: func(traceID, rootSpanName string, rootServiceName string, spans int) {}, }) - _, err = c.Compact(ctx, l, r, func(*backend.BlockMeta, time.Time) backend.Writer { return w }, inputs) + _, err = c.Compact(ctx, l, r, w, inputs) require.NoError(b, err) } } @@ -207,7 +206,7 @@ func TestCompact(t *testing.T) { inputs := []*backend.BlockMeta{meta1, meta2} - newMeta, err := c.Compact(context.Background(), log.NewNopLogger(), r, func(*backend.BlockMeta, time.Time) backend.Writer { return w }, inputs) + newMeta, err := c.Compact(context.Background(), log.NewNopLogger(), r, w, inputs) require.NoError(t, err) require.Len(t, newMeta, 1) require.Equal(t, 20, newMeta[0].TotalObjects) diff --git a/tempodb/encoding/vparquet3/copy.go b/tempodb/encoding/vparquet3/copy.go index a378be8f15a..67213b3bde2 100644 --- a/tempodb/encoding/vparquet3/copy.go +++ b/tempodb/encoding/vparquet3/copy.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/encoding/common" ) @@ -22,13 +23,15 @@ func CopyBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from ba } // Read entire object and attempt to cache - cpy := func(name string) error { - b, err := from.Read(ctx, name, fromMeta.BlockID, fromMeta.TenantID, true) + cpy := func(name string, cacheInfo *backend.CacheInfo) error { + cacheInfo.Meta = fromMeta + b, err := from.Read(ctx, name, fromMeta.BlockID, fromMeta.TenantID, cacheInfo) if err != nil { return fmt.Errorf("error reading %s: %w", name, err) } - return to.Write(ctx, name, toMeta.BlockID, toMeta.TenantID, b, true) + cacheInfo.Meta = toMeta + return to.Write(ctx, name, toMeta.BlockID, toMeta.TenantID, b, cacheInfo) } // Data @@ -38,15 +41,16 @@ func CopyBlock(ctx context.Context, fromMeta, toMeta *backend.BlockMeta, from ba } // Bloom + cacheInfo := &backend.CacheInfo{Role: cache.RoleBloom} for i := 0; i < common.ValidateShardCount(int(fromMeta.BloomShardCount)); i++ { - err = cpy(common.BloomName(i)) + err = cpy(common.BloomName(i), cacheInfo) if err != nil { return err } } // Index (may not exist) - err = cpy(common.NameIndex) + err = cpy(common.NameIndex, &backend.CacheInfo{Role: cache.RoleTraceIDIdx}) if err != nil && !errors.Is(err, backend.ErrDoesNotExist) { return err } @@ -62,9 +66,14 @@ func writeBlockMeta(ctx context.Context, w backend.Writer, meta *backend.BlockMe if err != nil { return err } + + cacheInfo := &backend.CacheInfo{ + Meta: meta, + Role: cache.RoleBloom, + } for i, bloom := range blooms { nameBloom := common.BloomName(i) - err := w.Write(ctx, nameBloom, meta.BlockID, meta.TenantID, bloom, true) + err := w.Write(ctx, nameBloom, meta.BlockID, meta.TenantID, bloom, cacheInfo) if err != nil { return fmt.Errorf("unexpected error writing bloom-%d: %w", i, err) } @@ -75,7 +84,10 @@ func writeBlockMeta(ctx context.Context, w backend.Writer, meta *backend.BlockMe if err != nil { return err } - err = w.Write(ctx, common.NameIndex, meta.BlockID, meta.TenantID, i, true) + err = w.Write(ctx, common.NameIndex, meta.BlockID, meta.TenantID, i, &backend.CacheInfo{ + Meta: meta, + Role: cache.RoleTraceIDIdx, + }) if err != nil { return err } diff --git a/tempodb/encoding/vparquet3/create.go b/tempodb/encoding/vparquet3/create.go index 85053f64740..692e3b8aeb1 100644 --- a/tempodb/encoding/vparquet3/create.go +++ b/tempodb/encoding/vparquet3/create.go @@ -224,7 +224,7 @@ func (b *streamingBlock) Complete() (int, error) { // Read the footer size out of the parquet footer buf := make([]byte, 8) - err = b.r.ReadRange(b.ctx, DataFileName, b.meta.BlockID, b.meta.TenantID, b.meta.Size-8, buf, false) + err = b.r.ReadRange(b.ctx, DataFileName, b.meta.BlockID, b.meta.TenantID, b.meta.Size-8, buf, nil) if err != nil { return 0, fmt.Errorf("error reading parquet file footer: %w", err) } diff --git a/tempodb/encoding/vparquet3/readers.go b/tempodb/encoding/vparquet3/readers.go index 5f6b93bd150..82894ca30c9 100644 --- a/tempodb/encoding/vparquet3/readers.go +++ b/tempodb/encoding/vparquet3/readers.go @@ -5,11 +5,10 @@ import ( "encoding/binary" "io" - "github.com/google/uuid" "go.uber.org/atomic" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/tempodb/backend" - "github.com/grafana/tempo/tempodb/encoding/common" ) // This stack of readers is used to bridge the gap between the backend.Reader and the parquet.File. @@ -21,32 +20,34 @@ import ( // BackendReaderAt is used to track backend requests and present a io.ReaderAt interface backed // by a backend.Reader type BackendReaderAt struct { - ctx context.Context - r backend.Reader - name string - blockID uuid.UUID - tenantID string + ctx context.Context + r backend.Reader + name string + meta *backend.BlockMeta bytesRead atomic.Uint64 } var _ io.ReaderAt = (*BackendReaderAt)(nil) -func NewBackendReaderAt(ctx context.Context, r backend.Reader, name string, blockID uuid.UUID, tenantID string) *BackendReaderAt { - return &BackendReaderAt{ctx, r, name, blockID, tenantID, atomic.Uint64{}} +func NewBackendReaderAt(ctx context.Context, r backend.Reader, name string, meta *backend.BlockMeta) *BackendReaderAt { + return &BackendReaderAt{ctx, r, name, meta, atomic.Uint64{}} } func (b *BackendReaderAt) ReadAt(p []byte, off int64) (int, error) { b.bytesRead.Add(uint64(len(p))) - err := b.r.ReadRange(b.ctx, b.name, b.blockID, b.tenantID, uint64(off), p, false) + err := b.r.ReadRange(b.ctx, b.name, b.meta.BlockID, b.meta.TenantID, uint64(off), p, nil) if err != nil { return 0, err } return len(p), err } -func (b *BackendReaderAt) ReadAtWithCache(p []byte, off int64) (int, error) { - err := b.r.ReadRange(b.ctx, b.name, b.blockID, b.tenantID, uint64(off), p, true) +func (b *BackendReaderAt) ReadAtWithCache(p []byte, off int64, role cache.Role) (int, error) { + err := b.r.ReadRange(b.ctx, b.name, b.meta.BlockID, b.meta.TenantID, uint64(off), p, &backend.CacheInfo{ + Role: role, + Meta: b.meta, + }) if err != nil { return 0, err } @@ -88,46 +89,45 @@ func (r *parquetOptimizedReaderAt) ReadAt(p []byte, off int64) (int, error) { return r.r.ReadAt(p, off) } +type cachedObjectRecord struct { + length int64 + role cache.Role +} + // cachedReaderAt is used to route specific reads to the caching layer. this must be passed directly into // the parquet.File so thet Set*Section() methods get called. type cachedReaderAt struct { r io.ReaderAt br *BackendReaderAt - cacheControl common.CacheControl - cachedObjects map[int64]int64 // storing offsets and length of objects we want to cache + cachedObjects map[int64]cachedObjectRecord // storing offsets and length of objects we want to cache } var _ io.ReaderAt = (*cachedReaderAt)(nil) -func newCachedReaderAt(br io.ReaderAt, rr *BackendReaderAt, cc common.CacheControl) *cachedReaderAt { - return &cachedReaderAt{br, rr, cc, map[int64]int64{}} +func newCachedReaderAt(br io.ReaderAt, rr *BackendReaderAt) *cachedReaderAt { + return &cachedReaderAt{br, rr, map[int64]cachedObjectRecord{}} } // called by parquet-go in OpenFile() to set offset and length of footer section func (r *cachedReaderAt) SetFooterSection(offset, length int64) { - if r.cacheControl.Footer { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetFooter} } // called by parquet-go in OpenFile() to set offset and length of column indexes func (r *cachedReaderAt) SetColumnIndexSection(offset, length int64) { - if r.cacheControl.ColumnIndex { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetColumnIdx} } // called by parquet-go in OpenFile() to set offset and length of offset index section func (r *cachedReaderAt) SetOffsetIndexSection(offset, length int64) { - if r.cacheControl.OffsetIndex { - r.cachedObjects[offset] = length - } + r.cachedObjects[offset] = cachedObjectRecord{length, cache.RoleParquetOffsetIdx} } func (r *cachedReaderAt) ReadAt(p []byte, off int64) (int, error) { // check if the offset and length is stored as a special object - if r.cachedObjects[off] == int64(len(p)) { - return r.br.ReadAtWithCache(p, off) + rec := r.cachedObjects[off] + if rec.length == int64(len(p)) { + return r.br.ReadAtWithCache(p, off, rec.role) } return r.r.ReadAt(p, off) diff --git a/tempodb/encoding/vparquet3/readers_test.go b/tempodb/encoding/vparquet3/readers_test.go index 69640bb901e..1bc1ff095a6 100644 --- a/tempodb/encoding/vparquet3/readers_test.go +++ b/tempodb/encoding/vparquet3/readers_test.go @@ -10,7 +10,6 @@ import ( "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/local" - "github.com/grafana/tempo/tempodb/encoding/common" ) var tenantID = "single-tenant" @@ -48,7 +47,7 @@ func TestParquetGoSetsMetadataSections(t *testing.T) { meta, err := r.BlockMeta(ctx, blocks[0], tenantID) require.NoError(t, err) - br := NewBackendReaderAt(ctx, r, DataFileName, meta.BlockID, tenantID) + br := NewBackendReaderAt(ctx, r, DataFileName, meta) dr := &dummyReader{r: br} _, err = parquet.OpenFile(dr, int64(meta.Size)) require.NoError(t, err) @@ -104,10 +103,10 @@ func TestCachingReaderAt(t *testing.T) { meta, err := r.BlockMeta(ctx, blocks[0], tenantID) require.NoError(t, err) - br := NewBackendReaderAt(ctx, r, DataFileName, meta.BlockID, tenantID) + br := NewBackendReaderAt(ctx, r, DataFileName, meta) rr := &recordingReaderAt{} - cr := newCachedReaderAt(rr, br, common.CacheControl{Footer: true, ColumnIndex: true, OffsetIndex: true}) + cr := newCachedReaderAt(rr, br) // cached items should not hit rr cr.SetColumnIndexSection(1, 34) diff --git a/tempodb/retention_test.go b/tempodb/retention_test.go index 20c15554192..59783b85e43 100644 --- a/tempodb/retention_test.go +++ b/tempodb/retention_test.go @@ -39,7 +39,7 @@ func TestRetention(t *testing.T) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) assert.NoError(t, err) ctx := context.Background() @@ -102,7 +102,7 @@ func TestRetentionUpdatesBlocklistImmediately(t *testing.T) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) assert.NoError(t, err) ctx := context.Background() @@ -171,7 +171,7 @@ func TestBlockRetentionOverride(t *testing.T) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err) overrides := &mockOverrides{} diff --git a/tempodb/tempodb.go b/tempodb/tempodb.go index b168970f7af..d8525139a3c 100644 --- a/tempodb/tempodb.go +++ b/tempodb/tempodb.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "errors" "fmt" "time" @@ -12,15 +13,18 @@ import ( gkLog "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/google/uuid" - pkg_cache "github.com/grafana/tempo/pkg/cache" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/grafana/tempo/modules/cache/memcached" + "github.com/grafana/tempo/modules/cache/redis" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/tempopb" "github.com/grafana/tempo/pkg/traceql" "github.com/grafana/tempo/pkg/util/log" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/azure" - "github.com/grafana/tempo/tempodb/backend/cache" - "github.com/grafana/tempo/tempodb/backend/cache/memcached" - "github.com/grafana/tempo/tempodb/backend/cache/redis" + backend_cache "github.com/grafana/tempo/tempodb/backend/cache" "github.com/grafana/tempo/tempodb/backend/gcs" "github.com/grafana/tempo/tempodb/backend/local" "github.com/grafana/tempo/tempodb/backend/s3" @@ -112,9 +116,6 @@ type readerWriter struct { w backend.Writer c backend.Compactor - uncachedReader backend.Reader - uncachedWriter backend.Writer - wal *wal.WAL pool *pool.Pool @@ -131,7 +132,7 @@ type readerWriter struct { } // New creates a new tempodb -func New(cfg *Config, logger gkLog.Logger) (Reader, Writer, Compactor, error) { +func New(cfg *Config, cacheProvider cache.Provider, logger gkLog.Logger) (Reader, Writer, Compactor, error) { var rawR backend.RawReader var rawW backend.RawWriter var c backend.Compactor @@ -158,20 +159,22 @@ func New(cfg *Config, logger gkLog.Logger) (Reader, Writer, Compactor, error) { return nil, nil, nil, err } - uncachedReader := backend.NewReader(rawR) - uncachedWriter := backend.NewWriter(rawW) - - var cacheBackend pkg_cache.Cache + // build a caching layer if we have a provider + if cacheProvider != nil { + legacyCache, roles, err := createLegacyCache(cfg, logger) + if err != nil { + return nil, nil, nil, err + } - switch cfg.Cache { - case "redis": - cacheBackend = redis.NewClient(cfg.Redis, cfg.BackgroundCache, logger) - case "memcached": - cacheBackend = memcached.NewClient(cfg.Memcached, cfg.BackgroundCache, logger) - } + // inject legacy cache into the cache provider for the roles + for _, role := range roles { + err = cacheProvider.AddCache(role, legacyCache) + if err != nil { + return nil, nil, nil, fmt.Errorf("error adding legacy cache to provider: %w", err) + } + } - if cacheBackend != nil { - rawR, rawW, err = cache.NewCache(rawR, rawW, cacheBackend) + rawR, rawW, err = backend_cache.NewCache(&cfg.BloomCacheCfg, rawR, rawW, cacheProvider, logger) if err != nil { return nil, nil, nil, err } @@ -180,15 +183,13 @@ func New(cfg *Config, logger gkLog.Logger) (Reader, Writer, Compactor, error) { r := backend.NewReader(rawR) w := backend.NewWriter(rawW) rw := &readerWriter{ - c: c, - r: r, - uncachedReader: uncachedReader, - uncachedWriter: uncachedWriter, - w: w, - cfg: cfg, - logger: logger, - pool: pool.NewPool(cfg.Pool), - blocklist: blocklist.New(), + c: c, + r: r, + w: w, + cfg: cfg, + logger: logger, + pool: pool.NewPool(cfg.Pool), + blocklist: blocklist.New(), } rw.wal, err = wal.New(rw.cfg.WAL) @@ -200,8 +201,7 @@ func New(cfg *Config, logger gkLog.Logger) (Reader, Writer, Compactor, error) { } func (rw *readerWriter) WriteBlock(ctx context.Context, c WriteableBlock) error { - w := rw.getWriterForBlock(c.BlockMeta(), time.Now()) - return c.Write(ctx, w) + return c.Write(ctx, rw.w) } // CompleteBlock iterates the given WAL block and flushes it to the TempoDB backend. @@ -317,11 +317,9 @@ func (rw *readerWriter) Find(ctx context.Context, tenantID string, id common.ID, rw.cfg.Search.ApplyToOptions(&opts) } - curTime := time.Now() partialTraces, funcErrs, err := rw.pool.RunJobs(ctx, copiedBlocklist, func(ctx context.Context, payload interface{}) (interface{}, error) { meta := payload.(*backend.BlockMeta) - r := rw.getReaderForBlock(meta, curTime) - block, err := encoding.OpenBlock(meta, r) + block, err := encoding.OpenBlock(meta, rw.r) if err != nil { return nil, fmt.Errorf("error opening block for reading, blockID: %s: %w", meta.BlockID.String(), err) } @@ -361,6 +359,7 @@ func (rw *readerWriter) Search(ctx context.Context, meta *backend.BlockMeta, req return block.Search(ctx, req, opts) } + func (rw *readerWriter) SearchForTags(ctx context.Context, meta *backend.BlockMeta, scope string, opts common.SearchOptions) (*tempopb.SearchTagsResponse, error) { attributeScope := traceql.AttributeScopeFromString(scope) @@ -437,6 +436,8 @@ func (rw *readerWriter) SearchForTagValuesV2(ctx context.Context, meta *backend. return resp, nil } + +// it only uses rw.r which has caching enabled func (rw *readerWriter) Fetch(ctx context.Context, meta *backend.BlockMeta, req traceql.FetchSpansRequest, opts common.SearchOptions) (traceql.FetchSpansResponse, error) { block, err := encoding.OpenBlock(meta, rw.r) if err != nil { @@ -547,36 +548,6 @@ func (rw *readerWriter) pollBlocklist() { rw.blocklist.ApplyPollResults(blocklist, compactedBlocklist) } -func (rw *readerWriter) shouldCache(meta *backend.BlockMeta, curTime time.Time) bool { - // compaction level is _atleast_ CacheMinCompactionLevel - if rw.cfg.CacheMinCompactionLevel > 0 && meta.CompactionLevel < rw.cfg.CacheMinCompactionLevel { - return false - } - - // block is not older than CacheMaxBlockAge - if rw.cfg.CacheMaxBlockAge > 0 && curTime.Sub(meta.StartTime) > rw.cfg.CacheMaxBlockAge { - return false - } - - return true -} - -func (rw *readerWriter) getReaderForBlock(meta *backend.BlockMeta, curTime time.Time) backend.Reader { - if rw.shouldCache(meta, curTime) { - return rw.r - } - - return rw.uncachedReader -} - -func (rw *readerWriter) getWriterForBlock(meta *backend.BlockMeta, curTime time.Time) backend.Writer { - if rw.shouldCache(meta, curTime) { - return rw.w - } - - return rw.uncachedWriter -} - // includeBlock indicates whether a given block should be included in a backend search func includeBlock(b *backend.BlockMeta, _ common.ID, blockStart []byte, blockEnd []byte, timeStart int64, timeEnd int64) bool { // todo: restore this functionality once it works. min/max ids are currently not recorded @@ -610,3 +581,50 @@ func includeCompactedBlock(c *backend.CompactedBlockMeta, id common.ID, blockSta } return includeBlock(&c.BlockMeta, id, blockStart, blockEnd, timeStart, timeEnd) } + +// createLegacyCache uses the config to return a cache and a list of roles. +func createLegacyCache(cfg *Config, logger gkLog.Logger) (cache.Cache, []cache.Role, error) { + var legacyCache cache.Cache + // if there's any cache configured, it always handles bloom filters and the trace id index + roles := []cache.Role{cache.RoleBloom, cache.RoleTraceIDIdx} + + switch cfg.Cache { + case "redis": + legacyCache = redis.NewClient(cfg.Redis, cfg.BackgroundCache, "legacy", logger) + case "memcached": + legacyCache = memcached.NewClient(cfg.Memcached, cfg.BackgroundCache, "legacy", logger) + } + + if legacyCache == nil { + if cfg.Search != nil && + (cfg.Search.CacheControl.ColumnIndex || + cfg.Search.CacheControl.Footer || + cfg.Search.CacheControl.OffsetIndex) { + return nil, nil, errors.New("no legacy cache configured, but cache_control is enabled. Please use the new top level cache configuration.") + } + + return nil, nil, nil + } + + // accumulate additional search roles + if cfg.Search != nil { + if cfg.Search.CacheControl.ColumnIndex { + roles = append(roles, cache.RoleParquetColumnIdx) + } + if cfg.Search.CacheControl.Footer { + roles = append(roles, cache.RoleParquetFooter) + } + if cfg.Search.CacheControl.OffsetIndex { + roles = append(roles, cache.RoleParquetOffsetIdx) + } + } + + // log the roles + rolesStr := make([]string, len(roles)) + for i, role := range roles { + rolesStr[i] = string(role) + } + level.Warn(logger).Log("msg", "legacy cache configured with the following roles. Please migrate to the new top level cache configuration.", "roles", rolesStr) + + return legacyCache, roles, nil +} diff --git a/tempodb/tempodb_search_test.go b/tempodb/tempodb_search_test.go index e2a24213439..36ccbe29327 100644 --- a/tempodb/tempodb_search_test.go +++ b/tempodb/tempodb_search_test.go @@ -1021,7 +1021,7 @@ func runCompleteBlockSearchTest(t *testing.T, blockVersion string, runners ...ru ReadBufferCount: 8, ReadBufferSizeBytes: 4 * 1024 * 1024, }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err) err = c.EnableCompaction(context.Background(), &CompactorConfig{ @@ -1451,7 +1451,7 @@ func TestWALBlockGetMetrics(t *testing.T) { ReadBufferCount: 8, ReadBufferSizeBytes: 4 * 1024 * 1024, }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err) err = c.EnableCompaction(context.Background(), &CompactorConfig{ diff --git a/tempodb/tempodb_test.go b/tempodb/tempodb_test.go index 1e59e8d1d3b..327a109e6df 100644 --- a/tempodb/tempodb_test.go +++ b/tempodb/tempodb_test.go @@ -2,6 +2,7 @@ package tempodb import ( "context" + "errors" "fmt" "math/rand" "os" @@ -12,9 +13,13 @@ import ( "github.com/go-kit/log" "github.com/golang/protobuf/proto" //nolint:all "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/grafana/tempo/modules/cache/memcached" + "github.com/grafana/tempo/modules/cache/redis" + "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/model" "github.com/grafana/tempo/pkg/model/trace" "github.com/grafana/tempo/pkg/tempopb" @@ -59,7 +64,7 @@ func testConfig(t *testing.T, enc backend.Encoding, blocklistPoll time.Duration, opt(cfg) } - r, w, c, err := New(cfg, log.NewNopLogger()) + r, w, c, err := New(cfg, nil, log.NewNopLogger()) require.NoError(t, err) return r, w, c, tempDir } @@ -668,7 +673,7 @@ func testCompleteBlockHonorsStartStopTimes(t *testing.T, targetBlockVersion stri Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(t, err) dec := model.MustNewSegmentDecoder(model.CurrentEncoding) @@ -696,72 +701,6 @@ func testCompleteBlockHonorsStartStopTimes(t *testing.T, targetBlockVersion stri require.Equal(t, now.Unix(), complete.BlockMeta().EndTime.Unix()) } -func TestShouldCache(t *testing.T) { - tempDir := t.TempDir() - - r, _, _, err := New(&Config{ - Backend: backend.Local, - Local: &local.Config{ - Path: path.Join(tempDir, "traces"), - }, - Block: &common.BlockConfig{ - IndexDownsampleBytes: 17, - BloomFP: .01, - BloomShardSizeBytes: 100_000, - Version: encoding.DefaultEncoding().Version(), - Encoding: backend.EncLZ4_256k, - IndexPageSizeBytes: 1000, - }, - WAL: &wal.Config{ - Filepath: path.Join(tempDir, "wal"), - }, - BlocklistPoll: 0, - CacheMaxBlockAge: time.Hour, - CacheMinCompactionLevel: 1, - }, log.NewNopLogger()) - require.NoError(t, err) - - rw := r.(*readerWriter) - - testCases := []struct { - name string - compactionLevel uint8 - startTime time.Time - cache bool - }{ - { - name: "both pass", - compactionLevel: 1, - startTime: time.Now(), - cache: true, - }, - { - name: "startTime fail", - compactionLevel: 2, - startTime: time.Now().Add(-2 * time.Hour), - cache: false, - }, - { - name: "compactionLevel fail", - compactionLevel: 0, - startTime: time.Now(), - cache: false, - }, - { - name: "both fail", - compactionLevel: 0, - startTime: time.Now(), - cache: false, - }, - } - - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.cache, rw.shouldCache(&backend.BlockMeta{CompactionLevel: tt.compactionLevel, StartTime: tt.startTime}, time.Now())) - }) - } -} - func writeTraceToWal(t require.TestingT, b common.WALBlock, dec model.SegmentDecoder, id common.ID, tr *tempopb.Trace, start, end uint32) { b1, err := dec.PrepareForWrite(tr, 0, 0) require.NoError(t, err) @@ -806,7 +745,7 @@ func benchmarkCompleteBlock(b *testing.B, e encoding.VersionedEncoding) { Filepath: path.Join(tempDir, "wal"), }, BlocklistPoll: 0, - }, log.NewNopLogger()) + }, nil, log.NewNopLogger()) require.NoError(b, err) dec := model.MustNewSegmentDecoder(model.CurrentEncoding) @@ -835,3 +774,81 @@ func benchmarkCompleteBlock(b *testing.B, e encoding.VersionedEncoding) { require.NoError(b, err) } } + +func TestCreateLegacyCache(t *testing.T) { + tcs := []struct { + name string + cfg *Config + expectedErr error + expectedRoles []cache.Role + expectedCache bool + }{ + { + name: "no caches", + cfg: &Config{}, + }, + { + name: "redis", + cfg: &Config{ + Cache: "redis", + Redis: &redis.Config{}, + BackgroundCache: &cache.BackgroundConfig{}, + }, + expectedRoles: []cache.Role{cache.RoleBloom, cache.RoleTraceIDIdx}, + expectedCache: true, + }, + { + name: "memcached", + cfg: &Config{ + Cache: "memcached", + Memcached: &memcached.Config{}, + BackgroundCache: &cache.BackgroundConfig{}, + }, + expectedRoles: []cache.Role{cache.RoleBloom, cache.RoleTraceIDIdx}, + expectedCache: true, + }, + { + name: "no cache but cache control", + cfg: &Config{ + Search: &SearchConfig{ + CacheControl: CacheControlConfig{ + Footer: true, + }, + }, + }, + expectedErr: errors.New("no legacy cache configured, but cache_control is enabled. Please use the new top level cache configuration."), + }, + { + name: "memcached + cache control", + cfg: &Config{ + Cache: "memcached", + Memcached: &memcached.Config{}, + BackgroundCache: &cache.BackgroundConfig{}, + Search: &SearchConfig{ + CacheControl: CacheControlConfig{ + Footer: true, + ColumnIndex: true, + OffsetIndex: true, + }, + }, + }, + expectedRoles: []cache.Role{cache.RoleBloom, cache.RoleTraceIDIdx, cache.RoleParquetColumnIdx, cache.RoleParquetFooter, cache.RoleParquetOffsetIdx}, + expectedCache: true, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + prometheus.DefaultRegisterer = prometheus.NewRegistry() // prevent duplicate registration + + cache, actualRoles, actualErr := createLegacyCache(tc.cfg, log.NewNopLogger()) + require.Equal(t, tc.expectedErr, actualErr) + require.Equal(t, tc.expectedRoles, actualRoles) + if tc.expectedCache { + require.NotNil(t, cache) + } else { + require.Nil(t, cache) + } + }) + } +}