Merge branch 'main' into tags_backend

CHANGELOG.md

@@ -1,21 +1,38 @@
 ## main / unreleased
 
-+ [BUGFIX] Change exit code if config is successfully verified [#3174](https://github.com/grafana/tempo/pull/3174) (@am3o @agrib-01)
+* [BUGFIX] Change exit code if config is successfully verified [#3174](https://github.com/grafana/tempo/pull/3174) (@am3o @agrib-01)
+* [BUGFIX] The tempo-cli analyse blocks command no longer fails on compacted blocks [#3183](https://github.com/grafana/tempo/pull/3183) (@stoewer)
 * [ENHANCEMENT] Introduced `AttributePolicyMatch` & `IntrinsicPolicyMatch` structures to match span attributes based on strongly typed values & precompiled regexp [#3025](https://github.com/grafana/tempo/pull/3025) (@andriusluk)
 * [CHANGE] TraceQL/Structural operators performance improvement. [#3088](https://github.com/grafana/tempo/pull/3088) (@joe-elliott)
 * [CHANGE] Merge the processors overrides set through runtime overrides and user-configurable overrides [#3125](https://github.com/grafana/tempo/pull/3125) (@kvrhdn)
 * [CHANGE] Make vParquet3 the default block encoding [#2526](https://github.com/grafana/tempo/pull/3134) (@stoewer)
+* [CHANGE] Major cache refactor to allow multiple role based caches to be configured [#3166](https://github.com/grafana/tempo/pull/3166).
+  **BREAKING CHANGE** Deprecate the following fields. These have all been migrated to a top level "cache:" field.
+  ```
+  storage:
+    trace:
+      cache:
+      search:
+        cache_control:
+      background_cache:
+      memcached:
+      redis:
+  ```
 * [FEATURE] Introduce list_blocks_concurrency on GCS and S3 backends to control backend load and performance. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala)
 * [FEATURE] Add per-tenant compaction window [#3129](https://github.com/grafana/tempo/pull/3129) (@zalegrala)
-* [BUGFIX] Include statusMessage intrinsic attribute in tag search. [#3084](https://github.com/grafana/tempo/pull/3084) (@rcrowe)
 * [ENHANCEMENT] Make the trace ID label name configurable for remote written exemplars [#3074](https://github.com/grafana/tempo/pull/3074)
 * [ENHANCEMENT] Update poller to make use of previous results and reduce backend load. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala)
 * [ENHANCEMENT] Improve TraceQL regex performance in certain queries. [#3139](https://github.com/grafana/tempo/pull/3139) (@joe-elliott)
 * [ENHANCEMENT] Improve TraceQL performance in complex queries. [#3113](https://github.com/grafana/tempo/pull/3113) (@joe-elliott)
-* [BUGFIX] Fix compactor ignore configured S3 headers [#3149](https://github.com/grafana/tempo/pull/3154) (@Batkilin) 
 * [BUGFIX] Prevent building parquet iterators that would loop forever. [#3159](https://github.com/grafana/tempo/pull/3159) (@mapno)
 * [BUGFIX] Sanitize name in mapped dimensions in span-metrics processor [#3171](https://github.com/grafana/tempo/pull/3171) (@mapno)
 
+## v2.3.1 / 2023-11-28
+
+* [BUGFIX] Include statusMessage intrinsic attribute in tag search. [#3084](https://github.com/grafana/tempo/pull/3084) (@rcrowe)
+* [BUGFIX] Fix compactor ignore configured S3 headers [#3149](https://github.com/grafana/tempo/pull/3154) (@Batkilin) 
+* [BUGFIX] Readd session token to s3 credentials. [#3144](https://github.com/grafana/tempo/pull/3144) (@farodin91)
+
 ## v2.3.0 / 2023-10-30
 
 * [CHANGE] Update Go to 1.21 [#2486](https://github.com/grafana/tempo/pull/2829) (@zalegrala)

cmd/tempo-cli/cmd-analyse-block.go

@@ -104,6 +104,9 @@ func (cmd *analyseBlockCmd) Run(ctx *globalOptions) error {
 
 	blockSum, err := processBlock(r, c, cmd.TenantID, cmd.BlockID, time.Hour, 0)
 	if err != nil {
+		if errors.Is(err, backend.ErrDoesNotExist) {
+			return fmt.Errorf("unable to analyze block: block has no block.meta because it was compacted")
+		}
 		return err
 	}
 
@@ -118,27 +121,21 @@ func processBlock(r backend.Reader, _ backend.Compactor, tenantID, blockID strin
 	id := uuid.MustParse(blockID)
 
 	meta, err := r.BlockMeta(context.TODO(), id, tenantID)
-	if err != nil && !errors.Is(err, backend.ErrDoesNotExist) {
+	if err != nil {
 		return nil, err
 	}
-
-	if meta == nil {
-		fmt.Println("Unable to load any meta for block", blockID)
-		return nil, nil
-	}
-
 	if meta.CompactionLevel < minCompactionLvl {
 		return nil, nil
 	}
 
 	var reader io.ReaderAt
 	switch meta.Version {
 	case vparquet.VersionString:
-		reader = vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta.BlockID, meta.TenantID)
+		reader = vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta)
 	case vparquet2.VersionString:
-		reader = vparquet2.NewBackendReaderAt(context.Background(), r, vparquet2.DataFileName, meta.BlockID, meta.TenantID)
+		reader = vparquet2.NewBackendReaderAt(context.Background(), r, vparquet2.DataFileName, meta)
 	case vparquet3.VersionString:
-		reader = vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta.BlockID, meta.TenantID)
+		reader = vparquet3.NewBackendReaderAt(context.Background(), r, vparquet3.DataFileName, meta)
 	default:
 		fmt.Println("Unsupported block version:", meta.Version)
 		return nil, nil
@@ -344,7 +341,7 @@ func printSummary(scope string, max int, summary genericAttrSummary) error {
 	return w.Flush()
 }
 
-func printDedicatedColumnOverridesJsonnet(spanSummary genericAttrSummary, resourceSummary genericAttrSummary) {
+func printDedicatedColumnOverridesJsonnet(spanSummary, resourceSummary genericAttrSummary) {
 	fmt.Println("")
 	fmt.Printf("parquet_dedicated_columns: [\n")
 

cmd/tempo-cli/cmd-analyse-blocks.go

@@ -2,7 +2,12 @@ package main
 
 import (
 	"context"
+	"errors"
 	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/grafana/tempo/tempodb/backend"
 )
 
 type analyseBlocksCmd struct {
@@ -26,17 +31,30 @@ func (cmd *analyseBlocksCmd) Run(ctx *globalOptions) error {
 		return err
 	}
 
-	processedBlocks := 0
+	processedBlocks := map[uuid.UUID]struct{}{}
 	topSpanAttrs, topResourceAttrs := make(map[string]uint64), make(map[string]uint64)
 	totalSpanBytes, totalResourceBytes := uint64(0), uint64(0)
-	for _, block := range blocks {
-		if processedBlocks >= cmd.MaxBlocks {
-			break
+
+	for i := 0; i < len(blocks) && len(processedBlocks) < cmd.MaxBlocks; i++ {
+		block := blocks[i]
+		if _, ok := processedBlocks[block]; ok {
+			continue
 		}
 
 		blockSum, err := processBlock(r, c, cmd.TenantID, block.String(), time.Hour, uint8(cmd.MinCompactionLevel))
 		if err != nil {
-			return err
+			if !errors.Is(err, backend.ErrDoesNotExist) {
+				return err
+			}
+
+			// the block was already compacted and blocks might be outdated: refreshing blocks
+			blocks, _, err = r.Blocks(context.Background(), cmd.TenantID)
+			if err != nil {
+				return err
+			}
+			i = -1
+
+			continue
 		}
 
 		if blockSum == nil {
@@ -53,8 +71,9 @@ func (cmd *analyseBlocksCmd) Run(ctx *globalOptions) error {
 		}
 		totalResourceBytes += blockSum.resourceSummary.totalBytes
 
-		processedBlocks++
+		processedBlocks[block] = struct{}{}
 	}
+
 	// Get top N attributes from map
 	return (&blockSummary{
 		spanSummary: genericAttrSummary{

cmd/tempo-cli/cmd-gen-bloom.go

@@ -120,7 +120,7 @@ func (cmd *bloomCmd) Run(ctx *globalOptions) error {
 	}
 
 	for i := 0; i < len(bloomBytes); i++ {
-		err = w.Write(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, bloomBytes[i], false)
+		err = w.Write(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, bloomBytes[i], nil)
 		if err != nil {
 			fmt.Println("error writing bloom filter to backend", err)
 			return err
@@ -132,7 +132,7 @@ func (cmd *bloomCmd) Run(ctx *globalOptions) error {
 	// verify generated bloom
 	shardedBloomFilter := make([]*willf_bloom.BloomFilter, meta.BloomShardCount)
 	for i := 0; i < int(meta.BloomShardCount); i++ {
-		bloomBytes, err := r.Read(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, false)
+		bloomBytes, err := r.Read(context.TODO(), bloomFilePrefix+strconv.Itoa(i), blockID, cmd.TenantID, nil)
 		if err != nil {
 			fmt.Println("error reading bloom from backend")
 			return nil

cmd/tempo-cli/cmd-gen-index.go

@@ -140,7 +140,7 @@ func (cmd *indexCmd) Run(ctx *globalOptions) error {
 	}
 
 	// write to the local backend
-	err = w.Write(context.TODO(), "index", blockID, cmd.TenantID, indexBytes, false)
+	err = w.Write(context.TODO(), "index", blockID, cmd.TenantID, indexBytes, nil)
 	if err != nil {
 		fmt.Println("error writing index to backend", err)
 		return err

cmd/tempo-cli/cmd-list-column.go

@@ -32,7 +32,7 @@ func (cmd *listColumnCmd) Run(ctx *globalOptions) error {
 		return err
 	}
 
-	rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta.BlockID, meta.TenantID)
+	rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta)
 	pf, err := parquet.OpenFile(rr, int64(meta.Size))
 	if err != nil {
 		return err

cmd/tempo-cli/cmd-view-pq-schema.go

@@ -36,7 +36,7 @@ func (cmd *viewSchemaCmd) Run(ctx *globalOptions) error {
 	fmt.Printf("\n***************     block meta    *********************\n\n\n")
 	fmt.Printf("%+v\n", meta)
 
-	rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta.BlockID, meta.TenantID)
+	rr := vparquet.NewBackendReaderAt(context.Background(), r, vparquet.DataFileName, meta)
 	pf, err := parquet.OpenFile(rr, int64(meta.Size))
 	if err != nil {
 		return err

cmd/tempo/app/app.go

@@ -38,6 +38,7 @@ import (
 	"github.com/grafana/tempo/modules/querier"
 	"github.com/grafana/tempo/modules/storage"
 	"github.com/grafana/tempo/pkg/api"
+	"github.com/grafana/tempo/pkg/cache"
 	"github.com/grafana/tempo/pkg/usagestats"
 	"github.com/grafana/tempo/pkg/util"
 	"github.com/grafana/tempo/pkg/util/log"
@@ -77,6 +78,7 @@ type App struct {
 	generator      *generator.Generator
 	store          storage.Store
 	usageReport    *usagestats.Reporter
+	cacheProvider  cache.Provider
 	MemberlistKV   *memberlist.KVInitService
 
 	HTTPAuthMiddleware       middleware.Interface

cmd/tempo/app/config.go

@@ -10,6 +10,7 @@ import (
 	"github.com/grafana/dskit/server"
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/grafana/tempo/modules/cache"
 	"github.com/grafana/tempo/modules/compactor"
 	"github.com/grafana/tempo/modules/distributor"
 	"github.com/grafana/tempo/modules/frontend"
@@ -53,6 +54,7 @@ type Config struct {
 	Overrides       overrides.Config        `yaml:"overrides,omitempty"`
 	MemberlistKV    memberlist.KVConfig     `yaml:"memberlist,omitempty"`
 	UsageReport     usagestats.Config       `yaml:"usage_report,omitempty"`
+	CacheProvider   cache.Config            `yaml:"cache,omitempty"`
 }
 
 func newDefaultConfig() *Config {
@@ -209,6 +211,10 @@ func (c *Config) CheckConfig() []ConfigWarning {
 		warnings = append(warnings, warnNativeAWSAuthEnabled)
 	}
 
+	if c.StorageConfig.Trace.Cache != "" {
+		warnings = append(warnings, warnConfiguredLegacyCache)
+	}
+
 	return warnings
 }
 
@@ -275,6 +281,11 @@ var (
 		Message: "c.StorageConfig.Trace.S3.NativeAWSAuthEnabled is deprecated and will be removed in a future release.",
 		Explain: "This setting is no longer necessary and will be ignored.",
 	}
+
+	warnConfiguredLegacyCache = ConfigWarning{
+		Message: "c.StorageConfig.Trace.Cache is deprecated and will be removed in a future release.",
+		Explain: "Please migrate to the top level cache settings config.",
+	}
 )
 
 func newV2Warning(setting string) ConfigWarning {

cmd/tempo/app/config_test.go

@@ -33,6 +33,7 @@ func TestConfig_CheckConfig(t *testing.T) {
 				Target: MetricsGenerator,
 				StorageConfig: storage.Config{
 					Trace: tempodb.Config{
+						Cache:         "supercache",
 						Backend:       backend.S3,
 						BlocklistPoll: time.Minute,
 						Block: &common.BlockConfig{
@@ -57,6 +58,7 @@ func TestConfig_CheckConfig(t *testing.T) {
 				warnBlocklistPollConcurrency,
 				warnLogReceivedTraces,
 				warnNativeAWSAuthEnabled,
+				warnConfiguredLegacyCache,
 			},
 		},
 		{

cmd/tempo/app/modules.go

@@ -21,6 +21,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 
+	"github.com/grafana/tempo/modules/cache"
 	"github.com/grafana/tempo/modules/compactor"
 	"github.com/grafana/tempo/modules/distributor"
 	"github.com/grafana/tempo/modules/frontend"
@@ -54,6 +55,8 @@ const (
 	UsageReport    string = "usage-report"
 	Overrides      string = "overrides"
 	OverridesAPI   string = "overrides-api"
+	CacheProvider  string = "cache-provider"
+
 	// rings
 	IngesterRing          string = "ring"
 	SecondaryIngesterRing string = "secondary-ring"
@@ -430,7 +433,7 @@ func (t *App) initCompactor() (services.Service, error) {
 }
 
 func (t *App) initStore() (services.Service, error) {
-	store, err := tempo_storage.NewStore(t.cfg.StorageConfig, log.Logger)
+	store, err := tempo_storage.NewStore(t.cfg.StorageConfig, t.cacheProvider, log.Logger)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create store: %w", err)
 	}
@@ -510,6 +513,16 @@ func (t *App) initUsageReport() (services.Service, error) {
 	return ur, nil
 }
 
+func (t *App) initCacheProvider() (services.Service, error) {
+	c, err := cache.NewProvider(&t.cfg.CacheProvider, util_log.Logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create cache provider: %w", err)
+	}
+
+	t.cacheProvider = c
+	return c, nil
+}
+
 func (t *App) setupModuleManager() error {
 	mm := modules.NewManager(log.Logger)
 
@@ -523,6 +536,7 @@ func (t *App) setupModuleManager() error {
 	mm.RegisterModule(Overrides, t.initOverrides, modules.UserInvisibleModule)
 	mm.RegisterModule(OverridesAPI, t.initOverridesAPI)
 	mm.RegisterModule(UsageReport, t.initUsageReport)
+	mm.RegisterModule(CacheProvider, t.initCacheProvider, modules.UserInvisibleModule)
 	mm.RegisterModule(IngesterRing, t.initIngesterRing, modules.UserInvisibleModule)
 	mm.RegisterModule(MetricsGeneratorRing, t.initGeneratorRing, modules.UserInvisibleModule)
 	mm.RegisterModule(SecondaryIngesterRing, t.initSecondaryIngesterRing, modules.UserInvisibleModule)
@@ -540,8 +554,9 @@ func (t *App) setupModuleManager() error {
 	mm.RegisterModule(ScalableSingleBinary, nil)
 
 	deps := map[string][]string{
-		// Store:          nil,
 		// InternalServer: nil,
+		// CacheProvider:  nil,
+		Store:                 {CacheProvider},
 		Server:                {InternalServer},
 		Overrides:             {Server},
 		OverridesAPI:          {Server, Overrides},

docs/sources/tempo/_index.md

@@ -13,6 +13,8 @@ cascade:
 
 Grafana Tempo is an open source, easy-to-use, and high-volume distributed tracing backend. Tempo is cost-efficient, and only requires an object storage to operate. Tempo is deeply integrated with Grafana, Mimir, Prometheus, and Loki. You can use Tempo with open-source tracing protocols, including Jaeger, Zipkin, or OpenTelemetry.
 
+Tempo implements [TraceQL](/docs/tempo/latest/traceql), a traces-first query language inspired by LogQL and PromQL. This query language allows users to very precisely and easily select spans and jump directly to the spans fulfilling the specified conditions.
+
 Tempo integrates well with a number of existing open source tools:
 
 - **Grafana** ships with native support for Tempo using the built-in [Tempo data source](/docs/grafana/latest/datasources/tempo/).