Merge branch 'main' into main

.github/workflows/tests.yml

@@ -14,8 +14,8 @@ jobs:
     strategy:
       matrix:
         go:
-          - ^1.19
-          - ^1.20
+          - ^1.21
+          - ^1.22
           - ^1
     steps:
 

CHANGES.md

@@ -17,6 +17,31 @@ Changes
 
 CHANGELOG
 ---------
+**0.16.1**
+ - [Build] Update build version of golang to 1.21.0
+ - [Improvement] Better error messages
+ - [Improvement] Implement /metrics/expand API
+ - [Improvement] Implement aliasQuery
+ - [Fix] aliasByNode/Tag should not disacrd tags anymore
+ - [Fix] VicotriaMetrics cluster now should support autocomplete for tags
+ - [Fix] ConstantSeries should now work as in graphite-web
+ - [Fix] Fix logic of hitcount function (should behave like in graphite-web)
+ - [Fix] alignToInterval should work well in hitcount function
+ - [Fix] smartSummarize should handle stop time properly
+ - [Fix] smartSummarize should now handle alignTo properly
+ - [Fix] smartSummarize should handle consolidation functiosn better
+ - [Fix] summarize should work as in graphite-web for functions that have only NaN values
+ - [Fix] toUpper and toLower now should work correctly with function names
+ - [Fix] Skip all whitespace during expression parsing
+ - [Fix] time units now case insensitive
+ - [Fix] use specified bootstrapInterval to adjust start time in holtWinters* class of functions
+ - [Fix] groupByNode should return exepcted amount of series
+ - [Fix] multiple fixes for exponentialMovingAverage
+ - [Fix] holtWinters supports seasonality argument
+ - [Fix] fix holtWintersAbberation behavior
+ - [Fix] fix behavior of Below function
+ - [Fix] fix sin and exp functions (description was not accurate or correct)
+
 **0.16.0.1**
  - [Build] Fix automation that builds docker images
  - [Build] Add rockylinux-9 packages (RHEL 9)

README.md

@@ -119,9 +119,9 @@ Golang compatibility matrix:
 | 1.12           | 0.12.6                           |
 | 1.16 / 1.17    | 0.15.6                           |
 | 1.18           | 0.16.0-patch2                    |
-| 1.19           | current                          |
+| 1.20           | 0.16.1                           |
 
-Overall rule of thumb is that carbonapi supports last 2 major go versions. E.x. at this moment Go 1.19 and 1.18 are supported.
+Overall rule of thumb is that carbonapi supports last 2 major go versions. E.x. at this moment Go 1.22 and 1.21 are supported.
 
 You can verify current versions that are being tested in [CI Configuration](https://github.com/go-graphite/carbonapi/blob/main/.github/workflows/tests.yml#L14).
 
@@ -156,18 +156,19 @@ Internal Metrics
 ----------------------------------
 The internal metrics are configured inside the [graphite](https://github.com/go-graphite/carbonapi/blob/main/doc/configuration.md#graphite) subsection and sent to your destinated host on an specified interval. The metrics are:
 
-cache_items - if caching is enabled, this metric will contain many metrics are stored in cache
-cache_size - configured query cache size in bytes
-request_cache_hits - how many requests were served from cache. (this is for requests to /render endpoint)
-request_cache_misses - how many requests were not in cache. (this is for requests to /render endpoint)
-request_cache_overhead_ns - how much time in ns it took to talk to cache (that is useful to assess if cache actually helps you in terms of latency) (this is for requests to /render endpoint)
-find_requests - requests server by endpoint /metrics/find
-requests - requests served by endpoint /render
-requests_in_XX_to_XX - request response times in percentiles
-timeouts - number of timeouts while fetching from backend
-backend_cache_hits - how many requests were not read from backend
-backend_cache_misses - how many requests were not found in the backend
-
+| Metric Name | Description |
+| ----------- | ----------- |
+| cache_items | if caching is enabled, this metric will contain many metrics are stored in cache |
+| cache_size | configured query cache size in bytes |
+| request_cache_hits | how many requests were served from cache. (this is for requests to /render endpoint) |
+| request_cache_misses | how many requests were not in cache. (this is for requests to /render endpoint) |
+| request_cache_overhead_ns | how much time in ns it took to talk to cache (that is useful to assess if cache actually helps you in terms of latency) (this is for  |requests to /render endpoint)
+| find_requests | requests server by endpoint /metrics/find |
+| requests | requests served by endpoint /render |
+| requests_in_XX_to_XX | request response times in percentiles |
+| timeouts | number of timeouts while fetching from backend |
+| backend_cache_hits | how many requests were not read from backend |
+| backend_cache_misses | how many requests were not found in the backend |
 
 OSX Build Notes
 ---------------

cmd/carbonapi/config/config.go

@@ -107,7 +107,8 @@ type ConfigType struct {
 	TruncateTimeMap map[time.Duration]time.Duration `mapstructure:"truncateTime"`
 	TruncateTime    []DurationTruncate              `mapstructure:"-" json:"-"` // produce from TruncateTimeMap and sort in reverse order
 
-	CombineMultipleTargetsInOne bool `mapstructure:"combineMultipleTargetsInOne"`
+	MaxQueryLength              uint64 `mapstructure:"maxQueryLength"`
+	CombineMultipleTargetsInOne bool   `mapstructure:"combineMultipleTargetsInOne"`
 
 	ResponseCache cache.BytesCache `mapstructure:"-" json:"-"`
 	BackendCache  cache.BytesCache `mapstructure:"-" json:"-"`

cmd/carbonapi/config/init.go

@@ -330,7 +330,7 @@ func createCache(logger *zap.Logger, cacheName string, cacheConfig *CacheConfig)
 	}
 }
 
-func SetUpViper(logger *zap.Logger, configPath *string, viperPrefix string) {
+func SetUpViper(logger *zap.Logger, configPath *string, exactConfig bool, viperPrefix string) {
 	if *configPath != "" {
 		b, err := os.ReadFile(*configPath)
 		if err != nil {
@@ -386,24 +386,30 @@ func SetUpViper(logger *zap.Logger, configPath *string, viperPrefix string) {
 	viper.SetDefault("upstreams.internalRoutingCache", "600s")
 	viper.SetDefault("upstreams.buckets", 10)
 	viper.SetDefault("upstreams.sumBuckets", false)
-	viper.SetDefault("upstreams.bucketsWeight", []int64{})
-	viper.SetDefault("upstreams.bucketsNames", []string{})
+	viper.SetDefault("upstreams.bucketsWidth", []int64{})
+	viper.SetDefault("upstreams.bucketsLabels", []string{})
 	viper.SetDefault("upstreams.slowLogThreshold", "1s")
-	viper.SetDefault("upstreams.timeouts.global", "10s")
-	viper.SetDefault("upstreams.timeouts.afterStarted", "2s")
+	viper.SetDefault("upstreams.timeouts.find", "2s")
+	viper.SetDefault("upstreams.timeouts.render", "10s")
 	viper.SetDefault("upstreams.timeouts.connect", "200ms")
-	viper.SetDefault("upstreams.concurrencyLimit", 0)
+	viper.SetDefault("upstreams.concurrencyLimitPerServer", 0)
 	viper.SetDefault("upstreams.keepAliveInterval", "30s")
 	viper.SetDefault("upstreams.maxIdleConnsPerHost", 100)
 	viper.SetDefault("upstreams.scaleToCommonStep", true)
-	viper.SetDefault("upstreams.graphite09compat", false)
+	viper.SetDefault("graphite09compat", false)
 	viper.SetDefault("expireDelaySec", 600)
 	viper.SetDefault("useCachingDNSResolver", false)
 	viper.SetDefault("logger", map[string]string{})
 	viper.SetDefault("combineMultipleTargetsInOne", false)
 	viper.AutomaticEnv()
 
-	err := viper.Unmarshal(&Config)
+	var err error
+	if exactConfig {
+		err = viper.UnmarshalExact(&Config)
+	} else {
+		err = viper.Unmarshal(&Config)
+	}
+
 	if err != nil {
 		logger.Fatal("failed to parse config",
 			zap.Error(err),

cmd/carbonapi/http/expand_handler.go

@@ -76,6 +76,12 @@ func expandHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if queryLengthLimitExceeded(query, config.Config.MaxQueryLength) {
+		setError(w, &accessLogDetails, "query length limit exceeded", http.StatusBadRequest, uid.String())
+		logAsError = true
+		return
+	}
+
 	var pv3Request pbv3.MultiGlobRequest
 	pv3Request.Metrics = query
 	pv3Request.StartTime = from64

cmd/carbonapi/http/find_handlers.go

@@ -221,6 +221,12 @@ func findHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if queryLengthLimitExceeded(query, config.Config.MaxQueryLength) {
+		setError(w, &accessLogDetails, "query length limit exceeded", http.StatusBadRequest, uid.String())
+		logAsError = true
+		return
+	}
+
 	if format == completerFormat {
 		var replacer = strings.NewReplacer("/", ".")
 		for i := range query {

cmd/carbonapi/http/helper.go

@@ -282,3 +282,23 @@ func timestampTruncate(ts int64, duration time.Duration, durations []config.Dura
 	}
 	return ts
 }
+
+func setError(w http.ResponseWriter, accessLogDetails *carbonapipb.AccessLogDetails, msg string, status int, carbonapiUUID string) {
+	w.Header().Set(ctxHeaderUUID, carbonapiUUID)
+	http.Error(w, http.StatusText(status)+": "+msg, status)
+	accessLogDetails.Reason = msg
+	accessLogDetails.HTTPCode = int32(status)
+}
+
+func queryLengthLimitExceeded(query []string, maxLength uint64) bool {
+	if maxLength > 0 {
+		var queryLengthSum uint64 = 0
+		for _, q := range query {
+			queryLengthSum += uint64(len(q))
+		}
+		if queryLengthSum > maxLength {
+			return true
+		}
+	}
+	return false
+}

cmd/carbonapi/http/helper_test.go

@@ -2,6 +2,7 @@ package http
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -68,3 +69,55 @@ func Test_timestampTruncate(t *testing.T) {
 		})
 	}
 }
+
+func Test_queryLengthLimitExceeded(t *testing.T) {
+	tests := []struct {
+		query     []string
+		maxLength uint64
+		want      bool
+	}{
+		{
+			query:     []string{"a.b.c.d.e", "a.a.a.a.a.a.a.a.a.a.a.a"},
+			maxLength: 20,
+			want:      true,
+		},
+		{
+			query:     []string{"a.b.c", "a.b.d"},
+			maxLength: 10,
+			want:      false,
+		},
+		{
+			query:     []string{"a.b.c", "a.b.d"},
+			maxLength: 9,
+			want:      true,
+		},
+		{
+			query:     []string{"a.b.c.d.e", "a.a.a.a.a.a.a.a.a.a.a.a"},
+			maxLength: 0,
+			want:      false,
+		},
+		{
+			query:     []string{"a.b.b.c.*", "a.d.e", "a.b.c", "a.b.e", "a.a.a.b", "a.f.s.x.w.g"},
+			maxLength: 30,
+			want:      true,
+		},
+		{
+			query:     []string{"a.b.c.d.e", "a.b.c.d.f", "b.a.*"},
+			maxLength: 10000,
+			want:      false,
+		},
+		{
+			query:     []string{},
+			maxLength: 0,
+			want:      false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("queryLengthLimitExceeded([%s], %d) -> %t", strings.Join(tt.query, ", "), tt.maxLength, tt.want), func(t *testing.T) {
+			if got := queryLengthLimitExceeded(tt.query, tt.maxLength); got != tt.want {
+				t.Errorf("queryLengthLimitExceeded() = %t, want %t", got, tt.want)
+			}
+		})
+	}
+}

cmd/carbonapi/http/main_test.go

@@ -114,7 +114,7 @@ func init() {
 	logger := zapwriter.Logger("main")
 
 	cfgFile := ""
-	config.SetUpViper(logger, &cfgFile, "CARBONAPI_")
+	config.SetUpViper(logger, &cfgFile, false, "CARBONAPI_")
 	config.Config.Upstreams.Backends = []string{"dummy"}
 	config.SetUpConfigUpstreams(logger)
 	config.SetUpConfig(logger, "(test)")

cmd/carbonapi/http/render_handler.go

@@ -42,13 +42,6 @@ func cleanupParams(r *http.Request) {
 	r.Form.Del("_t") // Used by jquery.graphite.js
 }
 
-func setError(w http.ResponseWriter, accessLogDetails *carbonapipb.AccessLogDetails, msg string, status int, carbonapiUUID string) {
-	w.Header().Set(ctxHeaderUUID, carbonapiUUID)
-	http.Error(w, http.StatusText(status)+": "+msg, status)
-	accessLogDetails.Reason = msg
-	accessLogDetails.HTTPCode = int32(status)
-}
-
 func getCacheTimeout(logger *zap.Logger, r *http.Request, now32, until32 int64, duration time.Duration, cacheConfig *config.CacheConfig) int32 {
 	if tstr := r.FormValue("cacheTimeout"); tstr != "" {
 		t, err := strconv.Atoi(tstr)
@@ -232,6 +225,12 @@ func renderHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if queryLengthLimitExceeded(targets, config.Config.MaxQueryLength) {
+		setError(w, accessLogDetails, "total target length limit exceeded", http.StatusBadRequest, uid.String())
+		logAsError = true
+		return
+	}
+
 	if useCache {
 		tc := time.Now()
 		response, err := config.Config.ResponseCache.Get(responseCacheKey)

cmd/carbonapi/http/tags_handler.go

@@ -80,6 +80,12 @@ func tagHandler(w http.ResponseWriter, r *http.Request) {
 	q.Del("pretty")
 	rawQuery := q.Encode()
 
+	if queryLengthLimitExceeded(r.Form["query"], config.Config.MaxQueryLength) {
+		setError(w, accessLogDetails, "query length limit exceeded", http.StatusBadRequest, uuid.String())
+		logAsError = true
+		return
+	}
+
 	// TODO(civil): Implement caching
 	var res []string
 	if strings.HasSuffix(r.URL.Path, "tags") || strings.HasSuffix(r.URL.Path, "tags/") {

cmd/carbonapi/main.go

@@ -35,6 +35,7 @@ func main() {
 
 	configPath := flag.String("config", "", "Path to the `config file`.")
 	checkConfig := flag.Bool("check-config", false, "Check config file and exit.")
+	exactConfig := flag.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.")
 	envPrefix := flag.String("envprefix", "CARBONAPI", "Prefix for environment variables override")
 	if *envPrefix == "(empty)" {
 		*envPrefix = ""
@@ -43,7 +44,7 @@ func main() {
 		logger.Warn("empty prefix is not recommended due to possible collisions with OS environment variables")
 	}
 	flag.Parse()
-	config.SetUpViper(logger, configPath, *envPrefix)
+	config.SetUpViper(logger, configPath, *exactConfig, *envPrefix)
 	if *checkConfig {
 		os.Exit(0)
 	}

cmd/mockbackend/testcases/pr817/carbonapi.yaml

@@ -0,0 +1,61 @@
+listen: "localhost:8081"
+expvar:
+  enabled: true
+  pprofEnabled: false
+  listen: ""
+concurency: 1000
+notFoundStatusCode: 200
+cache:
+   type: "mem"
+   size_mb: 0
+   defaultTimeoutSec: 60
+cpus: 0
+tz: ""
+maxBatchSize: 500
+maxQueryLength: 20
+combineMultipleTargetsInOne: true
+graphite:
+    host: ""
+    interval: "60s"
+    prefix: "carbon.api"
+    pattern: "{prefix}.{fqdn}"
+idleConnections: 10
+pidFile: ""
+upstreams:
+    buckets: 10
+    timeouts:
+        find: "2s"
+        render: "10s"
+        connect: "200ms"
+    concurrencyLimitPerServer: 100
+    keepAliveInterval: "30s"
+    maxIdleConnsPerHost: 100
+    doMultipleRequestsIfSplit: true
+    backendsv2:
+        backends:
+          -
+            groupName: "mock-001"
+            protocol: "auto"
+            lbMethod: "all"
+            maxTries: 3
+            maxBatchSize: 500
+            keepAliveInterval: "10s"
+            concurrencyLimit: 0
+            forceAttemptHTTP2: true
+            maxIdleConnsPerHost: 1000
+            doMultipleRequestsIfSplit: true
+            timeouts:
+                find: "15s"
+                render: "50s"
+                connect: "200ms"
+            servers:
+                - "http://127.0.0.1:9070"
+    graphite09compat: false
+expireDelaySec: 10
+logger:
+    - logger: ""
+      file: "stderr"
+      level: "debug"
+      encoding: "console"
+      encodingTime: "iso8601"
+      encodingDuration: "seconds"