feat(monitor): auto create monitor default ttl config for orgs

cmd/monitor/monitor/bootstrap.yaml

@@ -125,7 +125,7 @@ elasticsearch.index.loader@log:
         - "<org>-{number}"
         - "<org>.<key>-{number}"
 storage-retention-strategy@log:
-  _enable: ${QUERY_LOG_FROM_ES_ENABLE:false}
+  _enable: true
   default_ttl: "${LOG_TTL:168h}"
   load_from_database: true
   ttl_reload_interval: "3m"
@@ -251,7 +251,7 @@ elasticsearch.index.loader@metric:
         - "<metric>-<namespace>.<key>"
         - "<metric>-<namespace>"
 storage-retention-strategy@metric:
-  _enable: ${QUERY_METRIC_FROM_ES_ENABLE:false}
+  _enable: true
   default_ttl: "${METRIC_TTL:168h}"
   load_from_database: true
   ttl_reload_interval: "3m"
@@ -423,6 +423,8 @@ org-apis:
   offline_sleep: "${DASHBOARD_OFFLINE_SLEEP:3m}"
 runtime-apis:
 project-apis:
+etcd-election@monitor-config-creator:
+  root_path: "/erda/component-leader/monitor/action/sync-config"
 erda.core.monitor.settings:
 erda.core.monitor.dataview:
 dataview-v1:

internal/tools/monitor/core/settings/provider.go

@@ -15,6 +15,7 @@
 package settings
 
 import (
+	"context"
 	"net/http"
 	"time"
 
@@ -25,23 +26,31 @@ import (
 	"github.com/erda-project/erda-infra/pkg/transport"
 	transhttp "github.com/erda-project/erda-infra/pkg/transport/http"
 	"github.com/erda-project/erda-infra/pkg/transport/http/encoding"
+	election "github.com/erda-project/erda-infra/providers/etcd-election"
 	"github.com/erda-project/erda-infra/providers/i18n"
 	"github.com/erda-project/erda-proto-go/core/monitor/settings/pb"
 	"github.com/erda-project/erda/bundle"
 	"github.com/erda-project/erda/internal/core/org"
+	"github.com/erda-project/erda/internal/tools/monitor/core/settings/retention-strategy"
+	"github.com/erda-project/erda/internal/tools/pipeline/providers/reconciler/rutil"
 	"github.com/erda-project/erda/pkg/common/apis"
 	"github.com/erda-project/erda/pkg/http/httpclient"
 )
 
-type config struct{}
+type config struct {
+	SyncMonitorTypes []string `file:"sync_monitor_types" default:"logs,metrics"`
+}
 
 // +provider
 type provider struct {
 	Cfg             *config
 	Log             logs.Logger
-	Register        transport.Register `autowired:"service-register" optional:"true"`
-	DB              *gorm.DB           `autowired:"mysql-client"`
-	Trans           i18n.Translator    `autowired:"i18n" translator:"settings"`
+	Register        transport.Register  `autowired:"service-register" optional:"true"`
+	DB              *gorm.DB            `autowired:"mysql-client"`
+	Trans           i18n.Translator     `autowired:"i18n" translator:"settings"`
+	Election        election.Interface  `autowired:"etcd-election@monitor-config-creator"`
+	LogRetention    retention.Interface `autowired:"storage-retention-strategy@log"`
+	MetricRetention retention.Interface `autowired:"storage-retention-strategy@metric"`
 	settingsService *settingsService
 	Org             org.ClientInterface
 }
@@ -111,6 +120,18 @@ func (p *provider) Provide(ctx servicehub.DependencyContext, args ...interface{}
 	return p
 }
 
+func (p *provider) Run(ctx context.Context) error {
+	p.Election.OnLeader(func(ctx context.Context) {
+		rutil.ContinueWorking(ctx, p.Log, func(ctx context.Context) rutil.WaitDuration {
+			if err := p.syncCreateOrgMonitorConfig(); err != nil {
+				p.Log.Errorf("failed to syncCreateOrgMonitorConfig: %v", err)
+			}
+			return rutil.ContinueWorkingWithCustomInterval(time.Minute * 10)
+		})
+	})
+	return nil
+}
+
 func init() {
 	servicehub.Register("erda.core.monitor.settings", &servicehub.Spec{
 		Services:             pb.ServiceNames(),

internal/tools/monitor/core/settings/sync_config.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2021 Terminus, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package settings
+
+import (
+	"context"
+	"fmt"
+
+	"google.golang.org/protobuf/types/known/structpb"
+
+	"github.com/erda-project/erda-proto-go/core/monitor/settings/pb"
+	orgpb "github.com/erda-project/erda-proto-go/core/org/pb"
+	"github.com/erda-project/erda/internal/tools/monitor/oap/collector/lib"
+	"github.com/erda-project/erda/pkg/common/apis"
+	"github.com/erda-project/erda/pkg/discover"
+)
+
+func (p *provider) syncCreateOrgMonitorConfig() error {
+	allOrgs, err := p.Org.ListOrg(apis.WithInternalClientContext(context.Background(), discover.SvcMonitor), &orgpb.ListOrgRequest{
+		PageSize: 9999,
+		PageNo:   1,
+	})
+	if err != nil {
+		return err
+	}
+	defCfg := p.settingsService.getDefaultConfig(apis.Language(context.Background()), "")
+	for _, org := range allOrgs.List {
+		defSetting := &pb.PutSettingsRequest{
+			Data: make(map[string]*pb.ConfigGroups),
+		}
+		for ns, cfg := range defCfg {
+			if ns == "general" {
+				continue
+			}
+			for _, monitorType := range p.Cfg.SyncMonitorTypes {
+				isEmpty := p.isEmptyConfig(monitorType, ns, org)
+				if !isEmpty {
+					continue
+				}
+				monitorTTL := p.getTTL(monitorType, cfg)
+				if monitorTTL == nil {
+					continue
+				}
+				if defSetting.Data[ns] == nil {
+					defSetting.Data[ns] = &pb.ConfigGroups{
+						Groups: []*pb.ConfigGroup{
+							{
+								Key: "monitor",
+								Items: []*pb.ConfigItem{
+									{
+										Key:   fmt.Sprintf("%s_ttl", monitorType),
+										Value: monitorTTL,
+									},
+								},
+							},
+						},
+					}
+				} else {
+					defSetting.Data[ns].Groups[0].Items = append(defSetting.Data[ns].Groups[0].Items, &pb.ConfigItem{
+						Key:   fmt.Sprintf("%s_ttl", monitorType),
+						Value: monitorTTL,
+					})
+				}
+			}
+		}
+		if len(defSetting.Data) > 0 {
+			defSetting.OrgID = int64(org.ID)
+			if _, err := p.settingsService.PutSettings(apis.WithInternalClientContext(context.Background(), discover.SvcMonitor), defSetting); err != nil {
+				p.Log.Errorf("failed to create default monitor config for org: %s, err: %v", org.Name, err)
+			}
+		}
+	}
+	return nil
+}
+
+func (p *provider) isEmptyConfig(monitorType string, ns string, org *orgpb.Org) bool {
+	var key string
+	switch monitorType {
+	case "logs":
+		tags := map[string]string{
+			lib.DiceOrgNameKey: org.Name,
+			lib.DiceWorkspace:  ns,
+		}
+		key = p.LogRetention.GetConfigKey("container", tags)
+	case "metrics":
+		tags := map[string]string{
+			lib.OrgNameKey:    org.Name,
+			lib.DiceWorkspace: ns,
+		}
+		key = p.MetricRetention.GetConfigKey("docker_container_summary", tags)
+	default:
+		return false
+	}
+	return len(key) == 0
+}
+
+func (p *provider) getTTL(monitorType string, cfg map[string]map[string]*pb.ConfigItem) *structpb.Value {
+	if cfg == nil {
+		return nil
+	}
+	monitorCfg := cfg["monitor"]
+	if monitorCfg == nil {
+		return nil
+	}
+	ttl := monitorCfg[fmt.Sprintf("%s_ttl", monitorType)]
+	return ttl.Value
+}
+
+func (p *provider) autoCreateOrgMonitorConfig(orgID uint64) error {
+	p.settingsService.PutSettings(context.Background(), &pb.PutSettingsRequest{
+		OrgID: int64(orgID),
+	})
+	return nil
+}

internal/tools/monitor/oap/collector/lib/const.go

@@ -15,5 +15,7 @@
 package lib
 
 const (
-	OrgNameKey = "org_name"
+	OrgNameKey     = "org_name"
+	DiceOrgNameKey = "dice_org_name"
+	DiceWorkspace  = "dice_workspace"
 )