From 14f3794bc1fd42ca39c2bd8f0451aa954d56d6d4 Mon Sep 17 00:00:00 2001 From: joey Date: Wed, 28 Feb 2024 16:41:54 +0800 Subject: [PATCH] feat auto create monitor default ttl config for orgs --- cmd/monitor/monitor/bootstrap.yaml | 6 +- .../tools/monitor/core/settings/provider.go | 29 +++- .../monitor/core/settings/sync_config.go | 126 ++++++++++++++++++ .../tools/monitor/oap/collector/lib/const.go | 4 +- 4 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 internal/tools/monitor/core/settings/sync_config.go diff --git a/cmd/monitor/monitor/bootstrap.yaml b/cmd/monitor/monitor/bootstrap.yaml index 6eddde64ec8..4aed80fbf3c 100644 --- a/cmd/monitor/monitor/bootstrap.yaml +++ b/cmd/monitor/monitor/bootstrap.yaml @@ -125,7 +125,7 @@ elasticsearch.index.loader@log: - "-{number}" - ".-{number}" storage-retention-strategy@log: - _enable: ${QUERY_LOG_FROM_ES_ENABLE:false} + _enable: true default_ttl: "${LOG_TTL:168h}" load_from_database: true ttl_reload_interval: "3m" @@ -251,7 +251,7 @@ elasticsearch.index.loader@metric: - "-." - "-" storage-retention-strategy@metric: - _enable: ${QUERY_METRIC_FROM_ES_ENABLE:false} + _enable: true default_ttl: "${METRIC_TTL:168h}" load_from_database: true ttl_reload_interval: "3m" @@ -423,6 +423,8 @@ org-apis: offline_sleep: "${DASHBOARD_OFFLINE_SLEEP:3m}" runtime-apis: project-apis: +etcd-election@monitor-config-creator: + root_path: "/erda/component-leader/monitor/action/sync-config" erda.core.monitor.settings: erda.core.monitor.dataview: dataview-v1: diff --git a/internal/tools/monitor/core/settings/provider.go b/internal/tools/monitor/core/settings/provider.go index 8041689a272..e269c099f7a 100644 --- a/internal/tools/monitor/core/settings/provider.go +++ b/internal/tools/monitor/core/settings/provider.go @@ -15,6 +15,7 @@ package settings import ( + "context" "net/http" "time" @@ -25,23 +26,31 @@ import ( "github.com/erda-project/erda-infra/pkg/transport" transhttp "github.com/erda-project/erda-infra/pkg/transport/http" "github.com/erda-project/erda-infra/pkg/transport/http/encoding" + election "github.com/erda-project/erda-infra/providers/etcd-election" "github.com/erda-project/erda-infra/providers/i18n" "github.com/erda-project/erda-proto-go/core/monitor/settings/pb" "github.com/erda-project/erda/bundle" "github.com/erda-project/erda/internal/core/org" + "github.com/erda-project/erda/internal/tools/monitor/core/settings/retention-strategy" + "github.com/erda-project/erda/internal/tools/pipeline/providers/reconciler/rutil" "github.com/erda-project/erda/pkg/common/apis" "github.com/erda-project/erda/pkg/http/httpclient" ) -type config struct{} +type config struct { + SyncMonitorTypes []string `file:"sync_monitor_types" default:"logs,metrics"` +} // +provider type provider struct { Cfg *config Log logs.Logger - Register transport.Register `autowired:"service-register" optional:"true"` - DB *gorm.DB `autowired:"mysql-client"` - Trans i18n.Translator `autowired:"i18n" translator:"settings"` + Register transport.Register `autowired:"service-register" optional:"true"` + DB *gorm.DB `autowired:"mysql-client"` + Trans i18n.Translator `autowired:"i18n" translator:"settings"` + Election election.Interface `autowired:"etcd-election@monitor-config-creator"` + LogRetention retention.Interface `autowired:"storage-retention-strategy@log"` + MetricRetention retention.Interface `autowired:"storage-retention-strategy@metric"` settingsService *settingsService Org org.ClientInterface } @@ -111,6 +120,18 @@ func (p *provider) Provide(ctx servicehub.DependencyContext, args ...interface{} return p } +func (p *provider) Run(ctx context.Context) error { + p.Election.OnLeader(func(ctx context.Context) { + rutil.ContinueWorking(ctx, p.Log, func(ctx context.Context) rutil.WaitDuration { + if err := p.syncCreateOrgMonitorConfig(); err != nil { + p.Log.Errorf("failed to syncCreateOrgMonitorConfig: %v", err) + } + return rutil.ContinueWorkingWithCustomInterval(time.Minute * 10) + }) + }) + return nil +} + func init() { servicehub.Register("erda.core.monitor.settings", &servicehub.Spec{ Services: pb.ServiceNames(), diff --git a/internal/tools/monitor/core/settings/sync_config.go b/internal/tools/monitor/core/settings/sync_config.go new file mode 100644 index 00000000000..f65f84e3542 --- /dev/null +++ b/internal/tools/monitor/core/settings/sync_config.go @@ -0,0 +1,126 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package settings + +import ( + "context" + "fmt" + + "google.golang.org/protobuf/types/known/structpb" + + "github.com/erda-project/erda-proto-go/core/monitor/settings/pb" + orgpb "github.com/erda-project/erda-proto-go/core/org/pb" + "github.com/erda-project/erda/internal/tools/monitor/oap/collector/lib" + "github.com/erda-project/erda/pkg/common/apis" + "github.com/erda-project/erda/pkg/discover" +) + +func (p *provider) syncCreateOrgMonitorConfig() error { + allOrgs, err := p.Org.ListOrg(apis.WithInternalClientContext(context.Background(), discover.SvcMonitor), &orgpb.ListOrgRequest{ + PageSize: 9999, + PageNo: 1, + }) + if err != nil { + return err + } + defCfg := p.settingsService.getDefaultConfig(apis.Language(context.Background()), "") + for _, org := range allOrgs.List { + defSetting := &pb.PutSettingsRequest{ + Data: make(map[string]*pb.ConfigGroups), + } + for ns, cfg := range defCfg { + if ns == "general" { + continue + } + for _, monitorType := range p.Cfg.SyncMonitorTypes { + isEmpty := p.isEmptyConfig(monitorType, ns, org) + if !isEmpty { + continue + } + monitorTTL := p.getTTL(monitorType, cfg) + if monitorTTL == nil { + continue + } + if defSetting.Data[ns] == nil { + defSetting.Data[ns] = &pb.ConfigGroups{ + Groups: []*pb.ConfigGroup{ + { + Key: "monitor", + Items: []*pb.ConfigItem{ + { + Key: fmt.Sprintf("%s_ttl", monitorType), + Value: monitorTTL, + }, + }, + }, + }, + } + } else { + defSetting.Data[ns].Groups[0].Items = append(defSetting.Data[ns].Groups[0].Items, &pb.ConfigItem{ + Key: fmt.Sprintf("%s_ttl", monitorType), + Value: monitorTTL, + }) + } + } + } + if len(defSetting.Data) > 0 { + defSetting.OrgID = int64(org.ID) + if _, err := p.settingsService.PutSettings(apis.WithInternalClientContext(context.Background(), discover.SvcMonitor), defSetting); err != nil { + p.Log.Errorf("failed to create default monitor config for org: %s, err: %v", org.Name, err) + } + } + } + return nil +} + +func (p *provider) isEmptyConfig(monitorType string, ns string, org *orgpb.Org) bool { + var key string + switch monitorType { + case "logs": + tags := map[string]string{ + lib.DiceOrgNameKey: org.Name, + lib.DiceWorkspace: ns, + } + key = p.LogRetention.GetConfigKey("container", tags) + case "metrics": + tags := map[string]string{ + lib.OrgNameKey: org.Name, + lib.DiceWorkspace: ns, + } + key = p.MetricRetention.GetConfigKey("docker_container_summary", tags) + default: + return false + } + return len(key) == 0 +} + +func (p *provider) getTTL(monitorType string, cfg map[string]map[string]*pb.ConfigItem) *structpb.Value { + if cfg == nil { + return nil + } + monitorCfg := cfg["monitor"] + if monitorCfg == nil { + return nil + } + ttl := monitorCfg[fmt.Sprintf("%s_ttl", monitorType)] + return ttl.Value +} + +func (p *provider) autoCreateOrgMonitorConfig(orgID uint64) error { + p.settingsService.PutSettings(context.Background(), &pb.PutSettingsRequest{ + OrgID: int64(orgID), + }) + return nil +} diff --git a/internal/tools/monitor/oap/collector/lib/const.go b/internal/tools/monitor/oap/collector/lib/const.go index 7d213098f8e..a1a776f8cfe 100644 --- a/internal/tools/monitor/oap/collector/lib/const.go +++ b/internal/tools/monitor/oap/collector/lib/const.go @@ -15,5 +15,7 @@ package lib const ( - OrgNameKey = "org_name" + OrgNameKey = "org_name" + DiceOrgNameKey = "dice_org_name" + DiceWorkspace = "dice_workspace" )