fix(streaming): acquire conn timeout (#5977)

* upgrade erda-infra pkg

* close clickhouse conn after used

* fix typo for `dial_timeout`

* add `net-tools` in erda-base

.erda/pipelines/ci-build-ce.yml

@@ -42,7 +42,7 @@ stages:
       - custom-script:
           alias: build-erda
           description: 运行自定义命令
-          image: registry.erda.cloud/erda/erda-base:20230130
+          image: registry.erda.cloud/erda/erda-base:20230712
           commands:
             - cd ${{ dirs.erda }}
             - mkdir -p extensions

.erda/pipelines/ci-deploy.yml

@@ -60,7 +60,7 @@ stages:
   - stage:
       - custom-script:
           alias: build-erda
-          image: registry.erda.cloud/erda/erda-base:20230130
+          image: registry.erda.cloud/erda/erda-base:20230712
           commands:
             - cd ${{ dirs.erda }}
             - mkdir -p extensions
@@ -83,7 +83,7 @@ stages:
             mem: 2048
       - custom-script:
           alias: build-erda-cli
-          image: registry.erda.cloud/erda/erda-base:20230130
+          image: registry.erda.cloud/erda/erda-base:20230712
           commands:
             - cd ${{ dirs.erda }}
             - export ERDA_VERSION=${{ outputs.extract-repo-version.major_minor_version }}
@@ -101,7 +101,7 @@ stages:
             mem: 2048
       - custom-script:
           alias: build-agent
-          image: registry.erda.cloud/erda/erda-base:20230130
+          image: registry.erda.cloud/erda/erda-base:20230712
           commands:
             - cd ${{ dirs.erda }}
             - export ERDA_VERSION=${{ outputs.extract-repo-version.major_minor_version }}
@@ -138,7 +138,7 @@ stages:
             mem: 1024
       - custom-script:
           alias: build-erda-cli-linux
-          image: registry.erda.cloud/erda/erda-base:20230130
+          image: registry.erda.cloud/erda/erda-base:20230712
           description: 运行自定义命令
           commands:
             - workDir=$(pwd)
@@ -156,7 +156,7 @@ stages:
             mem: 2048
       - custom-script:
           alias: build-erda-cli-mac
-          image: registry.erda.cloud/erda/erda-base:20230130
+          image: registry.erda.cloud/erda/erda-base:20230712
           description: 运行自定义命令
           commands:
             - workDir=$(pwd)

.github/workflows/ci-it.yml

@@ -145,7 +145,7 @@ jobs:
   CODE-TEST:
     runs-on: ubuntu-latest
     container:
-      image: registry.erda.cloud/erda/erda-base:20230130
+      image: registry.erda.cloud/erda/erda-base:20230712
     needs:
       - PREPARE
     strategy:

Makefile

@@ -130,7 +130,7 @@ run-test:
 	go run tools/gotools/go-test-sum/main.go
 
 full-test:
-	docker run --rm -ti -v $$(pwd):/go/src/output registry.erda.cloud/erda/erda-base:20230130 \
+	docker run --rm -ti -v $$(pwd):/go/src/output registry.erda.cloud/erda/erda-base:20230712 \
 		bash -c 'cd /go/src/output && build/scripts/test_in_container.sh'
 
 # docker image

build/dockerfiles/base/Dockerfile

@@ -17,6 +17,7 @@ RUN apt update && apt install -y --no-install-recommends --no-install-suggests \
     jq \
     python3-pip \
     librdkafka-dev \
+    net-tools \
  && rm -fr /var/lib/apt/lists/*
 
 # nodejs

build/dockerfiles/base/deploy.sh

@@ -6,17 +6,16 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 # create docker buildx builder
 docker buildx create --name erda-base-builder --platform linux/amd64,linux/arm64 --use || true
 
-images=()
+image=registry.erda.cloud/erda/erda-base:$(date +"%Y%m%d")
+
+docker buildx build --platform linux/amd64,linux/arm64 -t ${image} -f Dockerfile --push .
 
 archs=(amd64 arm64)
 for arch in ${archs[@]}; do
-    image=registry.erda.cloud/erda/${arch}/erda-base:$(date +"%Y%m%d")
-    images+=(${image})
-    docker buildx build --platform linux/${arch} -t ${image} -f Dockerfile --push .
     # generate Software Bill Of Materials (SBOM)
-    docker sbom ${image} --output sbom-${arch}.txt
+    docker sbom ${image} --platform linux/${arch} --output sbom-${arch}.txt
 done
 
-for image in ${images[@]}; do
-    echo "action meta: erda-base=$image"
-done
+echo "action meta: erda-base=$image"
+
+docker buildx rm erda-base-builder

build/scripts/buildkit_image.sh

@@ -4,7 +4,7 @@ set -o errexit -o pipefail
 
 cd "$(git rev-parse --show-toplevel)"
 
-BASE_DOCKER_IMAGE=registry.erda.cloud/erda/erda-base:20230130
+BASE_DOCKER_IMAGE=registry.erda.cloud/erda/erda-base:20230712
 IMAGE_TAG="${IMAGE_TAG:-$(build/scripts/make-version.sh tag)}"
 DOCKERFILE=./build/dockerfiles
 

build/scripts/docker_image.sh

@@ -35,7 +35,7 @@ ARCH="${ARCH:-$(go env GOARCH)}"
 VERSION="$(build/scripts/make-version.sh)"
 IMAGE_TAG="${IMAGE_TAG:-$(build/scripts/make-version.sh tag)}"
 DOCKERFILE_DEFAULT="build/dockerfiles/Dockerfile"
-BASE_DOCKER_IMAGE="registry.erda.cloud/erda/erda-base:20230130"
+BASE_DOCKER_IMAGE="registry.erda.cloud/erda/erda-base:20230712"
 DOCKERFILE=${DOCKERFILE_DEFAULT}
 
 # setup single module envionment variables

cmd/monitor/monitor/bootstrap.yaml

@@ -73,7 +73,7 @@ clickhouse:
   conn_max_lifetime: "${CLICKHOUSE_CONN_MAX_LIFETIME:1m}"
   max_idle_conns: "${CLICKHOUSE_MAX_IDLE_CONNS:10}"
   max_open_conns: "${CLICKHOUSE_MAX_OPEN_CONNS:200}"
-  dail_timeout: "${CLICKHOUSE_DAIL_TIMEOUT:1s}"
+  dial_timeout: "${CLICKHOUSE_DIAL_TIMEOUT:1s}"
   debug: ${CLICKHOUSE_DEBUG:false}
 
 

cmd/monitor/streaming/bootstrap.yaml

@@ -16,6 +16,8 @@ kafkago:
   client_id: streaming
   debug_client: ${KAFKAGO_DEBUG_CLIENT:true}
 
+profile-agent:
+
 
 #kafka.topic.initializer:
 #  force: ${KAFKA_INIT_FORCE_SUCCESS:true}
@@ -73,10 +75,10 @@ clickhouse:
   username: "${CLICKHOUSE_USERNAME:default}"
   password: "${CLICKHOUSE_PASSWORD:default}"
   conn_open_strategy: "round_robin"
-  conn_max_lifetime: "${CLICKHOUSE_CONN_MAX_LIFETIME:1m}"
-  max_idle_conns: "${CLICKHOUSE_MAX_IDLE_CONNS:10}"
+  conn_max_lifetime: "${CLICKHOUSE_CONN_MAX_LIFETIME:1h}"
+  max_idle_conns: "${CLICKHOUSE_MAX_IDLE_CONNS:195}"
   max_open_conns: "${CLICKHOUSE_MAX_OPEN_CONNS:200}"
-  dail_timeout: "${CLICKHOUSE_DAIL_TIMEOUT:1s}"
+  dial_timeout: "${CLICKHOUSE_DIAL_TIMEOUT:30s}"
 
 
 storage-retention-strategy@span:

cmd/monitor/streaming/main.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/erda-project/erda/internal/apps/msp/apm/trace/persist"
 	_ "github.com/erda-project/erda/internal/apps/msp/apm/trace/storage/cassandra_v1"
 	_ "github.com/erda-project/erda/internal/apps/msp/apm/trace/storage/elasticsearch"
+	_ "github.com/erda-project/erda/internal/pkg/profileagent"
 	_ "github.com/erda-project/erda/internal/tools/monitor/notify/storage/notify-record"
 
 	// modules

go.mod

@@ -432,7 +432,7 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.27.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0 // indirect
-	go.opentelemetry.io/otel/internal/metric v0.25.0 // indirect
+	go.opentelemetry.io/otel/internal/metric v0.26.0 // indirect
 	go.opentelemetry.io/otel/metric v0.36.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
@@ -474,7 +474,7 @@ replace (
 	github.com/containerd/containerd => github.com/containerd/containerd v1.4.4
 	github.com/coreos/bbolt => go.etcd.io/bbolt v1.3.5
 	github.com/docker/spdystream => github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96
-	github.com/erda-project/erda-infra => github.com/erda-project/erda-infra v0.0.0-20220928061127-9fca81f05238
+	github.com/erda-project/erda-infra => github.com/erda-project/erda-infra v0.0.0-20230711160226-c6156f3b1be1
 	github.com/erda-project/erda-proto-go v1.4.0 => ./api/proto-go
 	github.com/getkin/kin-openapi => github.com/getkin/kin-openapi v0.49.0
 	github.com/go-logr/logr => github.com/go-logr/logr v0.4.0
@@ -490,7 +490,7 @@ replace (
 	go.etcd.io/bbolt => github.com/coreos/bbolt v1.3.5
 	go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.2.0
 	go.opentelemetry.io/otel/exporters/stdout => go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0
-	go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.25.0
+	go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.26.0
 	go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.2.0
 	go.opentelemetry.io/proto/otlp v0.11.0 => github.com/recallsong/opentelemetry-proto-go/otlp v0.11.1-0.20211202093058-995eca7123f5
 	google.golang.org/grpc => google.golang.org/grpc v1.26.0

go.sum

@@ -559,8 +559,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrp
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/erda-project/elastic v0.0.1-ex h1:5ajfxQ5S5YjpzFqY9LzL9hiKWCn6q/JDT4n8sNv7+pU=
 github.com/erda-project/elastic v0.0.1-ex/go.mod h1:iAVsas6fcmt9pxtge1+dErMhecv+RLSXlD4rnZRJVW0=
-github.com/erda-project/erda-infra v0.0.0-20220928061127-9fca81f05238 h1:MaCjW2hHEOGPkXBObOJWkB/pKEIzRz2EH/HVR6h80wA=
-github.com/erda-project/erda-infra v0.0.0-20220928061127-9fca81f05238/go.mod h1:jXSyP8z53jqWmBTRvvQX2hSrK7CQl8+oI8eRpvTIS88=
+github.com/erda-project/erda-infra v0.0.0-20230711160226-c6156f3b1be1 h1:JP7JMOxYizNPisb4ysohU/qrgAGzRSjJ+1fLsUpTlDQ=
+github.com/erda-project/erda-infra v0.0.0-20230711160226-c6156f3b1be1/go.mod h1:Lbtn3ilkB2nrZKqX4dfwD/IAbYe4LIrsFxFfEwN/NiA=
 github.com/erda-project/erda-infra/tools v0.0.0-20220922112628-1965c0260662 h1:HmnHrnhYxEc3xrNf/+MQ1jbvUJEJbvcUtjRFzx87j+g=
 github.com/erda-project/erda-infra/tools v0.0.0-20220922112628-1965c0260662/go.mod h1:wy47I5tWqonst1Gt0YEjqFHnOpEYShZhsJo95KTBuvY=
 github.com/erda-project/erda-oap-thirdparty-protocol v0.0.0-20210907135609-15886a136d5b h1:GWf2ChasZFerFwQoTokIvjJLWH57ligTSLD2hUb7UWk=
@@ -2033,10 +2033,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0 h1:Ydage/
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0 h1:OiYdrCq1Ctwnovp6EofSPwlp5aGy4LgKNbkg7PtEUw8=
 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0/go.mod h1:DUFCmFkXr0VtAHl5Zq2JRx24G6ze5CAq8YfdD36RdX8=
-go.opentelemetry.io/otel/internal/metric v0.25.0 h1:w/7RXe16WdPylaIXDgcYM6t/q0K5lXgSdZOEbIEyliE=
-go.opentelemetry.io/otel/internal/metric v0.25.0/go.mod h1:Nhuw26QSX7d6n4duoqAFi5KOQR4AuzyMcl5eXOgwxtc=
-go.opentelemetry.io/otel/metric v0.25.0 h1:7cXOnCADUsR3+EOqxPaSKwhEuNu0gz/56dRN1hpIdKw=
-go.opentelemetry.io/otel/metric v0.25.0/go.mod h1:E884FSpQfnJOMMUaq+05IWlJ4rjZpk2s/F1Ju+TEEm8=
+go.opentelemetry.io/otel/internal/metric v0.26.0 h1:dlrvawyd/A+X8Jp0EBT4wWEe4k5avYaXsXrBr4dbfnY=
+go.opentelemetry.io/otel/internal/metric v0.26.0/go.mod h1:CbBP6AxKynRs3QCbhklyLUtpfzbqCLiafV9oY2Zj1Jk=
+go.opentelemetry.io/otel/metric v0.26.0 h1:VaPYBTvA13h/FsiWfxa3yZnZEm15BhStD8JZQSA773M=
+go.opentelemetry.io/otel/metric v0.26.0/go.mod h1:c6YL0fhRo4YVoNs6GoByzUgBp36hBL523rECoZA5UWg=
 go.opentelemetry.io/otel/sdk v1.2.0 h1:wKN260u4DesJYhyjxDa7LRFkuhH7ncEVKU37LWcyNIo=
 go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U=
 go.opentelemetry.io/otel/trace v1.1.0/go.mod h1:i47XtdcBQiktu5IsrPqOHe8w+sBmnLwwHt8wiUsWGTI=

internal/tools/monitor/apm/topology/topology_v2.go

@@ -261,6 +261,7 @@ func (topology *provider) GetTopologyV2(orgName string, lang i18n.LanguageCodes,
 			}
 
 			allNodeRelations = append(allNodeRelations, topology.parseToTypologyNodeV2(lang, rows, tg)...)
+			rows.Close()
 		}
 	}
 	handleTargetOtherNodesByHttpUrl(allNodeRelations, topology.Cfg.TargetOtherNodeOptions)

internal/tools/monitor/core/alert/alert-apis/db/alert_rule.go

@@ -32,6 +32,7 @@ func (db *AlertRuleDB) DistinctAlertTypeByScope(scope string) ([]string, error)
 	if err != nil {
 		return nil, err
 	}
+	defer rows.Close()
 
 	var alertTypes []string
 	for rows.Next() {

internal/tools/monitor/core/alert/details-apis/clickhouse_source.go

@@ -108,6 +108,7 @@ func (chs *ClickhouseSource) getPod(ctx context.Context, orgName, clusterName, p
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to query pod row")
 	}
+	defer row.Close()
 
 	row.Next()
 	pod := podRow{}
@@ -145,6 +146,7 @@ func (chs *ClickhouseSource) getContainers(ctx context.Context, orgName, cluster
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to query container row")
 	}
+	defer rows.Close()
 
 	var containers []*containerRow
 	for rows.Next() {

internal/tools/monitor/core/log/storage/clickhouse/iterator.go

@@ -274,6 +274,12 @@ func (it *clickhouseIterator) fetch(dir iteratorDir) {
 		reverse = true
 	}
 	it.buffer = nil
+	defer func() {
+		if it.lastResp != nil {
+			it.lastResp.Close()
+			it.lastResp = nil
+		}
+	}()
 	for it.err == nil && len(it.buffer) == 0 {
 		fetchingRemote := false
 		if it.lastResp == nil {
@@ -338,8 +344,6 @@ func (it *clickhouseIterator) fetch(dir iteratorDir) {
 			if fetchingRemote {
 				it.err = io.EOF
 			}
-			_ = it.lastResp.Close()
-			it.lastResp = nil
 			return
 		}
 	}

internal/tools/monitor/core/storekit/clickhouse/table/meta/meta_loader.go

@@ -137,6 +137,7 @@ func (p *provider) reloadMetaFromClickhouse(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to query metric meta")
 	}
+	defer rows.Close()
 
 	var metas []MetricMeta
 

internal/tools/monitor/dashboard/org-apis/clickhouse_source.go

@@ -190,6 +190,7 @@ func (chs *ClickhouseSource) GetHostTypes(req *http.Request, params struct {
 		chs.Log.Errorf("failed to query clickhouse, %v", err)
 		return nil
 	}
+	defer rows.Close()
 
 	var hostTypes []*hostTypeRow
 	for rows.Next() {
@@ -355,6 +356,7 @@ func (chs *ClickhouseSource) GetGroupHosts(req *http.Request, params struct {
 		chs.Log.Errorf("failed to query clickhouse, %v", err)
 		return nil
 	}
+	defer rows.Close()
 
 	var hosts []*hostData
 	for rows.Next() {

pipeline.yml

@@ -146,7 +146,7 @@ stages:
           disable: true
       - custom-script:
           alias: build-erda-cli-linux
-          image: registry.erda.cloud/erda/erda-base:20230130
+          image: registry.erda.cloud/erda/erda-base:20230712
           description: 运行自定义命令
           commands:
             - workDir=$(pwd)
@@ -164,7 +164,7 @@ stages:
             mem: 2048
       - custom-script:
           alias: build-erda-cli-mac
-          image: registry.erda.cloud/erda/erda-base:20230130
+          image: registry.erda.cloud/erda/erda-base:20230712
           description: 运行自定义命令
           commands:
             - workDir=$(pwd)