feat: webconsole relay优化

bcs-services/bcs-webconsole/app/app.go

@@ -430,6 +430,10 @@ func (c *WebConsoleManager) Run() error {
 		return uploader.IntervalUpload(ctx)
 	})
 
+	eg.Go(func() error {
+		return uploader.IntervalDelete(ctx)
+	})
+
 	// 定时上报用户延迟命令列表数据
 	performance := perf.GetGlobalPerformance()
 	eg.Go(func() error {

bcs-services/bcs-webconsole/console/audit/record/uploader.go

@@ -19,6 +19,7 @@ import (
 	"io/fs"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"sync"
 	"time"
 
@@ -122,6 +123,41 @@ func (u *Uploader) IntervalUpload(ctx context.Context) error {
 	}
 }
 
+// IntervalDelete 定时删除
+func (u *Uploader) IntervalDelete(ctx context.Context) error {
+	if !config.G.Audit.Enabled || u.storage == nil {
+		blog.Info("audit not enabled or storage type not set, delete just ignore")
+		return nil
+	}
+
+	timer := time.NewTicker(time.Second * 30)
+	defer timer.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			// 获取退出信号, 删除一次
+			err := u.deleteFolders()
+			if err != nil {
+				blog.Errorf("interval remain delete folders failed, err: %s", err)
+				return err
+			}
+
+			blog.Infof("interval remain delete folders done")
+			return nil
+
+		case <-timer.C:
+			err := u.deleteFolders()
+			if err != nil {
+				blog.Errorf("interval delete folders failed, err: %s", err)
+				continue
+			}
+
+			blog.Infof("interval delete folders done")
+		}
+	}
+}
+
 func (u *Uploader) checkFileCanUpload(path string) bool {
 	u.lock.Lock()
 	defer u.lock.Unlock()
@@ -233,3 +269,42 @@ func (u *Uploader) upload(cast *castFile) error {
 
 	return nil
 }
+
+// 删除文件夹
+func (u *Uploader) deleteFolders() error {
+
+	// 1 分钟删除时间
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+
+	folderNames, err := u.storage.ListFolders(ctx, "")
+	if err != nil {
+		blog.Errorf("list folders failed, err: %s", err)
+		return err
+	}
+
+	var expireFolders []string
+	reserveDay := time.Duration(config.G.Audit.ReserveDay)
+	if reserveDay == 0 {
+		reserveDay = 30 // 默认30天
+	}
+	for _, v := range folderNames {
+		v = strings.Trim(v, "/")
+		date, err := time.Parse(dateTimeFormat, v)
+		if err != nil {
+			continue
+		}
+		// 删除 reserveDay 天前的文件夹
+		if date.Before(time.Now().Add(-time.Hour * 24 * reserveDay)) {
+			expireFolders = append(expireFolders, v)
+			if err := u.storage.DeleteFolders(ctx, v); err != nil {
+				blog.Errorf("delete folders %s failed, err: %s", v, err)
+				return err
+			}
+		}
+	}
+
+	blog.Infof("delete folders success: %v", expireFolders)
+
+	return nil
+}

bcs-services/bcs-webconsole/console/config/audit.go

@@ -19,8 +19,9 @@ import (
 
 // AuditConf 终端session记录配置
 type AuditConf struct {
-	Enabled bool   `yaml:"enabled"`
-	DataDir string `yaml:"data_dir"` // 格式如 ./data
+	Enabled    bool   `yaml:"enabled"`
+	DataDir    string `yaml:"data_dir"` // 格式如 ./data
+	ReserveDay int    `yaml:"reserve_day"`
 }
 
 func (t *AuditConf) defaultPath() string {

bcs-services/bcs-webconsole/console/repository/bkrepo.go

@@ -185,6 +185,29 @@ func (b *bkrepoStorage) ListFolders(ctx context.Context, folderName string) ([]s
 	return folders, nil
 }
 
+// DeleteFolders delete of folders under current bkRepo folder
+func (b *bkrepoStorage) DeleteFolders(ctx context.Context, folderName string) error {
+	// 节点详情 https://github.com/TencentBlueKing/bk-repo/blob/master/docs/apidoc/node/node.md
+	// GET /repository/api/node/page/{projectId}/{repoName}/{fullPath}
+	rawURL := fmt.Sprintf("%s/repository/api/node/delete/%s/%s/%s", config.G.Repository.Bkrepo.Endpoint,
+		config.G.Repository.Bkrepo.Project, config.G.Repository.Bkrepo.Repo, folderName)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodDelete, rawURL, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := b.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("delete folder failed, Err code: %v", resp.StatusCode)
+	}
+
+	return nil
+}
+
 // DownloadFile download file from bkRepo
 func (b *bkrepoStorage) DownloadFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
 	// 下载文件API PUT /generic/{project}/{repoName}/{fullPath}

bcs-services/bcs-webconsole/console/repository/cos.go

@@ -132,6 +132,36 @@ func (c *cosStorage) ListFolders(ctx context.Context, folderName string) ([]stri
 	return folders, nil
 }
 
+// DeleteFolders delete folder from cos
+func (c *cosStorage) DeleteFolders(ctx context.Context, folderName string) error {
+	folderName = strings.Trim(folderName, "/")
+	folderName += "/"
+
+	var marker string
+	opt := &cos.BucketGetOptions{
+		Prefix:  folderName, // 表示要查询的文件夹
+		MaxKeys: 1000,       // 设置最大遍历出多少个对象, 一次 listobject 最大支持1000
+	}
+	isTruncated := true
+	for isTruncated {
+		opt.Marker = marker
+		v, _, err := c.client.Bucket.Get(ctx, opt)
+		if err != nil {
+			return err
+		}
+		for _, content := range v.Contents {
+			_, err = c.client.Object.Delete(ctx, content.Key)
+			if err != nil {
+				return err
+			}
+		}
+		isTruncated = v.IsTruncated
+		marker = v.NextMarker
+	}
+
+	return nil
+}
+
 // DownloadFile download file from cos
 func (c *cosStorage) DownloadFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
 	resp, err := c.client.Object.Get(ctx, filePath, nil)

bcs-services/bcs-webconsole/console/repository/repo.go

@@ -25,6 +25,7 @@ type Provider interface {
 	UploadFileByReader(ctx context.Context, r io.Reader, filePath string) error
 	ListFile(ctx context.Context, folderName string) ([]string, error)
 	ListFolders(ctx context.Context, folderName string) ([]string, error)
+	DeleteFolders(ctx context.Context, folderName string) error
 	IsExist(ctx context.Context, filePath string) (bool, error)
 	DownloadFile(ctx context.Context, filePath string) (io.ReadCloser, error)
 }

bcs-services/bcs-webconsole/console/web/service.go

@@ -18,6 +18,9 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"sort"
+	"strings"
+	"time"
 
 	gintrace "github.com/Tencent/bk-bcs/bcs-common/pkg/otel/trace/gin"
 	"github.com/gin-gonic/gin"
@@ -80,6 +83,10 @@ func (s *service) ReplayFilesPageHandler(c *gin.Context) {
 		rest.APIError(c, fmt.Sprintf("List storage files err: %v\n", err))
 		return
 	}
+	// 按照时间逆序排
+	sort.Slice(fileNames, func(i, j int) bool {
+		return fileNames[i] > fileNames[j]
+	})
 	data := gin.H{
 		"folder_name":     folderName,
 		"file_names":      fileNames,
@@ -100,8 +107,29 @@ func (s *service) ReplayFoldersPageHandler(c *gin.Context) {
 		rest.APIError(c, fmt.Sprintf("List storage folder err: %v", err))
 		return
 	}
+	// 将字符串切片转换为 time.Time 切片，按照时间逆序排
+	var dates []time.Time
+	for _, v := range folderNames {
+		v = strings.Trim(v, "/")
+		date, err := time.Parse(time.DateOnly, v)
+		if err != nil {
+			continue
+		}
+		dates = append(dates, date)
+	}
+
+	// 按照时间逆序排
+	sort.Slice(dates, func(i, j int) bool {
+		return dates[i].After(dates[j])
+	})
+
+	var newFolderNames []string
+	for _, date := range dates {
+		newFolderNames = append(newFolderNames, date.Format("2006-01-02"))
+	}
+
 	data := gin.H{
-		"folder_names":    folderNames,
+		"folder_names":    newFolderNames,
 		"SITE_STATIC_URL": s.opts.RoutePrefix,
 	}
 	c.HTML(http.StatusOK, "replay.html", data)