-
Notifications
You must be signed in to change notification settings - Fork 249
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: webconsole relay优化 #3593
base: master
Are you sure you want to change the base?
feat: webconsole relay优化 #3593
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,6 +19,7 @@ import ( | |
"io/fs" | ||
"path/filepath" | ||
"regexp" | ||
"strings" | ||
"sync" | ||
"time" | ||
|
||
|
@@ -122,6 +123,41 @@ func (u *Uploader) IntervalUpload(ctx context.Context) error { | |
} | ||
} | ||
|
||
// IntervalDelete 定时删除 | ||
func (u *Uploader) IntervalDelete(ctx context.Context) error { | ||
if !config.G.Audit.Enabled || u.storage == nil { | ||
blog.Info("audit not enabled or storage type not set, delete just ignore") | ||
return nil | ||
} | ||
|
||
timer := time.NewTicker(time.Second * 30) | ||
defer timer.Stop() | ||
|
||
for { | ||
select { | ||
case <-ctx.Done(): | ||
// 获取退出信号, 删除一次 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这里数据不会丢失,直接退出 |
||
err := u.deleteFolders() | ||
if err != nil { | ||
blog.Errorf("interval remain delete folders failed, err: %s", err) | ||
return err | ||
} | ||
|
||
blog.Infof("interval remain delete folders done") | ||
return nil | ||
|
||
case <-timer.C: | ||
err := u.deleteFolders() | ||
if err != nil { | ||
blog.Errorf("interval delete folders failed, err: %s", err) | ||
continue | ||
} | ||
|
||
blog.Infof("interval delete folders done") | ||
} | ||
} | ||
} | ||
|
||
func (u *Uploader) checkFileCanUpload(path string) bool { | ||
u.lock.Lock() | ||
defer u.lock.Unlock() | ||
|
@@ -233,3 +269,42 @@ func (u *Uploader) upload(cast *castFile) error { | |
|
||
return nil | ||
} | ||
|
||
// 删除文件夹 | ||
func (u *Uploader) deleteFolders() error { | ||
|
||
// 1 分钟删除时间 | ||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute) | ||
defer cancel() | ||
|
||
folderNames, err := u.storage.ListFolders(ctx, "") | ||
if err != nil { | ||
blog.Errorf("list folders failed, err: %s", err) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 错误和日志记录一个就好, 调用函数入口已经记录,可以返回fmt.Errorf("list folders: %w", err), 下同 https://tonybai.com/2024/04/14/either-return-error-or-log-them-do-not-do-both/ |
||
return err | ||
} | ||
|
||
var expireFolders []string | ||
reserveDay := time.Duration(config.G.Audit.ReserveDay) | ||
if reserveDay == 0 { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 0应该是不删除, 默认30天在配置文件中, 记得输出 ./bcs-webconsole -o yaml > etc/config.yaml |
||
reserveDay = 30 // 默认30天 | ||
} | ||
for _, v := range folderNames { | ||
v = strings.Trim(v, "/") | ||
date, err := time.Parse(dateTimeFormat, v) | ||
if err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 解析错误打印warnning日志,人工处理 |
||
continue | ||
} | ||
// 删除 reserveDay 天前的文件夹 | ||
if date.Before(time.Now().Add(-time.Hour * 24 * reserveDay)) { | ||
expireFolders = append(expireFolders, v) | ||
if err := u.storage.DeleteFolders(ctx, v); err != nil { | ||
blog.Errorf("delete folders %s failed, err: %s", v, err) | ||
return err | ||
} | ||
} | ||
} | ||
|
||
blog.Infof("delete folders success: %v", expireFolders) | ||
|
||
return nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,8 +19,9 @@ import ( | |
|
||
// AuditConf 终端session记录配置 | ||
type AuditConf struct { | ||
Enabled bool `yaml:"enabled"` | ||
DataDir string `yaml:"data_dir"` // 格式如 ./data | ||
Enabled bool `yaml:"enabled"` | ||
DataDir string `yaml:"data_dir"` // 格式如 ./data | ||
ReserveDay int `yaml:"reserve_day"` | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. RetentionDays |
||
} | ||
|
||
func (t *AuditConf) defaultPath() string { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -185,6 +185,29 @@ func (b *bkrepoStorage) ListFolders(ctx context.Context, folderName string) ([]s | |
return folders, nil | ||
} | ||
|
||
// DeleteFolders delete of folders under current bkRepo folder | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. List 显示不全是不是没有处理 |
||
func (b *bkrepoStorage) DeleteFolders(ctx context.Context, folderName string) error { | ||
// 节点详情 https://github.com/TencentBlueKing/bk-repo/blob/master/docs/apidoc/node/node.md | ||
// GET /repository/api/node/page/{projectId}/{repoName}/{fullPath} | ||
rawURL := fmt.Sprintf("%s/repository/api/node/delete/%s/%s/%s", config.G.Repository.Bkrepo.Endpoint, | ||
config.G.Repository.Bkrepo.Project, config.G.Repository.Bkrepo.Repo, folderName) | ||
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, rawURL, nil) | ||
if err != nil { | ||
return err | ||
} | ||
resp, err := b.client.Do(req) | ||
if err != nil { | ||
return err | ||
} | ||
defer resp.Body.Close() | ||
if resp.StatusCode != 200 { | ||
return fmt.Errorf("delete folder failed, Err code: %v", resp.StatusCode) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
// DownloadFile download file from bkRepo | ||
func (b *bkrepoStorage) DownloadFile(ctx context.Context, filePath string) (io.ReadCloser, error) { | ||
// 下载文件API PUT /generic/{project}/{repoName}/{fullPath} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,6 +18,9 @@ import ( | |
"net/http" | ||
"net/url" | ||
"path" | ||
"sort" | ||
"strings" | ||
"time" | ||
|
||
gintrace "github.com/Tencent/bk-bcs/bcs-common/pkg/otel/trace/gin" | ||
"github.com/gin-gonic/gin" | ||
|
@@ -80,6 +83,10 @@ func (s *service) ReplayFilesPageHandler(c *gin.Context) { | |
rest.APIError(c, fmt.Sprintf("List storage files err: %v\n", err)) | ||
return | ||
} | ||
// 按照时间逆序排 | ||
sort.Slice(fileNames, func(i, j int) bool { | ||
return fileNames[i] > fileNames[j] | ||
}) | ||
data := gin.H{ | ||
"folder_name": folderName, | ||
"file_names": fileNames, | ||
|
@@ -100,8 +107,29 @@ func (s *service) ReplayFoldersPageHandler(c *gin.Context) { | |
rest.APIError(c, fmt.Sprintf("List storage folder err: %v", err)) | ||
return | ||
} | ||
// 将字符串切片转换为 time.Time 切片,按照时间逆序排 | ||
var dates []time.Time | ||
for _, v := range folderNames { | ||
v = strings.Trim(v, "/") | ||
date, err := time.Parse(time.DateOnly, v) | ||
if err != nil { | ||
continue | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这里如果目录不合法,效果是直接隐藏了, 直接按0值处理(相同的日志按字符串排序) |
||
} | ||
dates = append(dates, date) | ||
} | ||
|
||
// 按照时间逆序排 | ||
sort.Slice(dates, func(i, j int) bool { | ||
return dates[i].After(dates[j]) | ||
}) | ||
|
||
var newFolderNames []string | ||
for _, date := range dates { | ||
newFolderNames = append(newFolderNames, date.Format("2006-01-02")) | ||
} | ||
|
||
data := gin.H{ | ||
"folder_names": folderNames, | ||
"folder_names": newFolderNames, | ||
"SITE_STATIC_URL": s.opts.RoutePrefix, | ||
} | ||
c.HTML(http.StatusOK, "replay.html", data) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
不需要那么频繁List列表, 默认10分钟就好