feat(vacuum): add vacuum management for cleaning expired packages

.github/workflows/wc-integration-test.yaml

@@ -214,9 +214,25 @@ jobs:
       - run: aqua list
         working-directory: /tmp
 
+      # Test if commands succeed with vacuum enabled
+      - name: Test aqua install with AQUA_VACUUM_DAYS
+        env:
+          AQUA_VACUUM_DAYS: 1
+        run: aqua install
+      - name: Test aqua exec with AQUA_VACUUM_DAYS
+        env:
+          AQUA_VACUUM_DAYS: 1
+        run: cmdx -v
+      - name: Test vacuum Run command
+        env:
+          AQUA_VACUUM_DAYS: 1
+        run: aqua vacuum run
+
+      # Test update-aqua command
       - name: Test update-aqua
         run: aqua update-aqua
 
+
   integration-test-cargo:
     timeout-minutes: 30
     runs-on: ubuntu-latest

go.mod

@@ -48,6 +48,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
+	github.com/dustin/go-humanize v1.0.1
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/gdamore/tcell/v2 v2.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
@@ -82,6 +83,7 @@ require (
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.etcd.io/bbolt v1.3.11
 	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/sync v0.9.0 // indirect

go.sum

@@ -56,6 +56,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
 github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI=
@@ -255,6 +257,8 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
+go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

pkg/cli/exec/command.go

@@ -42,8 +42,10 @@ func (i *command) action(c *cli.Context) error {
 	}
 	defer profiler.Stop()
 
+	logE := i.r.LogE
+
 	param := &config.Param{}
-	if err := util.SetParam(c, i.r.LogE, "exec", param, i.r.LDFlags); err != nil {
+	if err := util.SetParam(c, logE, "exec", param, i.r.LDFlags); err != nil {
 		return fmt.Errorf("parse the command line arguments: %w", err)
 	}
 	ctrl, err := controller.InitializeExecCommandController(c.Context, param, http.DefaultClient, i.r.Runtime)
@@ -54,5 +56,6 @@ func (i *command) action(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("parse args: %w", err)
 	}
-	return ctrl.Exec(c.Context, i.r.LogE, param, exeName, args...) //nolint:wrapcheck
+	defer ctrl.CloseVacuum(logE)
+	return ctrl.Exec(c.Context, logE, param, exeName, args...) //nolint:wrapcheck
 }

pkg/cli/install/command.go

@@ -80,13 +80,16 @@ func (i *command) action(c *cli.Context) error {
 	}
 	defer profiler.Stop()
 
+	logE := i.r.LogE
+
 	param := &config.Param{}
-	if err := util.SetParam(c, i.r.LogE, "install", param, i.r.LDFlags); err != nil {
+	if err := util.SetParam(c, logE, "install", param, i.r.LDFlags); err != nil {
 		return fmt.Errorf("parse the command line arguments: %w", err)
 	}
 	ctrl, err := controller.InitializeInstallCommandController(c.Context, param, http.DefaultClient, i.r.Runtime)
 	if err != nil {
 		return fmt.Errorf("initialize a InstallController: %w", err)
 	}
+	defer ctrl.CloseVacuum(logE)
 	return ctrl.Install(c.Context, i.r.LogE, param) //nolint:wrapcheck
 }

pkg/cli/runner.go

@@ -20,6 +20,7 @@ import (
 	"github.com/aquaproj/aqua/v2/pkg/cli/update"
 	"github.com/aquaproj/aqua/v2/pkg/cli/updateaqua"
 	"github.com/aquaproj/aqua/v2/pkg/cli/util"
+	"github.com/aquaproj/aqua/v2/pkg/cli/vacuum"
 	"github.com/aquaproj/aqua/v2/pkg/cli/version"
 	"github.com/aquaproj/aqua/v2/pkg/cli/which"
 	"github.com/urfave/cli/v2"
@@ -105,6 +106,7 @@ func Run(ctx context.Context, param *util.Param, args ...string) error { //nolin
 			upc.New,
 			remove.New,
 			update.New,
+			vacuum.New,
 		),
 	}
 

pkg/cli/util/util.go

@@ -33,15 +33,24 @@ type LDFlags struct {
 	Date    string
 }
 
-func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, ldFlags *LDFlags) error { //nolint:funlen,cyclop
+func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, ldFlags *LDFlags) error {
 	wd, err := os.Getwd()
 	if err != nil {
 		return fmt.Errorf("get the current directory: %w", err)
 	}
 	param.Args = c.Args().Slice()
-	if logLevel := c.String("log-level"); logLevel != "" {
-		param.LogLevel = logLevel
+	setBasicParams(c, logE, commandName, param, wd, ldFlags)
+	setLogParams(c, param, logE)
+	if err := setEnvParams(param); err != nil {
+		return fmt.Errorf("error during setting params from Env vars: %w", err)
 	}
+	if err := setChecksumParams(param); err != nil {
+		return fmt.Errorf("error during setting params from checksum params: %w", err)
+	}
+	return nil
+}
+
+func setBasicParams(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, wd string, ldFlags *LDFlags) {
 	param.ConfigFilePath = c.String("config")
 	param.Dest = c.String("o")
 	param.OutTestData = c.String("out-testdata")
@@ -66,14 +75,11 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con
 	if cmd := c.String("cmd"); cmd != "" {
 		param.Commands = strings.Split(cmd, ",")
 	}
-	param.LogColor = os.Getenv("AQUA_LOG_COLOR")
 	param.AQUAVersion = ldFlags.Version
 	param.AquaCommitHash = ldFlags.Commit
 	param.RootDir = config.GetRootDir(osenv.New())
 	homeDir, _ := os.UserHomeDir()
 	param.HomeDir = homeDir
-	log.SetLevel(param.LogLevel, logE)
-	log.SetColor(param.LogColor, logE)
 	param.MaxParallelism = config.GetMaxParallelism(os.Getenv("AQUA_MAX_PARALLELISM"), logE)
 	param.GlobalConfigFilePaths = finder.ParseGlobalConfigFilePaths(wd, os.Getenv("AQUA_GLOBAL_CONFIG"))
 	param.Deep = c.Bool("deep")
@@ -84,7 +90,18 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con
 	param.ProgressBar = os.Getenv("AQUA_PROGRESS_BAR") == "true"
 	param.Tags = parseTags(strings.Split(c.String("tags"), ","))
 	param.ExcludedTags = parseTags(strings.Split(c.String("exclude-tags"), ","))
+}
 
+func setLogParams(c *cli.Context, param *config.Param, logE *logrus.Entry) {
+	if logLevel := c.String("log-level"); logLevel != "" {
+		param.LogLevel = logLevel
+	}
+	param.LogColor = os.Getenv("AQUA_LOG_COLOR")
+	log.SetLevel(param.LogLevel, logE)
+	log.SetColor(param.LogColor, logE)
+}
+
+func setEnvParams(param *config.Param) error { //nolint:cyclop
 	if a := os.Getenv("AQUA_DISABLE_LAZY_INSTALL"); a != "" {
 		disableLazyInstall, err := strconv.ParseBool(a)
 		if err != nil {
@@ -108,6 +125,20 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con
 			}
 		}
 	}
+	if a := os.Getenv("AQUA_VACUUM_DAYS"); a != "" {
+		vacuumDays, err := strconv.Atoi(a)
+		if err != nil {
+			return fmt.Errorf("parse the environment variable AQUA_VACUUM_DAYS as an integer: %w", err)
+		}
+		if vacuumDays <= 0 {
+			return fmt.Errorf("the environment variable AQUA_VACUUM_DAYS must be a positive integer: %d", vacuumDays)
+		}
+		param.VacuumDays = vacuumDays
+	}
+	return nil
+}
+
+func setChecksumParams(param *config.Param) error {
 	if a := os.Getenv("AQUA_CHECKSUM"); a != "" {
 		chksm, err := strconv.ParseBool(a)
 		if err != nil {

pkg/cli/vacuum/command.go

@@ -0,0 +1,111 @@
+package vacuum
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/aquaproj/aqua/v2/pkg/cli/profile"
+	"github.com/aquaproj/aqua/v2/pkg/cli/util"
+	"github.com/aquaproj/aqua/v2/pkg/config"
+	"github.com/aquaproj/aqua/v2/pkg/controller"
+	"github.com/suzuki-shunsuke/logrus-error/logerr"
+	"github.com/urfave/cli/v2"
+)
+
+const description = `Perform vacuuming tasks.
+
+Enable vacuuming by setting the AQUA_VACUUM_DAYS environment variable to a value greater than 0.
+This command removes versions of packages that have not been used for the specified number of days.
+
+You can list all packages managed by the vacuum system or only expired packages.
+
+	# Show all packages managed by the vacuum system
+	$ aqua vacuum show
+
+	# Show only expired packages
+	$ aqua vacuum show --expired
+	$ aqua vacuum show -e
+
+	# Run vacuum cleaning
+	$ aqua vacuum run
+`
+
+type command struct {
+	r *util.Param
+}
+
+func New(r *util.Param) *cli.Command {
+	i := &command{
+		r: r,
+	}
+	return &cli.Command{
+		Name:        "vacuum",
+		Usage:       "Operate vacuuming tasks (If AQUA_VACUUM_DAYS is set)",
+		Aliases:     []string{"v"},
+		Description: description,
+		Subcommands: []*cli.Command{
+			{
+				Name:    "show",
+				Aliases: []string{"s"},
+				Usage:   "Show packages managed by vacuum system",
+				Action:  i.action,
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:    "expired",
+						Usage:   "Show only expired packages",
+						Aliases: []string{"e"},
+					},
+				},
+			},
+			{
+				Name:    "run",
+				Aliases: []string{"r"},
+				Usage:   "Run vacuum cleaning",
+				Action:  i.action,
+			},
+		},
+	}
+}
+
+func (i *command) action(c *cli.Context) error {
+	profiler, err := profile.Start(c)
+	if err != nil {
+		return fmt.Errorf("start CPU Profile or tracing: %w", err)
+	}
+	defer profiler.Stop()
+
+	logE := i.r.LogE
+
+	param := &config.Param{}
+	if err := util.SetParam(c, logE, "vacuum", param, i.r.LDFlags); err != nil {
+		return fmt.Errorf("parse the command line arguments: %w", err)
+	}
+
+	if param.VacuumDays == 0 {
+		return errors.New("vacuum is not enabled, please set the AQUA_VACUUM_DAYS environment variable")
+	}
+
+	ctrl := controller.InitializeVacuumCommandController(c.Context, param, http.DefaultClient, i.r.Runtime)
+
+	defer func() {
+		if err := ctrl.Close(logE); err != nil {
+			logerr.WithError(logE, err).Error("close vacuum controller")
+		}
+	}()
+
+	if c.Command.Name == "show" {
+		if err := ctrl.ListPackages(c.Context, logE, c.Bool("expired")); err != nil {
+			return fmt.Errorf("show packages: %w", err)
+		}
+		return nil
+	}
+
+	if c.Command.Name == "run" {
+		if err := ctrl.Vacuum(c.Context, logE); err != nil {
+			return fmt.Errorf("run: %w", err)
+		}
+	}
+
+	return nil
+}

pkg/config/package.go

@@ -31,15 +31,15 @@ func (p *Package) ExePath(rootDir string, file *registry.File, rt *runtime.Runti
 		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, p.Package.Version, "bin", file.Name), nil
 	}
 
-	pkgPath, err := p.PkgPath(rootDir, rt)
+	pkgPath, err := p.PkgPath(rt)
 	if err != nil {
 		return "", err
 	}
 	fileSrc, err := p.fileSrc(file, rt)
 	if err != nil {
 		return "", fmt.Errorf("get a file path: %w", err)
 	}
-	return filepath.Join(pkgPath, fileSrc), nil
+	return filepath.Join(rootDir, pkgPath, fileSrc), nil
 }
 
 func (p *Package) RenderAsset(rt *runtime.Runtime) (string, error) {
@@ -80,7 +80,7 @@ func (p *Package) RenderPath() (string, error) {
 	return p.RenderTemplateString(pkgInfo.GetPath(), &runtime.Runtime{})
 }
 
-func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) { //nolint:cyclop
+func (p *Package) PkgPath(rt *runtime.Runtime) (string, error) { //nolint:cyclop
 	pkgInfo := p.PackageInfo
 	pkg := p.Package
 	assetName, err := p.RenderAsset(rt)
@@ -89,23 +89,23 @@ func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) {
 	}
 	switch pkgInfo.Type {
 	case PkgInfoTypeGitHubArchive:
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version), nil
+		return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version), nil
 	case PkgInfoTypeGoBuild:
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, "src"), nil
+		return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, "src"), nil
 	case PkgInfoTypeGoInstall:
 		p, err := p.RenderPath()
 		if err != nil {
 			return "", fmt.Errorf("render Go Module Path: %w", err)
 		}
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, p, pkg.Version, "bin"), nil
+		return filepath.Join("pkgs", pkgInfo.Type, p, pkg.Version, "bin"), nil
 	case PkgInfoTypeCargo:
 		registry := "crates.io"
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, registry, pkgInfo.Crate, strings.TrimPrefix(pkg.Version, "v")), nil
+		return filepath.Join("pkgs", pkgInfo.Type, registry, pkgInfo.Crate, strings.TrimPrefix(pkg.Version, "v")), nil
 	case PkgInfoTypeGitHubContent, PkgInfoTypeGitHubRelease:
 		if pkgInfo.RepoOwner == "aquaproj" && (pkgInfo.RepoName == "aqua" || pkgInfo.RepoName == "aqua-proxy") {
-			return filepath.Join(rootDir, "internal", "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
+			return filepath.Join("internal", "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
 		}
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
+		return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
 	case PkgInfoTypeHTTP:
 		uS, err := p.RenderURL(rt)
 		if err != nil {
@@ -115,7 +115,7 @@ func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) {
 		if err != nil {
 			return "", fmt.Errorf("parse the URL: %w", err)
 		}
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, u.Host, u.Path), nil
+		return filepath.Join("pkgs", pkgInfo.Type, u.Host, u.Path), nil
 	}
 	return "", nil
 }
@@ -294,6 +294,7 @@ type Param struct {
 	Installed                         bool
 	PolicyConfigFilePaths             []string
 	Commands                          []string
+	VacuumDays                        int
 }
 
 func appendExt(s, format string) string {