feat: cli option to enable the new action cache

cmd/input.go

@@ -56,6 +56,7 @@ type Input struct {
 	matrix                             []string
 	actionCachePath                    string
 	logPrefixJobID                     bool
+	useNewActionCache                  bool
 }
 
 func (i *Input) resolve(path string) string {

cmd/root.go

@@ -96,6 +96,7 @@ func Execute(ctx context.Context, version string) {
 	rootCmd.PersistentFlags().StringVarP(&input.cacheServerAddr, "cache-server-addr", "", common.GetOutboundIP().String(), "Defines the address to which the cache server binds.")
 	rootCmd.PersistentFlags().Uint16VarP(&input.cacheServerPort, "cache-server-port", "", 0, "Defines the port where the artifact server listens. 0 means a randomly available port.")
 	rootCmd.PersistentFlags().StringVarP(&input.actionCachePath, "action-cache-path", "", filepath.Join(CacheHomeDir, "act"), "Defines the path where the actions get cached and host workspaces created.")
+	rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally")
 	rootCmd.SetArgs(args())
 
 	if err := rootCmd.Execute(); err != nil {
@@ -613,6 +614,11 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str
 			ReplaceGheActionTokenWithGithubCom: input.replaceGheActionTokenWithGithubCom,
 			Matrix:                             matrixes,
 		}
+		if input.useNewActionCache {
+			config.ActionCache = &runner.GoGitActionCache{
+				Path: CacheHomeDir,
+			}
+		}
 		r, err := runner.New(config)
 		if err != nil {
 			return err

pkg/container/docker_run.go

@@ -240,8 +240,8 @@
 
 	archMapper := map[string]string{
 		"x86_64":  "X64",
 		"386":     "X86",
 		"aarch64": "ARM64",
 	}
 	if arch, ok := archMapper[info.Architecture]; ok {
 		return arch
@@ -345,13 +345,13 @@
 		return nil, nil, fmt.Errorf("Cannot parse container options: '%s': '%w'", input.Options, err)
 	}
 
 	if len(copts.netMode.Value()) == 0 {
 		if err = copts.netMode.Set("host"); err != nil {
 			return nil, nil, fmt.Errorf("Cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err)
 		}
 	}
 
 	containerConfig, err := parse(flags, copts, runtime.GOOS)
 	if err != nil {
 		return nil, nil, fmt.Errorf("Cannot process container options: '%s': '%w'", input.Options, err)
 	}
@@ -586,7 +586,7 @@
 		}
 		exp := regexp.MustCompile(`\d+\n`)
 		found := exp.FindString(sid)
 		id, err := strconv.ParseInt(strings.TrimSpace(found), 10, 32)
 		if err != nil {
 			return nil
 		}
@@ -649,12 +649,30 @@
 	}
 }
 
 func (cr *containerReference) CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error {
-	err := cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, types.CopyToContainerOptions{})
+	// Mkdir
+	buf := &bytes.Buffer{}
+	tw := tar.NewWriter(buf)
+	_ = tw.WriteHeader(&tar.Header{
+		Name:     destPath,
+		Mode:     777,
+		Typeflag: tar.TypeDir,
+	})
+	tw.Close()
+	err := cr.cli.CopyToContainer(ctx, cr.id, "/", buf, types.CopyToContainerOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to mkdir to copy content to container: %w", err)
+	}
+	// Copy Content
+	err = cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, types.CopyToContainerOptions{})
 	if err != nil {
 		return fmt.Errorf("failed to copy content to container: %w", err)
 	}
+	// If this fails, then folders have wrong permissions on non root container
+	if cr.UID != 0 || cr.GID != 0 {
+		_ = cr.Exec([]string{"chown", "-R", fmt.Sprintf("%d:%d", cr.UID, cr.GID), destPath}, nil, "0", "")(ctx)
+	}
 	return nil
 }
 
 func (cr *containerReference) copyDir(dstPath string, srcPath string, useGitIgnore bool) common.Executor {

pkg/runner/action.go

@@ -44,7 +44,7 @@
 	reader, closer, err := readFile("action.yml")
 	if os.IsNotExist(err) {
 		reader, closer, err = readFile("action.yaml")
-		if err != nil {
+		if os.IsNotExist(err) {
 			if _, closer, err2 := readFile("Dockerfile"); err2 == nil {
 				closer.Close()
 				action := &model.Action{
@@ -91,6 +91,8 @@
 				}
 			}
 			return nil, err
+		} else if err != nil {
+			return nil, err
 		}
 	} else if err != nil {
 		return nil, err
@@ -110,6 +112,17 @@
 	if stepModel.Type() != model.StepTypeUsesActionRemote {
 		return nil
 	}
+
+	if rc.Config != nil && rc.Config.ActionCache != nil {
+		raction := step.(*stepActionRemote)
+		ta, err := rc.Config.ActionCache.GetTarArchive(ctx, raction.cacheDir, raction.resolvedSha, "")
+		if err != nil {
+			return err
+		}
+		defer ta.Close()
+		return rc.JobContainer.CopyTarStream(ctx, containerActionDir, ta)
+	}
+
 	if err := removeGitIgnore(ctx, actionDir); err != nil {
 		return err
 	}
@@ -261,9 +274,16 @@
 			if localAction {
 				buildContext, err = rc.JobContainer.GetContainerArchive(ctx, contextDir+"/.")
 				if err != nil {
 					return err
 				}
 				defer buildContext.Close()
+			} else if rc.Config.ActionCache != nil {
+				rstep := step.(*stepActionRemote)
+				buildContext, err = rc.Config.ActionCache.GetTarArchive(ctx, rstep.cacheDir, rstep.resolvedSha, contextDir)
+				if err != nil {
+					return err
+				}
+				defer buildContext.Close()
 			}
 			prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{
 				ContextDir:   contextDir,
@@ -362,8 +382,8 @@
 	binds, mounts := rc.GetBindsAndMounts()
 	networkMode := fmt.Sprintf("container:%s", rc.jobContainerName())
 	if rc.IsHostEnv(ctx) {
 		networkMode = "default"
 	}
 	stepContainer := container.NewContainer(&container.NewContainerInput{
 		Cmd:         cmd,
 		Entrypoint:  entrypoint,

pkg/runner/runner.go

@@ -58,6 +58,7 @@
 	ReplaceGheActionWithGithubCom      []string                   // Use actions from GitHub Enterprise instance to GitHub
 	ReplaceGheActionTokenWithGithubCom string                     // Token of private action repo on GitHub.
 	Matrix                             map[string]map[string]bool // Matrix config to run
+	ActionCache                        ActionCache
 }
 
 type caller struct {
@@ -94,8 +95,8 @@
 		}
 		eventJSON, err := json.Marshal(eventMap)
 		if err != nil {
 			return nil, err
 		}
 		runner.eventJSON = string(eventJSON)
 	}
 	return runner, nil
@@ -153,7 +154,7 @@
 
 				var matrixes []map[string]interface{}
 				if m, err := job.GetMatrixes(); err != nil {
 					log.Errorf("Error while get job's matrix: %v", err)
 				} else {
 					log.Debugf("Job Matrices: %v", m)
 					log.Debugf("Runner Matrices: %v", runner.config.Matrix)
@@ -185,8 +186,8 @@
 						executor, err := rc.Executor()
 
 						if err != nil {
 							return err
 						}
 
 						return executor(common.WithJobErrorContainer(WithJobLogger(ctx, rc.Run.JobID, jobName, rc.Config, &rc.Masks, matrix)))
 					})
@@ -195,7 +196,7 @@
 			}
 			ncpu := runtime.NumCPU()
 			if 1 > ncpu {
 				ncpu = 1
 			}
 			log.Debugf("Detected CPUs: %d", ncpu)
 			return common.NewParallelExecutor(ncpu, pipeline...)(ctx)

pkg/runner/step.go

@@ -34,6 +34,9 @@
 	stepStagePost
 )
 
+// Controls how many symlinks are resolved for local and remote Actions
+const maxSymlinkDepth = 10
+
 func (s stepStage) String() string {
 	switch s {
 	case stepStagePre:
@@ -50,8 +53,8 @@
 	env := map[string]string{}
 	err := rc.JobContainer.UpdateFromEnv(path.Join(rc.JobContainer.GetActPath(), fileName), &env)(ctx)
 	if err != nil {
 		return err
 	}
 	for k, v := range env {
 		setter(ctx, map[string]string{"name": k}, v)
 	}
@@ -173,7 +176,7 @@
 		}
 		err = processRunnerEnvFileCommand(ctx, outputFileCommand, rc, rc.setOutput)
 		if err != nil {
 			return err
 		}
 		err = rc.UpdateExtraPath(ctx, path.Join(actPath, pathFileCommand))
 		if err != nil {
@@ -189,9 +192,9 @@
 func evaluateStepTimeout(ctx context.Context, exprEval ExpressionEvaluator, stepModel *model.Step) (context.Context, context.CancelFunc) {
 	timeout := exprEval.Interpolate(ctx, stepModel.TimeoutMinutes)
 	if timeout != "" {
 		if timeOutMinutes, err := strconv.ParseInt(timeout, 10, 64); err == nil {
 			return context.WithTimeout(ctx, time.Duration(timeOutMinutes)*time.Minute)
 		}
 	}
 	return ctx, func() {}
 }
@@ -274,7 +277,7 @@
 
 func mergeIntoMap(step step, target *map[string]string, maps ...map[string]string) {
 	if rc := step.getRunContext(); rc != nil && rc.JobContainer != nil && rc.JobContainer.IsEnvironmentCaseInsensitive() {
 		mergeIntoMapCaseInsensitive(*target, maps...)
 	} else {
 		mergeIntoMapCaseSensitive(*target, maps...)
 	}
@@ -298,8 +301,8 @@
 		if k, ok := foldKeys[foldKey]; ok {
 			return k
 		}
 		foldKeys[strings.ToLower(foldKey)] = s
 		return s
 	}
 	for _, m := range maps {
 		for k, v := range m {
@@ -307,3 +310,13 @@
 		}
 	}
 }
+
+func symlinkJoin(filename, sym, parent string) (string, error) {
+	dir := path.Dir(filename)
+	dest := path.Join(dir, sym)
+	prefix := path.Clean(parent) + "/"
+	if strings.HasPrefix(dest, prefix) || prefix == "./" {
+		return dest, nil
+	}
+	return "", fmt.Errorf("symlink tries to access file '%s' outside of '%s'", strings.ReplaceAll(dest, "'", "''"), strings.ReplaceAll(parent, "'", "''"))
+}

pkg/runner/step_action_local.go

@@ -3,7 +3,10 @@
 import (
 	"archive/tar"
 	"context"
+	"errors"
+	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -42,15 +45,31 @@
 		localReader := func(ctx context.Context) actionYamlReader {
 			_, cpath := getContainerActionPaths(sal.Step, path.Join(actionDir, ""), sal.RunContext)
 			return func(filename string) (io.Reader, io.Closer, error) {
-				tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, path.Join(cpath, filename))
-				if err != nil {
-					return nil, nil, os.ErrNotExist
+				spath := path.Join(cpath, filename)
+				for i := 0; i < maxSymlinkDepth; i++ {
+					tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, spath)
+					if errors.Is(err, fs.ErrNotExist) {
+						return nil, nil, err
+					} else if err != nil {
+						return nil, nil, fs.ErrNotExist
+					}
+					treader := tar.NewReader(tars)
+					header, err := treader.Next()
+					if errors.Is(err, io.EOF) {
+						return nil, nil, os.ErrNotExist
+					} else if err != nil {
+						return nil, nil, err
+					}
+					if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink {
+						spath, err = symlinkJoin(spath, header.Linkname, cpath)
+						if err != nil {
+							return nil, nil, err
+						}
+					} else {
+						return treader, tars, nil
+					}
 				}
-				treader := tar.NewReader(tars)
-				if _, err := treader.Next(); err != nil {
-					return nil, nil, os.ErrNotExist
-				}
-				return treader, tars, nil
+				return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath)
 			}
 		}
 

pkg/runner/step_action_remote.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"archive/tar"
 	"context"
 	"errors"
 	"fmt"
@@ -28,6 +29,8 @@
 	action              *model.Action
 	env                 map[string]string
 	remoteAction        *remoteAction
+	cacheDir            string
+	resolvedSha         string
 }
 
 var (
@@ -60,6 +63,46 @@
 				github.Token = sar.RunContext.Config.ReplaceGheActionTokenWithGithubCom
 			}
 		}
+		if sar.RunContext.Config.ActionCache != nil {
+			cache := sar.RunContext.Config.ActionCache
+
+			var err error
+			sar.cacheDir = fmt.Sprintf("%s/%s", sar.remoteAction.Org, sar.remoteAction.Repo)
+			sar.resolvedSha, err = cache.Fetch(ctx, sar.cacheDir, sar.remoteAction.URL+"/"+sar.cacheDir, sar.remoteAction.Ref, github.Token)
+			if err != nil {
+				return err
+			}
+
+			remoteReader := func(ctx context.Context) actionYamlReader {
+				return func(filename string) (io.Reader, io.Closer, error) {
+					spath := filename
+					for i := 0; i < maxSymlinkDepth; i++ {
+						tars, err := cache.GetTarArchive(ctx, sar.cacheDir, sar.resolvedSha, spath)
+						if err != nil {
+							return nil, nil, os.ErrNotExist
+						}
+						treader := tar.NewReader(tars)
+						header, err := treader.Next()
+						if err != nil {
+							return nil, nil, os.ErrNotExist
+						}
+						if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink {
+							spath, err = symlinkJoin(spath, header.Linkname, ".")
+							if err != nil {
+								return nil, nil, err
+							}
+						} else {
+							return treader, tars, nil
+						}
+					}
+					return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath)
+				}
+			}
+
+			actionModel, err := sar.readAction(ctx, sar.Step, sar.resolvedSha, sar.remoteAction.Path, remoteReader(ctx), os.WriteFile)
+			sar.action = actionModel
+			return err
+		}
 
 		actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses))
 		gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{