From e81412e5933f6e5489435a8dafb64e8f77bd0ccd Mon Sep 17 00:00:00 2001 From: Ashish <93992470+ashpect@users.noreply.github.com> Date: Wed, 14 Aug 2024 20:37:43 +0530 Subject: [PATCH] Refactored (reduced factorial languages combos) (#26) * refactor * rename file * refacored * patch * might work * retrigger tests * i think it might work now --- cmd/builder/deps.go | 11 ++-- cmd/builder/dev.go | 6 +- cmd/builder/runtime.go | 18 +++--- pkg/staging/deps.go | 90 --------------------------- pkg/staging/helper.go | 46 +++++++------- pkg/staging/stages.go | 137 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 180 insertions(+), 128 deletions(-) delete mode 100644 pkg/staging/deps.go create mode 100644 pkg/staging/stages.go diff --git a/cmd/builder/deps.go b/cmd/builder/deps.go index 47646de..449c359 100644 --- a/cmd/builder/deps.go +++ b/cmd/builder/deps.go @@ -21,14 +21,16 @@ func NewDepsCmd(o *DepOptions) *cobra.Command { Use: "deps", Short: "Build development dependencies base image for MetaCall", RunE: func(cmd *cobra.Command, args []string) error { - base := cmd.Context().Value(baseKey{}).(llb.State) - depsBase := staging.DepsBase(base, branch, args) - depsBase, err := o.Run(depsBase) + base := cmd.Context().Value(baseKey{}).(llb.State) // Image base from arg + depsBase := staging.DepsBase(base, branch, args) // Get base images for lang + + finalImage := staging.MergeStates(depsBase) // Merge all base images + finalImage, err := o.Run(finalImage) if err != nil { return err } - cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, depsBase)) + cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, finalImage)) return nil }, @@ -40,6 +42,7 @@ func NewDepsCmd(o *DepOptions) *cobra.Command { } func (do *DepOptions) Run(depsBase llb.State) (llb.State, error) { + // Add here : Any additional stuff if needed to be done // return depsBase.Dir("/").Run(llb.Shlex("rm -rf /usr/local/metacall")).Root(), nil return depsBase, nil } diff --git a/cmd/builder/dev.go b/cmd/builder/dev.go index 378191d..7706274 100644 --- a/cmd/builder/dev.go +++ b/cmd/builder/dev.go @@ -23,12 +23,14 @@ func NewDevCmd(o *DevOptions) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { base := cmd.Context().Value(baseKey{}).(llb.State) devBase := staging.DevBase(base, branch, args) - devBase, err := o.Run(devBase) + + finalImage := staging.MergeStates(devBase) + finalImage, err := o.Run(finalImage) if err != nil { return err } - cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, devBase)) + cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, finalImage)) return nil }, diff --git a/cmd/builder/runtime.go b/cmd/builder/runtime.go index df1dc28..b3b2986 100644 --- a/cmd/builder/runtime.go +++ b/cmd/builder/runtime.go @@ -2,7 +2,7 @@ package builder import ( "context" - + "github.com/metacall/builder/pkg/staging" "github.com/moby/buildkit/client/llb" "github.com/spf13/cobra" @@ -22,21 +22,21 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { Short: "Build runtime image for MetaCall", RunE: func(cmd *cobra.Command, args []string) error { if o.RuntimeImageFlags.MetacallCli { - args = append(args,"node") + args = append(args, "node") } base := cmd.Context().Value(baseKey{}).(llb.State) - devBase := staging.RemoveBuild(staging.DevBase(base, branch, []string{})) - devBaseLang := staging.RemoveBuild(staging.DevBase(base, branch, args)) - runtimeBase := staging.RuntimeBase(base, branch, args) - diffed := llb.Diff(devBase, devBaseLang) - runtime := llb.Merge([]llb.State{runtimeBase, diffed}) + devBaseLang := staging.DevBase(base, branch, args) + devImage := staging.MergeStates(devBaseLang) + + runtimeLang:= staging.RuntimeBase(base, branch, args) + runtimeImage := staging.MergeStates(runtimeLang) if o.RuntimeImageFlags.MetacallCli { - runtime = staging.AddCli(devBaseLang, runtime) + runtimeImage = staging.AddCli(devImage, runtimeImage) } - runtime, err := o.Run(runtime) + runtime, err := o.Run(runtimeImage) if err != nil { return err } diff --git a/pkg/staging/deps.go b/pkg/staging/deps.go deleted file mode 100644 index 9c9fb81..0000000 --- a/pkg/staging/deps.go +++ /dev/null @@ -1,90 +0,0 @@ -package staging - -import ( - "github.com/metacall/builder/pkg/env" - "github.com/moby/buildkit/client/llb" -) - -var languageMap = map[string]string{ - "base": "base", - "py": "python", - "rb": "ruby", - // "netcore": "netcore", - // "netcore2": "netcore2", - // "netcore5": "netcore5", - // "netcore7": "netcore7", - // "rapidjson": "rapidjson", - // "funchook": "funchook", - // "v8": "v8", - // "v8rep54": "v8rep54", - // "v8rep57": "v8rep57", - // "v8rep58": "v8rep58", - // "v8rep52": "v8rep52", - // "v8rep51": "v8rep51", - "node": "nodejs", - "ts": "typescript", - // "file": "file", - // "rpc": "rpc", - "wasm": "wasm", - "java": "java", - "c": "c", - "cob": "cobol", - "go": "go", - "rs": "rust", - // "backtrace" : "backtrace", - // "sandbox" : "sandbox", -} - -func DepsBase(base llb.State, branch string, args []string) llb.State { - cmdArgs, err := validateArgs(args) - if err != nil { - panic(err) - } - - return env.New(base). - DepsEnv(). - Base(). - MetaCallClone(branch). - MetacallEnvBase(cmdArgs). - Root() -} - -func DevBase(base llb.State, branch string, args []string) llb.State { - - newllb := DepsBase(base, branch, args) - - cmdArgs, err := validateArgs(args) - if err != nil { - panic(err) - } - - return env.New(newllb). - DevEnv(). - MetaCallConfigure(cmdArgs). - MetaCallBuild(cmdArgs). - Root() -} - -func RuntimeBase(base llb.State, branch string, args []string) llb.State { - - cmdArgs, err := validateArgs(args) - if err != nil { - panic(err) - } - - return env.New(base). - RuntimeEnv(). - Base(). - MetaCallClone(branch). - MetacallRuntime(cmdArgs). - Root() -} - -func AddCli(src llb.State, dst llb.State) llb.State { - return dst.With(copyFrom(src, "/usr/local/bin/metacallcli*", "/usr/local/bin/metacallcli")) -} - -func RemoveBuild(state llb.State) llb.State { - return state - // return state.File(llb.Rm("/usr/local/bin/metacall")) -} diff --git a/pkg/staging/helper.go b/pkg/staging/helper.go index 864474f..6331357 100644 --- a/pkg/staging/helper.go +++ b/pkg/staging/helper.go @@ -2,17 +2,17 @@ package staging import ( "errors" - "strings" "github.com/moby/buildkit/client/llb" ) -func validateArgs(args []string) (string, error) { +func validateArgs(args []string) ([]string, error) { cmdArgs := []string{} + // Better is to go through map key and then match , will keep args in one order for _, arg := range args { lang, ok := languageMap[arg] if !ok { - return "", errors.New("Invalid language: " + arg) + return []string{}, errors.New("Invalid language: " + arg) } isExists := false for _, str := range cmdArgs { @@ -24,7 +24,7 @@ func validateArgs(args []string) (string, error) { cmdArgs = append(cmdArgs, lang) } } - return strings.Join(cmdArgs, " "), nil + return cmdArgs, nil } func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption { @@ -41,22 +41,22 @@ func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.St })) } -// func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State { -// return dst.With( -// copyMultiple(src, srcpaths, dstpath), -// ) -// } - -// func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption { -// var stateOptions []llb.StateOption -// for _, srcPath := range srcPaths { -// stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath)) -// } - -// return func(s llb.State) llb.State { -// for _, stateOption := range stateOptions { -// s = stateOption(s) -// } -// return s -// } -// } +func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State { + return dst.With( + copyMultiple(src, srcpaths, dstpath), + ) +} + +func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption { + var stateOptions []llb.StateOption + for _, srcPath := range srcPaths { + stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath)) + } + + return func(s llb.State) llb.State { + for _, stateOption := range stateOptions { + s = stateOption(s) + } + return s + } +} diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go new file mode 100644 index 0000000..710090c --- /dev/null +++ b/pkg/staging/stages.go @@ -0,0 +1,137 @@ +package staging + +import ( + "github.com/metacall/builder/pkg/env" + "github.com/moby/buildkit/client/llb" +) + +var languageMap = map[string]string{ + "base": "base", + "py": "python", + "rb": "ruby", + // "netcore": "netcore", + // "netcore2": "netcore2", + // "netcore5": "netcore5", + // "netcore7": "netcore7", + // "rapidjson": "rapidjson", + // "funchook": "funchook", + // "v8": "v8", + // "v8rep54": "v8rep54", + // "v8rep57": "v8rep57", + // "v8rep58": "v8rep58", + // "v8rep52": "v8rep52", + // "v8rep51": "v8rep51", + "node": "nodejs", + "ts": "typescript", + // "file": "file", + // "rpc": "rpc", + "wasm": "wasm", + "java": "java", + "c": "c", + "cob": "cobol", + "go": "go", + "rs": "rust", + // "backtrace" : "backtrace", + // "sandbox" : "sandbox", +} + +func DepsBase(base llb.State, branch string, args []string) map[string]llb.State { + cmdArgs, err := validateArgs(args) + if err != nil { + panic(err) + } + + m := make(map[string]llb.State, len(cmdArgs)) + + envBase := env.New(base). + DepsEnv(). + Base(). + MetaCallClone(branch) + + if len(cmdArgs) == 0 { + foo := envBase + depsLang := foo.MetacallEnvBase("").Root() + m["emptyBase"] = depsLang + + } else { + for _, arg := range cmdArgs { + foo := envBase + depsLang := foo.MetacallEnvBase(arg).Root() + m[arg] = depsLang + } + } + + return m +} + +func DevBase(base llb.State, branch string, args []string) map[string]llb.State { + + langMapDev := DepsBase(base, branch, args) + + if value, exists := langMapDev["emptyBase"]; exists { + langDev := env.New(value). + DevEnv(). + MetaCallConfigure(""). + MetaCallBuild(""). + Root() + + langMapDev["emptyBase"] = langDev + + } else { + for lang, langDeps := range langMapDev { + langDev := env.New(langDeps). + DevEnv(). + MetaCallConfigure(lang). + MetaCallBuild(lang). + Root() + + langMapDev[lang] = langDev + } + } + + return langMapDev +} + +func RuntimeBase(base llb.State, branch string, args []string) map[string]llb.State { + + langMapDev := DevBase(base, branch, args) + emptyMapDev := DevBase(base, branch, []string{}) + + // Empty base to take diff from + emptyDevBase := emptyMapDev["emptyBase"] + emptyDevBase = RemoveBuild(emptyDevBase) + + // Runtime base + runtimeBase := env.New(base). + RuntimeEnv(). + Base(). + MetaCallClone(branch) + + for lang, langDev := range langMapDev { + foo := runtimeBase + langRuntimeBase := foo.MetacallRuntime(lang).Root() + + diffed := llb.Diff(emptyDevBase, langDev) + langRuntime := llb.Merge([]llb.State{langRuntimeBase, diffed}) + langMapDev[lang] = langRuntime + } + + return langMapDev +} + +func AddCli(src llb.State, dst llb.State) llb.State { + return dst.With(copyFrom(src, "/usr/local/bin/metacallcli*", "/usr/local/bin/metacallcli")) +} + +func RemoveBuild(state llb.State) llb.State { + // return state + return state.File(llb.Rm("/usr/local/metacall")) +} + +func MergeStates(individualLangStates map[string]llb.State) llb.State { + states := []llb.State{} + for _, state := range individualLangStates { + states = append(states, state) + } + return llb.Merge(states) +}