Skip to content

Commit

Permalink
Refactored (reduced factorial languages combos) (#26)
Browse files Browse the repository at this point in the history
* refactor

* rename file

* refacored

* patch

* might work

* retrigger tests

* i think it might work now
  • Loading branch information
ashpect authored Aug 14, 2024
1 parent 78d648a commit e81412e
Show file tree
Hide file tree
Showing 6 changed files with 180 additions and 128 deletions.
11 changes: 7 additions & 4 deletions cmd/builder/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,16 @@ func NewDepsCmd(o *DepOptions) *cobra.Command {
Use: "deps",
Short: "Build development dependencies base image for MetaCall",
RunE: func(cmd *cobra.Command, args []string) error {
base := cmd.Context().Value(baseKey{}).(llb.State)
depsBase := staging.DepsBase(base, branch, args)
depsBase, err := o.Run(depsBase)
base := cmd.Context().Value(baseKey{}).(llb.State) // Image base from arg
depsBase := staging.DepsBase(base, branch, args) // Get base images for lang

finalImage := staging.MergeStates(depsBase) // Merge all base images
finalImage, err := o.Run(finalImage)
if err != nil {
return err
}

cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, depsBase))
cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, finalImage))
return nil

},
Expand All @@ -40,6 +42,7 @@ func NewDepsCmd(o *DepOptions) *cobra.Command {
}

func (do *DepOptions) Run(depsBase llb.State) (llb.State, error) {
// Add here : Any additional stuff if needed to be done
// return depsBase.Dir("/").Run(llb.Shlex("rm -rf /usr/local/metacall")).Root(), nil
return depsBase, nil
}
6 changes: 4 additions & 2 deletions cmd/builder/dev.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@ func NewDevCmd(o *DevOptions) *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error {
base := cmd.Context().Value(baseKey{}).(llb.State)
devBase := staging.DevBase(base, branch, args)
devBase, err := o.Run(devBase)

finalImage := staging.MergeStates(devBase)
finalImage, err := o.Run(finalImage)
if err != nil {
return err
}

cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, devBase))
cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, finalImage))
return nil

},
Expand Down
18 changes: 9 additions & 9 deletions cmd/builder/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package builder

import (
"context"

"github.com/metacall/builder/pkg/staging"
"github.com/moby/buildkit/client/llb"
"github.com/spf13/cobra"
Expand All @@ -22,21 +22,21 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command {
Short: "Build runtime image for MetaCall",
RunE: func(cmd *cobra.Command, args []string) error {
if o.RuntimeImageFlags.MetacallCli {
args = append(args,"node")
args = append(args, "node")
}
base := cmd.Context().Value(baseKey{}).(llb.State)
devBase := staging.RemoveBuild(staging.DevBase(base, branch, []string{}))
devBaseLang := staging.RemoveBuild(staging.DevBase(base, branch, args))
runtimeBase := staging.RuntimeBase(base, branch, args)
diffed := llb.Diff(devBase, devBaseLang)

runtime := llb.Merge([]llb.State{runtimeBase, diffed})
devBaseLang := staging.DevBase(base, branch, args)
devImage := staging.MergeStates(devBaseLang)

runtimeLang:= staging.RuntimeBase(base, branch, args)
runtimeImage := staging.MergeStates(runtimeLang)

if o.RuntimeImageFlags.MetacallCli {
runtime = staging.AddCli(devBaseLang, runtime)
runtimeImage = staging.AddCli(devImage, runtimeImage)
}

runtime, err := o.Run(runtime)
runtime, err := o.Run(runtimeImage)
if err != nil {
return err
}
Expand Down
90 changes: 0 additions & 90 deletions pkg/staging/deps.go

This file was deleted.

46 changes: 23 additions & 23 deletions pkg/staging/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,17 @@ package staging

import (
"errors"
"strings"

"github.com/moby/buildkit/client/llb"
)

func validateArgs(args []string) (string, error) {
func validateArgs(args []string) ([]string, error) {
cmdArgs := []string{}
// Better is to go through map key and then match , will keep args in one order
for _, arg := range args {
lang, ok := languageMap[arg]
if !ok {
return "", errors.New("Invalid language: " + arg)
return []string{}, errors.New("Invalid language: " + arg)
}
isExists := false
for _, str := range cmdArgs {
Expand All @@ -24,7 +24,7 @@ func validateArgs(args []string) (string, error) {
cmdArgs = append(cmdArgs, lang)
}
}
return strings.Join(cmdArgs, " "), nil
return cmdArgs, nil
}

func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption {
Expand All @@ -41,22 +41,22 @@ func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.St
}))
}

// func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State {
// return dst.With(
// copyMultiple(src, srcpaths, dstpath),
// )
// }

// func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption {
// var stateOptions []llb.StateOption
// for _, srcPath := range srcPaths {
// stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath))
// }

// return func(s llb.State) llb.State {
// for _, stateOption := range stateOptions {
// s = stateOption(s)
// }
// return s
// }
// }
func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State {
return dst.With(
copyMultiple(src, srcpaths, dstpath),
)
}

func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption {
var stateOptions []llb.StateOption
for _, srcPath := range srcPaths {
stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath))
}

return func(s llb.State) llb.State {
for _, stateOption := range stateOptions {
s = stateOption(s)
}
return s
}
}
137 changes: 137 additions & 0 deletions pkg/staging/stages.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
package staging

import (
"github.com/metacall/builder/pkg/env"
"github.com/moby/buildkit/client/llb"
)

var languageMap = map[string]string{
"base": "base",
"py": "python",
"rb": "ruby",
// "netcore": "netcore",
// "netcore2": "netcore2",
// "netcore5": "netcore5",
// "netcore7": "netcore7",
// "rapidjson": "rapidjson",
// "funchook": "funchook",
// "v8": "v8",
// "v8rep54": "v8rep54",
// "v8rep57": "v8rep57",
// "v8rep58": "v8rep58",
// "v8rep52": "v8rep52",
// "v8rep51": "v8rep51",
"node": "nodejs",
"ts": "typescript",
// "file": "file",
// "rpc": "rpc",
"wasm": "wasm",
"java": "java",
"c": "c",
"cob": "cobol",
"go": "go",
"rs": "rust",
// "backtrace" : "backtrace",
// "sandbox" : "sandbox",
}

func DepsBase(base llb.State, branch string, args []string) map[string]llb.State {
cmdArgs, err := validateArgs(args)
if err != nil {
panic(err)
}

m := make(map[string]llb.State, len(cmdArgs))

envBase := env.New(base).
DepsEnv().
Base().
MetaCallClone(branch)

if len(cmdArgs) == 0 {
foo := envBase
depsLang := foo.MetacallEnvBase("").Root()
m["emptyBase"] = depsLang

} else {
for _, arg := range cmdArgs {
foo := envBase
depsLang := foo.MetacallEnvBase(arg).Root()
m[arg] = depsLang
}
}

return m
}

func DevBase(base llb.State, branch string, args []string) map[string]llb.State {

langMapDev := DepsBase(base, branch, args)

if value, exists := langMapDev["emptyBase"]; exists {
langDev := env.New(value).
DevEnv().
MetaCallConfigure("").
MetaCallBuild("").
Root()

langMapDev["emptyBase"] = langDev

} else {
for lang, langDeps := range langMapDev {
langDev := env.New(langDeps).
DevEnv().
MetaCallConfigure(lang).
MetaCallBuild(lang).
Root()

langMapDev[lang] = langDev
}
}

return langMapDev
}

func RuntimeBase(base llb.State, branch string, args []string) map[string]llb.State {

langMapDev := DevBase(base, branch, args)
emptyMapDev := DevBase(base, branch, []string{})

// Empty base to take diff from
emptyDevBase := emptyMapDev["emptyBase"]
emptyDevBase = RemoveBuild(emptyDevBase)

// Runtime base
runtimeBase := env.New(base).
RuntimeEnv().
Base().
MetaCallClone(branch)

for lang, langDev := range langMapDev {
foo := runtimeBase
langRuntimeBase := foo.MetacallRuntime(lang).Root()

diffed := llb.Diff(emptyDevBase, langDev)
langRuntime := llb.Merge([]llb.State{langRuntimeBase, diffed})
langMapDev[lang] = langRuntime
}

return langMapDev
}

func AddCli(src llb.State, dst llb.State) llb.State {
return dst.With(copyFrom(src, "/usr/local/bin/metacallcli*", "/usr/local/bin/metacallcli"))
}

func RemoveBuild(state llb.State) llb.State {
// return state
return state.File(llb.Rm("/usr/local/metacall"))
}

func MergeStates(individualLangStates map[string]llb.State) llb.State {
states := []llb.State{}
for _, state := range individualLangStates {
states = append(states, state)
}
return llb.Merge(states)
}

0 comments on commit e81412e

Please sign in to comment.