feat: ibmcloud be to use local setup rather than k8 kind cluster

Signed-off-by: aavarghese <[email protected]>
IBM · Jan 8, 2025 · 04c5519 · 04c5519
1 parent b3f948d
commit 04c5519
Show file tree

Hide file tree

Showing 38 changed files with 374 additions and 164 deletions.
diff --git a/cmd/subcommands/component.go b/cmd/subcommands/component.go
@@ -17,4 +17,5 @@ func init() {
 	cmd.AddCommand(component.Minio())
 	cmd.AddCommand(component.Worker())
 	cmd.AddCommand(component.WorkStealer())
+	cmd.AddCommand(component.RunLocally())
 }
diff --git a/cmd/subcommands/component/run-locally.go b/cmd/subcommands/component/run-locally.go
@@ -0,0 +1,42 @@
+package component
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+	"lunchpail.io/cmd/options"
+	"lunchpail.io/pkg/build"
+	"lunchpail.io/pkg/runtime"
+)
+
+type RunLocallyOptions struct {
+	Component string
+	LLIR      string
+	build.LogOptions
+}
+
+func AddRunLocallyOptions(cmd *cobra.Command) *RunLocallyOptions {
+	options := RunLocallyOptions{}
+	cmd.Flags().StringVarP(&options.Component, "component", "", "", "")
+	cmd.Flags().StringVar(&options.LLIR, "llir", "", "")
+	cmd.MarkFlagRequired("component")
+	cmd.MarkFlagRequired("llir")
+	return &options
+}
+
+func RunLocally() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "run-locally",
+		Short: "Commands for running a component locally",
+		Long:  "Commands for running a component locally",
+	}
+
+	runOpts := AddRunLocallyOptions(cmd)
+	options.AddLogOptions(cmd)
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		return runtime.RunLocally(context.Background(), runOpts.Component, runOpts.LLIR, runOpts.LogOptions)
+	}
+
+	return cmd
+}
diff --git a/cmd/subcommands/component/worker/run.go b/cmd/subcommands/component/worker/run.go
@@ -3,6 +3,7 @@ package worker
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -64,6 +65,7 @@ func Run() *cobra.Command {
 			PollingInterval:   pollingInterval,
 			LogOptions:        *logOpts,
 			RunContext:        run.ForStep(step).ForPool(poolName).ForWorker(workerName),
+			WorkerStartTime:   time.Now(),
 		})
 	}
 

diff --git a/cmd/subcommands/queue/cat.go b/cmd/subcommands/queue/cat.go
@@ -41,7 +41,7 @@ func Cat() *cobra.Command {
 			return err
 		}
 
-		return queue.Qcat(ctx, backend, run, args[0], *opts.Log)
+		return queue.Qcat(ctx, backend, run, args[0], q.Spec{}, *opts.Log)
 	}
 
 	return cmd

diff --git a/cmd/subcommands/queue/drain.go b/cmd/subcommands/queue/drain.go
@@ -43,7 +43,7 @@ func Drain() *cobra.Command {
 			return err
 		}
 
-		return queue.Drain(ctx, backend, run.ForStep(step), *opts.Log)
+		return queue.Drain(ctx, backend, run.ForStep(step), q.Spec{}, *opts.Log)
 	}
 
 	return cmd

diff --git a/cmd/subcommands/queue/ls.go b/cmd/subcommands/queue/ls.go
@@ -60,7 +60,7 @@ func Ls() *cobra.Command {
 			return err
 		}
 
-		files, errors, err := queue.Ls(ctx, backend, runContext.ForStep(step), path, *opts.Log)
+		files, errors, err := queue.Ls(ctx, backend, runContext.ForStep(step), path, q.Spec{}, *opts.Log)
 		if err != nil {
 			return err
 		}

diff --git a/cmd/subcommands/queue/upload.go b/cmd/subcommands/queue/upload.go
@@ -47,7 +47,7 @@ func Upload() *cobra.Command {
 			run = rrun.Name
 		}
 
-		return queue.UploadFiles(ctx, backend, q.RunContext{RunName: run}, []upload.Upload{upload.Upload{LocalPath: args[0], Bucket: args[1]}}, *opts.Log)
+		return queue.UploadFiles(ctx, backend, q.RunContext{RunName: run}, []upload.Upload{upload.Upload{LocalPath: args[0], Bucket: args[1]}}, q.Spec{}, *opts.Log)
 	}
 
 	return cmd

diff --git a/cmd/subcommands/up.go b/cmd/subcommands/up.go
@@ -4,7 +4,9 @@ package subcommands
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -68,7 +70,12 @@ func newUpCmd() *cobra.Command {
 			return err
 		}
 
-		_, err = boot.Up(ctx, backend, boot.UpOptions{BuildOptions: *buildOpts, DryRun: dryrunFlag, Watch: watchFlag, WatchUtil: watchFlag, Inputs: args, Executable: os.Args[0], NoRedirect: noRedirect})
+		upStartTime := time.Now()
+		_, err = boot.Up(ctx, backend, boot.UpOptions{BuildOptions: *buildOpts, DryRun: dryrunFlag, Watch: watchFlag, WatchUtil: watchFlag, Inputs: args, Executable: os.Args[0], NoRedirect: noRedirect, UpStartTime: time.Now()})
+		upEndTime := time.Now()
+		if buildOpts.Verbose() {
+			fmt.Fprintf(os.Stderr, "METRICS: Took %s for running app e2e\n", util.RelTime(upStartTime, upEndTime))
+		}
 		return err
 	}
 

diff --git a/demos/data-prep-kit/code/header-cleanser/requirements.txt b/demos/data-prep-kit/code/header-cleanser/requirements.txt
@@ -3,4 +3,4 @@ scancode-toolkit-mini
 # we can probably update to 18+, but we will have to re-generate expected output as pyarrow 18 seems to have resulted in a binary format change
 pyarrow<17
 
-setuptools
+setuptools ; platform_system == 'Darwin'
diff --git a/demos/data-prep-kit/language/lang-id/requirements.txt b/demos/data-prep-kit/language/lang-id/requirements.txt
@@ -1,6 +1,6 @@
 # hmm, needed for tests at least on macos
 wheel
-setuptools
+setuptools ; platform_system == 'Darwin'
 
 fasttext-wheel==0.9.2
 langcodes==3.3.0

diff --git a/demos/data-prep-kit/universal/doc-id/requirements.txt b/demos/data-prep-kit/universal/doc-id/requirements.txt
@@ -1,2 +1,2 @@
 pyarrow<17
-setuptools
+setuptools ; platform_system == 'Darwin'
diff --git a/pkg/be/backend.go b/pkg/be/backend.go
@@ -34,7 +34,7 @@ type Backend interface {
 	InstanceCount(ctx context.Context, c lunchpail.Component, run queue.RunContext) (int, error)
 
 	// Queue properties for a given run, plus ensure access to the endpoint from this client
-	AccessQueue(ctx context.Context, run queue.RunContext, opts build.LogOptions) (endpoint, accessKeyID, secretAccessKey, bucket string, stop func(), err error)
+	AccessQueue(ctx context.Context, run queue.RunContext, queue queue.Spec, opts build.LogOptions) (endpoint, accessKeyID, secretAccessKey, bucket string, stop func(), err error)
 
 	// Return a streamer
 	Streamer(ctx context.Context, run queue.RunContext) streamer.Streamer