From df24156cecde6b41cbd30aed33299b655fd8ce01 Mon Sep 17 00:00:00 2001 From: sukhman Date: Sat, 17 Aug 2024 16:15:03 +0530 Subject: [PATCH 1/4] refactor: Shift CheckAndPullImage function to Docker package from Helper package --- helper.go | 17 ----------------- lib/docker/image.go | 21 +++++++++++++++++++++ service_launchers.go | 15 ++++++++------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/helper.go b/helper.go index 4eebbcac..967d243a 100644 --- a/helper.go +++ b/helper.go @@ -14,23 +14,6 @@ import ( "google.golang.org/grpc" ) -func checkAndPullImages(imageList ...string) { - availableImages, err := docker.ListImages() - if err != nil { - utils.LogError("Main-Helper-1", err) - os.Exit(1) - } - for _, image := range imageList { - imageWithoutRepoName := strings.Replace(image, "docker.io/", "", -1) - if utils.Contains(availableImages, image) || utils.Contains(availableImages, imageWithoutRepoName) { - continue - } - utils.LogInfo("Main-Helper-2", "Image %s not present locally, pulling from DockerHUB", image) - if err = docker.DirectPull(image); err != nil { - utils.LogError("Main-Helper-3", err) - } - } -} func startGrpcServer(server *grpc.Server, port int) error { lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) diff --git a/lib/docker/image.go b/lib/docker/image.go index 088602d3..f29ff37d 100644 --- a/lib/docker/image.go +++ b/lib/docker/image.go @@ -4,11 +4,32 @@ import ( "io" "os" "os/exec" + "strings" "github.com/docker/docker/api/types" + "github.com/sdslabs/gasper/lib/utils" "golang.org/x/net/context" ) +// Check for available images and pull if not present +func CheckAndPullImages(imageList ...string) { + availableImages, err := ListImages() + if err != nil { + utils.LogError("Main-Helper-1", err) + os.Exit(1) + } + for _, image := range imageList { + imageWithoutRepoName := strings.Replace(image, "docker.io/", "", -1) + if utils.Contains(availableImages, image) || utils.Contains(availableImages, imageWithoutRepoName) { + continue + } + utils.LogInfo("Main-Helper-2", "Image %s not present locally, pulling from DockerHUB", image) + if err = DirectPull(image); err != nil { + utils.LogError("Main-Helper-3", err) + } + } +} + // ListImages function returns a list of docker images present in the system func ListImages() ([]string, error) { ctx := context.Background() diff --git a/service_launchers.go b/service_launchers.go index 6c8d5193..317f717d 100644 --- a/service_launchers.go +++ b/service_launchers.go @@ -5,6 +5,7 @@ import ( "runtime" "github.com/sdslabs/gasper/configs" + "github.com/sdslabs/gasper/lib/docker" "github.com/sdslabs/gasper/lib/utils" "github.com/sdslabs/gasper/services/appmaker" "github.com/sdslabs/gasper/services/dbmaker" @@ -59,19 +60,19 @@ var launcherBindings = map[string]*serviceLauncher{ func startDbMakerService() error { if configs.ServiceConfig.DbMaker.MySQL.PlugIn { - checkAndPullImages(configs.ImageConfig.Mysql) + docker.CheckAndPullImages(configs.ImageConfig.Mysql) setupDatabaseContainer(types.MySQL) } if configs.ServiceConfig.DbMaker.MongoDB.PlugIn { - checkAndPullImages(configs.ImageConfig.Mongodb) + docker.CheckAndPullImages(configs.ImageConfig.Mongodb) setupDatabaseContainer(types.MongoDB) } if configs.ServiceConfig.DbMaker.PostgreSQL.PlugIn { - checkAndPullImages(configs.ImageConfig.Postgresql) + docker.CheckAndPullImages(configs.ImageConfig.Postgresql) setupDatabaseContainer(types.PostgreSQL) } if configs.ServiceConfig.DbMaker.Redis.PlugIn { - checkAndPullImages(configs.ImageConfig.Redis) + docker.CheckAndPullImages(configs.ImageConfig.Redis) } return startGrpcServer(dbmaker.NewService(), configs.ServiceConfig.DbMaker.Port) } @@ -87,17 +88,17 @@ func startAppMakerService() error { configs.ImageConfig.Ruby, configs.ImageConfig.Rust, } - checkAndPullImages(images...) + docker.CheckAndPullImages(images...) return startGrpcServer(appmaker.NewService(), configs.ServiceConfig.AppMaker.Port) } func startMasterService() error { if configs.ServiceConfig.Master.MongoDB.PlugIn { - checkAndPullImages(configs.ImageConfig.Mongodb) + docker.CheckAndPullImages(configs.ImageConfig.Mongodb) setupDatabaseContainer(types.MongoDBGasper) } if configs.ServiceConfig.Master.Redis.PlugIn { - checkAndPullImages(configs.ImageConfig.Redis) + docker.CheckAndPullImages(configs.ImageConfig.Redis) setupDatabaseContainer(types.RedisGasper) } return buildHTTPServer(master.NewService(), configs.ServiceConfig.Master.Port).ListenAndServe() From 28eeff7b2919e0c5c2c8ab736e39d2472c93b388 Mon Sep 17 00:00:00 2001 From: sukhman Date: Sun, 18 Aug 2024 16:23:45 +0530 Subject: [PATCH 2/4] feat: Build Apps directly from custom images by passing docker_image --- lib/api/application.go | 6 +++--- services/appmaker/controller.go | 34 ++++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/lib/api/application.go b/lib/api/application.go index 223d6d48..c1c353fe 100644 --- a/lib/api/application.go +++ b/lib/api/application.go @@ -21,7 +21,7 @@ func setupContainer(app types.Application, storedir string, setup chan types.Res Name: app.GetName(), Image: app.GetDockerImage(), ApplicationPort: app.GetApplicationPort(), - ContainerPort: app.GetContainerPort(), + ContainerPort: app.GetContainerPort(), WorkDir: workdir, StoreDir: storedir, Env: app.GetEnvVars(), @@ -67,7 +67,7 @@ func setupContainer(app types.Application, storedir string, setup chan types.Res } // createBasicApplication spawns a new container with the application of a particular service -func createBasicApplication(app types.Application) []types.ResponseError { +func CreateBasicApplication(app types.Application) []types.ResponseError { storepath, _ := os.Getwd() storedir := filepath.Join(storepath, fmt.Sprintf("storage/%s", app.GetName())) setup := make(chan types.ResponseError) @@ -87,7 +87,7 @@ func SetupApplication(app types.Application) types.ResponseError { app.SetContainerPort(containerPort) - errList := createBasicApplication(app) + errList := CreateBasicApplication(app) for _, err := range errList { if err != nil { diff --git a/services/appmaker/controller.go b/services/appmaker/controller.go index ce10a887..db03db20 100644 --- a/services/appmaker/controller.go +++ b/services/appmaker/controller.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/sdslabs/gasper/configs" + "github.com/sdslabs/gasper/lib/api" "github.com/sdslabs/gasper/lib/cloudflare" "github.com/sdslabs/gasper/lib/docker" "github.com/sdslabs/gasper/lib/factory" @@ -68,18 +69,37 @@ func (s *server) Create(ctx context.Context, body *pb.RequestBody) (*pb.Response utils.LogError("AppMaker-Controller-1", fmt.Errorf("GenDNS instance %s is of invalid format", nameServer)) } } - + if pipeline[language] == nil { return nil, fmt.Errorf("language `%s` is not supported", language) } - resErr := pipeline[language].create(app) - if resErr != nil { - if resErr.Message() != "repository already exists" && resErr.Message() != "container already exists" { - go diskCleanup(app.GetName()) + utils.LogError("AppMaker-Controller-1", fmt.Errorf("%s", app.GetDockerImage())) + + if app.GetDockerImage() != "" { + utils.LogError("AppMaker-Controller-1", fmt.Errorf("docker img")) + docker.CheckAndPullImages(app.GetDockerImage()) + containerPort, err := utils.GetFreePort() + if err != nil { + return nil, types.NewResErr(500, "No free port available", err) } - return nil, fmt.Errorf(resErr.Error()) - } + app.SetContainerPort(containerPort) + errList := api.CreateBasicApplication(app) + for _, err := range errList { + if err != nil { + return nil, err + } + } + } else { + utils.LogError("AppMaker-Controller-1", fmt.Errorf("git img")) + resErr := pipeline[language].create(app) + if resErr != nil { + if resErr.Message() != "repository already exists" && resErr.Message() != "container already exists" { + go diskCleanup(app.GetName()) + } + return nil, fmt.Errorf(resErr.Error()) + } + } sshEntrypointIP := configs.ServiceConfig.GenSSH.EntrypointIP if len(sshEntrypointIP) == 0 { sshEntrypointIP = utils.HostIP From 082ebfca6f509391964ed3a0c44034b735a55d02 Mon Sep 17 00:00:00 2001 From: sukhman Date: Sun, 18 Aug 2024 16:56:34 +0530 Subject: [PATCH 3/4] docs: Add custom docekr_image field in request body --- docs/content/api/specs/openapi.json | 4 ++++ docs/content/api/specs/openapi.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/content/api/specs/openapi.json b/docs/content/api/specs/openapi.json index e91b38e8..d28f2544 100644 --- a/docs/content/api/specs/openapi.json +++ b/docs/content/api/specs/openapi.json @@ -133,6 +133,10 @@ "type": "string", "description": "Password required for SSH access to the application's docker container" }, + "docker_image": { + "type": "", + "description": "Docker image used in building the application's container" + }, "git": { "$ref": "#/components/schemas/Git" }, diff --git a/docs/content/api/specs/openapi.yaml b/docs/content/api/specs/openapi.yaml index 277adce9..dc8206c8 100644 --- a/docs/content/api/specs/openapi.yaml +++ b/docs/content/api/specs/openapi.yaml @@ -103,6 +103,10 @@ components: password: type: string description: Password required for SSH access to the application's docker container + docker_image: + type: string + description: Docker image used in building the application's container + example: sdsws/node:2.0 git: $ref: '#/components/schemas/Git' context: From 314f184bbb00e5a1fbf7b72760b3efd285c430f7 Mon Sep 17 00:00:00 2001 From: sukhman Date: Sun, 18 Aug 2024 17:54:31 +0530 Subject: [PATCH 4/4] refactor: Remove extra spaces --- lib/api/application.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/application.go b/lib/api/application.go index c1c353fe..6db223ec 100644 --- a/lib/api/application.go +++ b/lib/api/application.go @@ -21,7 +21,7 @@ func setupContainer(app types.Application, storedir string, setup chan types.Res Name: app.GetName(), Image: app.GetDockerImage(), ApplicationPort: app.GetApplicationPort(), - ContainerPort: app.GetContainerPort(), + ContainerPort: app.GetContainerPort(), WorkDir: workdir, StoreDir: storedir, Env: app.GetEnvVars(),