Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: reduce startup time and add skaffold #5970

Merged
merged 3 commits into from
Oct 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
.git
.gitignore
.github
LICENSE
README.md
Makefile
build
docs
assets
config
choco
test
test
skaffold.yaml
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,5 @@ movie.mp4
/dist
.vscode


build/_local/values.dev.yaml
46 changes: 46 additions & 0 deletions build/_local/agent-server.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
###################################
## Build
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

RUN apk --no-cache --update add ca-certificates && (rm -rf /var/cache/apk/* || 0)

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/agent-server cmd/api-server/main.go

###################################
## Debug
###################################
FROM golang:1.23-alpine AS debug

ENV GOTRACEBACK=all
RUN go install github.com/go-delve/delve/cmd/[email protected]

COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=builder /app/build/_local/agent-server /testkube/

ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/agent-server"]

###################################
## Distribution
###################################
FROM scratch AS dist

COPY LICENSE /testkube/
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=builder /app/build/_local/agent-server /testkube/

EXPOSE 8080
ENTRYPOINT ["/testkube/agent-server"]
63 changes: 63 additions & 0 deletions build/_local/testworkflow-init.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
ARG BUSYBOX_IMAGE="busybox:1.36.1-musl"

###################################
## Build testworkflow init
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go

###################################
## Build testworkflow toolkit
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go

###################################
## Debug
###################################
FROM golang:1.23-alpine AS debug

ENV GOTRACEBACK=all
RUN go install github.com/go-delve/delve/cmd/[email protected]

COPY --from=builder /app/build/_local/workflow-init /testkube/

ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/workflow-init"]

###################################
## Distribution
###################################
FROM ${BUSYBOX_IMAGE} AS dist
RUN cp -rf /bin /.tktw-bin
COPY --from=builder /app/build/_local/workflow-init /testkube/init
USER 1001
ENTRYPOINT ["/init"]

71 changes: 71 additions & 0 deletions build/_local/testworkflow-toolkit.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
ARG BUSYBOX_IMAGE="busybox:1.36.1-musl"
ARG ALPINE_IMAGE="alpine:3.20.0"
FROM ${BUSYBOX_IMAGE} AS busybox

###################################
## Build testworkflow-init
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-init

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go

###################################
## Build testworkflow-toolkit
###################################
FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-toolkit

ARG TARGETOS
ARG TARGETARCH
ARG GOMODCACHE="/root/.cache/go-build"
ARG GOCACHE="/go/pkg"
ARG SKAFFOLD_GO_GCFLAGS

RUN go install github.com/go-delve/delve/cmd/[email protected]

WORKDIR /app
COPY . .
RUN --mount=type=cache,target="$GOMODCACHE" \
--mount=type=cache,target="$GOCACHE" \
GOOS=$TARGETOS \
GOARCH=$TARGETARCH \
CGO_ENABLED=0 \
go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-toolkit cmd/testworkflow-toolkit/main.go

###################################
## Debug
###################################
FROM ${ALPINE_IMAGE} AS debug
RUN apk --no-cache add ca-certificates libssl3 git openssh-client
ENV GOTRACEBACK=all
COPY --from=builder-toolkit /go/bin/dlv /
COPY --from=busybox /bin /.tktw-bin
COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit
COPY --from=builder-init /app/build/_local/workflow-init /init
RUN adduser --disabled-password --home / --no-create-home --uid 1001 default
USER 1001
ENTRYPOINT ["/dlv", "exec", "--headless", "--accept-multiclient", "--listen=:56300", "--api-version=2", "/toolkit"]

###################################
## Distribution
###################################
FROM ${ALPINE_IMAGE} AS dist
RUN apk --no-cache add ca-certificates libssl3 git openssh-client
COPY --from=busybox /bin /.tktw-bin
COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit
COPY --from=builder-init /app/build/_local/workflow-init /init
RUN adduser --disabled-password --home / --no-create-home --uid 1001 default
USER 1001
ENTRYPOINT ["/toolkit"]
12 changes: 12 additions & 0 deletions build/_local/values.dev.tpl.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
testkube-dashboard:
enabled: false
mongodb:
enabled: false
testkube-api:
cloud:
url: testkube-enterprise-api:8089
tls:
enabled: false
existingSecret:
name: testkube-default-agent-token
key: agent-token
46 changes: 46 additions & 0 deletions docker-bake.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
variable "GOCACHE" {
default = "/go/pkg"
}
variable "GOMODCACHE" {
default = "/root/.cache/go-build"
}

group "default" {
targets = ["agent-server", "testworkflow-init", "testworkflow-toolkit"]
}

target "agent-server-meta" {}
target "agent-server" {
inherits = ["agent-server-meta"]
context="."
dockerfile = "build/_local/agent-server.Dockerfile"
platforms = ["linux/arm64"]
args = {
GOCACHE = "${GOCACHE}"
GOMODCACHE = "${GOMODCACHE}"
}
}

target "testworkflow-init-meta" {}
target "testworkflow-init" {
inherits = ["testworkflow-init-meta"]
context="."
dockerfile = "build/_local/testworkflow-init.Dockerfile"
platforms = ["linux/arm64"]
args = {
GOCACHE = "${GOCACHE}"
GOMODCACHE = "${GOMODCACHE}"
}
}

target "testworkflow-toolkit-meta" {}
target "testworkflow-toolkit" {
inherits = ["testworkflow-toolkit-meta"]
context="."
dockerfile = "build/_local/testworkflow-toolkit.Dockerfile"
platforms = ["linux/arm64"]
args = {
GOCACHE = "${GOCACHE}"
GOMODCACHE = "${GOMODCACHE}"
}
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ require (
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/avast/retry-go/v4 v4.6.0 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down
14 changes: 13 additions & 1 deletion pkg/event/bus/nats.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,19 @@ import (
"sync"
"time"

"github.com/avast/retry-go/v4"
"github.com/nats-io/nats.go"

"github.com/kubeshop/testkube/pkg/api/v1/testkube"
"github.com/kubeshop/testkube/pkg/event/kind/common"
"github.com/kubeshop/testkube/pkg/log"
"github.com/kubeshop/testkube/pkg/utils"
)

var (
_ Bus = (*NATSBus)(nil)

NATS_RETRY_ATTEMPTS uint = 20
)

const (
Expand Down Expand Up @@ -80,7 +84,15 @@ func NewNATSEncodedConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats.
func NewNATSConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats.Conn, error) {
opts = append(opts, optsFromConfig(cfg)...)

nc, err := nats.Connect(cfg.NatsURI, opts...)
log.DefaultLogger.Infoln("Connecting to NATS")
nc, err := retry.DoWithData(
func() (*nats.Conn, error) {
return nats.Connect(cfg.NatsURI, opts...)
},
retry.DelayType(retry.FixedDelay),
retry.Delay(utils.DefaultRetryDelay),
retry.Attempts(NATS_RETRY_ATTEMPTS),
)
if err != nil {
log.DefaultLogger.Fatalw("error connecting to nats", "error", err)
return nil, err
Expand Down
3 changes: 3 additions & 0 deletions pkg/utils/consts.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package utils

import "time"

const (
// DefaultDockerRegistry is the default registry used when no registry is specified in the image name.
DefaultDockerRegistry = "https://index.docker.io/v1/"
DefaultRetryDelay = time.Second * 3
)
58 changes: 58 additions & 0 deletions skaffold.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
apiVersion: skaffold/v4beta11
kind: Config
metadata:
name: testkube-agent
build:
local:
concurrency: 1 # for testing, should be possible to run all concurrently.
artifacts:
- image: docker.io/testkube-agent-server
context: .
custom:
buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set agent-server.tags="$IMAGE" --set agent-server.target="debug" agent-server
dependencies:
dockerfile:
path: build/_local/agent-server.Dockerfile
- image: docker.io/testworkflow-init
context: .
custom:
buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-init.tags="$IMAGE" testworkflow-init
dependencies:
dockerfile:
path: build/_local/testworkflow-init.Dockerfile
- image: docker.io/testworkflow-toolkit
context: .
custom:
buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-toolkit.tags="$IMAGE" --set testworkflow-toolkit.target="debug" testworkflow-toolkit
dependencies:
dockerfile:
path: build/_local/testworkflow-toolkit.Dockerfile
deploy:
helm:
# see https://skaffold.dev/docs/renderers/helm/#skaffoldyaml-configuration
releases:
- name: testkube-agent
repo: https://kubeshop.github.io/helm-charts
remoteChart: testkube
# Alternative: Local chart - useful for when you are actively making changes to the chart.
# chartPath: /Users/you/path/to/helm-charts/charts/testkube
upgradeOnChange: true
# skipBuildDependencies: true # This implies that you need to build dependencies yourself when you make local chart changes!
namespace: tk-dev
wait: true
createNamespace: true
valuesFiles: ['build/_local/values.dev.yaml'] # IMPORTANT: You will have to copy the values.dev.tpl.yaml template to get started!
setValueTemplates:
testkube-api.image.registry: '{{.IMAGE_DOMAIN_docker_io_testkube_agent_server}}'
testkube-api.image.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testkube_agent_server}}'
testkube-api.image.tag: '{{.IMAGE_TAG_docker_io_testkube_agent_server}}@{{.IMAGE_DIGEST_docker_io_testkube_agent_server}}'
testkube-api.imageTwInit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_init}}'
testkube-api.imageTwInit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_init}}'
testkube-api.imageTwInit.tag: '{{.IMAGE_TAG_docker_io_testworkflow_init}}@{{.IMAGE_DIGEST_docker_io_testworkflow_init}}'
testkube-api.imageTwToolkit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_toolkit}}'
testkube-api.imageTwToolkit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_toolkit}}'
testkube-api.imageTwToolkit.tag: '{{.IMAGE_TAG_docker_io_testworkflow_toolkit}}@{{.IMAGE_DIGEST_docker_io_testworkflow_toolkit}}'
flags:
upgrade: ["--no-hooks"]
statusCheckDeadlineSeconds: 300
tolerateFailuresUntilDeadline: true
Loading