diff --git a/.dockerignore b/.dockerignore index 8da3bd1184d..9aa61bd961d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,12 @@ .git .gitignore .github -LICENSE README.md Makefile +build docs assets config choco -test \ No newline at end of file +test +skaffold.yaml diff --git a/.gitignore b/.gitignore index f8c639837d7..8dddeecf2be 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,5 @@ movie.mp4 /dist .vscode + +build/_local/values.dev.yaml diff --git a/build/_local/agent-server.Dockerfile b/build/_local/agent-server.Dockerfile new file mode 100644 index 00000000000..2dbcee1ce39 --- /dev/null +++ b/build/_local/agent-server.Dockerfile @@ -0,0 +1,46 @@ +################################### +## Build +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +RUN apk --no-cache --update add ca-certificates && (rm -rf /var/cache/apk/* || 0) + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/agent-server cmd/api-server/main.go + +################################### +## Debug +################################### +FROM golang:1.23-alpine AS debug + +ENV GOTRACEBACK=all +RUN go install github.com/go-delve/delve/cmd/dlv@v1.23.1 + +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=builder /app/build/_local/agent-server /testkube/ + +ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/agent-server"] + +################################### +## Distribution +################################### +FROM scratch AS dist + +COPY LICENSE /testkube/ +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=builder /app/build/_local/agent-server /testkube/ + +EXPOSE 8080 +ENTRYPOINT ["/testkube/agent-server"] diff --git a/build/_local/testworkflow-init.Dockerfile b/build/_local/testworkflow-init.Dockerfile new file mode 100644 index 00000000000..40ccb861f92 --- /dev/null +++ b/build/_local/testworkflow-init.Dockerfile @@ -0,0 +1,63 @@ +ARG BUSYBOX_IMAGE="busybox:1.36.1-musl" + +################################### +## Build testworkflow init +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go + +################################### +## Build testworkflow toolkit +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go + +################################### +## Debug +################################### +FROM golang:1.23-alpine AS debug + +ENV GOTRACEBACK=all +RUN go install github.com/go-delve/delve/cmd/dlv@v1.23.1 + +COPY --from=builder /app/build/_local/workflow-init /testkube/ + +ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/workflow-init"] + +################################### +## Distribution +################################### +FROM ${BUSYBOX_IMAGE} AS dist +RUN cp -rf /bin /.tktw-bin +COPY --from=builder /app/build/_local/workflow-init /testkube/init +USER 1001 +ENTRYPOINT ["/init"] + diff --git a/build/_local/testworkflow-toolkit.Dockerfile b/build/_local/testworkflow-toolkit.Dockerfile new file mode 100644 index 00000000000..99821a27beb --- /dev/null +++ b/build/_local/testworkflow-toolkit.Dockerfile @@ -0,0 +1,71 @@ +ARG BUSYBOX_IMAGE="busybox:1.36.1-musl" +ARG ALPINE_IMAGE="alpine:3.20.0" +FROM ${BUSYBOX_IMAGE} AS busybox + +################################### +## Build testworkflow-init +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-init + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go + +################################### +## Build testworkflow-toolkit +################################### +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-toolkit + +ARG TARGETOS +ARG TARGETARCH +ARG GOMODCACHE="/root/.cache/go-build" +ARG GOCACHE="/go/pkg" +ARG SKAFFOLD_GO_GCFLAGS + +RUN go install github.com/go-delve/delve/cmd/dlv@v1.23.1 + +WORKDIR /app +COPY . . +RUN --mount=type=cache,target="$GOMODCACHE" \ + --mount=type=cache,target="$GOCACHE" \ + GOOS=$TARGETOS \ + GOARCH=$TARGETARCH \ + CGO_ENABLED=0 \ + go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-toolkit cmd/testworkflow-toolkit/main.go + +################################### +## Debug +################################### +FROM ${ALPINE_IMAGE} AS debug +RUN apk --no-cache add ca-certificates libssl3 git openssh-client +ENV GOTRACEBACK=all +COPY --from=builder-toolkit /go/bin/dlv / +COPY --from=busybox /bin /.tktw-bin +COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit +COPY --from=builder-init /app/build/_local/workflow-init /init +RUN adduser --disabled-password --home / --no-create-home --uid 1001 default +USER 1001 +ENTRYPOINT ["/dlv", "exec", "--headless", "--accept-multiclient", "--listen=:56300", "--api-version=2", "/toolkit"] + +################################### +## Distribution +################################### +FROM ${ALPINE_IMAGE} AS dist +RUN apk --no-cache add ca-certificates libssl3 git openssh-client +COPY --from=busybox /bin /.tktw-bin +COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit +COPY --from=builder-init /app/build/_local/workflow-init /init +RUN adduser --disabled-password --home / --no-create-home --uid 1001 default +USER 1001 +ENTRYPOINT ["/toolkit"] diff --git a/build/_local/values.dev.tpl.yaml b/build/_local/values.dev.tpl.yaml new file mode 100644 index 00000000000..fc89193a586 --- /dev/null +++ b/build/_local/values.dev.tpl.yaml @@ -0,0 +1,12 @@ +testkube-dashboard: + enabled: false +mongodb: + enabled: false +testkube-api: + cloud: + url: testkube-enterprise-api:8089 + tls: + enabled: false + existingSecret: + name: testkube-default-agent-token + key: agent-token \ No newline at end of file diff --git a/cmd/api-server/commons/commons.go b/cmd/api-server/commons/commons.go index 1b68a3ed17b..2754634026b 100644 --- a/cmd/api-server/commons/commons.go +++ b/cmd/api-server/commons/commons.go @@ -296,7 +296,7 @@ func ReadProContext(ctx context.Context, cfg *config.Config, grpcClient cloud.Te DashboardURI: cfg.TestkubeDashboardURI, } - if grpcClient == nil { + if cfg.TestkubeProAPIKey == "" || grpcClient == nil { return proContext } diff --git a/cmd/api-server/commons/deprecated.go b/cmd/api-server/commons/deprecated.go index 08554084150..97ef3c82fed 100644 --- a/cmd/api-server/commons/deprecated.go +++ b/cmd/api-server/commons/deprecated.go @@ -77,6 +77,7 @@ func (d *deprecatedClients) Templates() templatesclientv1.Interface { return d.templates } +// TODO: Move Templates() out of Deprecation, as it's used by Webhook Payload (?) func CreateDeprecatedClients(kubeClient client.Client, namespace string) DeprecatedClients { return &deprecatedClients{ executors: executorsclientv1.NewClient(kubeClient, namespace), diff --git a/cmd/api-server/main.go b/cmd/api-server/main.go index 443c1552b06..edcc2dbb2a4 100644 --- a/cmd/api-server/main.go +++ b/cmd/api-server/main.go @@ -2,8 +2,10 @@ package main import ( "context" + "errors" "flag" "fmt" + "strings" "github.com/gofiber/fiber/v2/middleware/cors" "google.golang.org/grpc" @@ -20,8 +22,8 @@ import ( "github.com/kubeshop/testkube/pkg/event/kind/k8sevent" "github.com/kubeshop/testkube/pkg/event/kind/webhook" ws "github.com/kubeshop/testkube/pkg/event/kind/websocket" + "github.com/kubeshop/testkube/pkg/executor/output" oauth2 "github.com/kubeshop/testkube/pkg/oauth" - testworkflow2 "github.com/kubeshop/testkube/pkg/repository/testworkflow" "github.com/kubeshop/testkube/pkg/secretmanager" "github.com/kubeshop/testkube/pkg/server" "github.com/kubeshop/testkube/pkg/tcl/checktcl" @@ -29,40 +31,26 @@ import ( "github.com/kubeshop/testkube/pkg/testworkflows/executionworker/executionworkertypes" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/presets" - domainstorage "github.com/kubeshop/testkube/pkg/storage" - "github.com/kubeshop/testkube/pkg/storage/minio" - "github.com/kubeshop/testkube/internal/common" - parser "github.com/kubeshop/testkube/internal/template" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/version" "golang.org/x/sync/errgroup" - "github.com/kubeshop/testkube/pkg/cloud" - "github.com/kubeshop/testkube/pkg/repository/sequence" - "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/agent" + "github.com/kubeshop/testkube/pkg/cloud" "github.com/kubeshop/testkube/pkg/event" "github.com/kubeshop/testkube/pkg/event/bus" - kubeexecutor "github.com/kubeshop/testkube/pkg/executor" - "github.com/kubeshop/testkube/pkg/executor/client" - "github.com/kubeshop/testkube/pkg/executor/containerexecutor" - logsclient "github.com/kubeshop/testkube/pkg/logs/client" - "github.com/kubeshop/testkube/pkg/scheduler" - "github.com/kubeshop/testkube/pkg/k8sclient" "github.com/kubeshop/testkube/pkg/triggers" kubeclient "github.com/kubeshop/testkube-operator/pkg/client" testtriggersclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testtriggers/v1" testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" - deprecatedapiv1 "github.com/kubeshop/testkube/internal/app/api/deprecatedv1" apiv1 "github.com/kubeshop/testkube/internal/app/api/v1" "github.com/kubeshop/testkube/pkg/configmap" "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/reconciler" "github.com/kubeshop/testkube/pkg/secret" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowexecutor" ) @@ -90,17 +78,34 @@ func main() { commons.MustFreePort(cfg.APIServerPort) commons.MustFreePort(cfg.GraphqlPort) + commons.MustFreePort(cfg.GRPCServerPort) + + configMapConfig := commons.MustGetConfigMapConfig(ctx, cfg.APIServerConfig, cfg.TestkubeNamespace, cfg.TestkubeAnalyticsEnabled) + + // Start local Control Plane + if mode == common.ModeStandalone { + controlPlane := services.CreateControlPlane(ctx, cfg, features, configMapConfig) + g.Go(func() error { + return controlPlane.Run(ctx) + }) + // Rewire connection + cfg.TestkubeProURL = fmt.Sprintf("%s:%d", cfg.APIServerFullname, cfg.GRPCServerPort) + cfg.TestkubeProTLSInsecure = true + } + + clusterId, _ := configMapConfig.GetUniqueClusterId(ctx) + telemetryEnabled, _ := configMapConfig.GetTelemetryEnabled(ctx) + + // k8s kubeClient, err := kubeclient.GetClient() commons.ExitOnError("Getting kubernetes client", err) - clientset, err := k8sclient.ConnectToK8s() commons.ExitOnError("Creating k8s clientset", err) - // k8s + // k8s clients secretClient := secret.NewClientFor(clientset, cfg.TestkubeNamespace) configMapClient := configmap.NewClientFor(clientset, cfg.TestkubeNamespace) - deprecatedClients := commons.CreateDeprecatedClients(kubeClient, cfg.TestkubeNamespace) webhooksClient := executorsclientv1.NewWebhooksClient(kubeClient, cfg.TestkubeNamespace) testTriggersClient := testtriggersclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) testWorkflowExecutionsClient := testworkflowsclientv1.NewTestWorkflowExecutionsClient(kubeClient, cfg.TestkubeNamespace) @@ -117,74 +122,60 @@ func main() { } secretManager := secretmanager.New(clientset, secretConfig) - nc := commons.MustCreateNATSConnection(cfg) - eventBus := bus.NewNATSBus(nc) - if cfg.Trace { - eventBus.TraceEvents() - } - eventsEmitter := event.NewEmitter(eventBus, cfg.TestkubeClusterName) - - var logGrpcClient logsclient.StreamGetter - var logsStream logsclient.Stream - if features.LogsV2 { - logGrpcClient = commons.MustGetLogsV2Client(cfg) - logsStream, err = logsclient.NewNatsLogStream(nc.Conn) - commons.ExitOnError("Creating logs streaming client", err) - } - - configMapConfig := commons.MustGetConfigMapConfig(ctx, cfg.APIServerConfig, cfg.TestkubeNamespace, cfg.TestkubeAnalyticsEnabled) - clusterId, _ := configMapConfig.GetUniqueClusterId(ctx) - telemetryEnabled, _ := configMapConfig.GetTelemetryEnabled(ctx) - envs := commons.GetEnvironmentVariables() - metrics := metrics.NewMetrics() - - defaultExecutors, images, err := commons.ReadDefaultExecutors(cfg) - commons.ExitOnError("Parsing default executors", err) - if !cfg.TestkubeReadonlyExecutors { - err := kubeexecutor.SyncDefaultExecutors(deprecatedClients.Executors(), cfg.TestkubeNamespace, defaultExecutors) - commons.ExitOnError("Sync default executors", err) - } - jobTemplates, err := parser.ParseJobTemplates(cfg) - commons.ExitOnError("Creating job templates", err) - containerTemplates, err := parser.ParseContainerTemplates(cfg) - commons.ExitOnError("Creating container job templates", err) - inspector := commons.CreateImageInspector(cfg, configMapClient, secretClient) - slackLoader := commons.MustCreateSlackLoader(cfg, envs) - - var deprecatedRepositories commons.DeprecatedRepositories - var testWorkflowResultsRepository testworkflow2.Repository - var testWorkflowOutputRepository testworkflow2.OutputRepository - var triggerLeaseBackend triggers.LeaseBackend - var artifactStorage domainstorage.ArtifactsStorage - var storageClient domainstorage.Client var testWorkflowsClient testworkflowsclientv1.Interface var testWorkflowTemplatesClient testworkflowsclientv1.TestWorkflowTemplatesInterface var grpcClient cloud.TestKubeCloudAPIClient var grpcConn *grpc.ClientConn - if mode == common.ModeAgent { - grpcConn, err = agent.NewGRPCConnection( - ctx, - cfg.TestkubeProTLSInsecure, - cfg.TestkubeProSkipVerify, - cfg.TestkubeProURL, - cfg.TestkubeProCertFile, - cfg.TestkubeProKeyFile, - cfg.TestkubeProCAFile, //nolint - log.DefaultLogger, - ) - commons.ExitOnError("error creating gRPC connection", err) + // Use local network for local access + controlPlaneUrl := cfg.TestkubeProURL + if strings.HasPrefix(controlPlaneUrl, fmt.Sprintf("%s:%d", cfg.APIServerFullname, cfg.GRPCServerPort)) { + controlPlaneUrl = fmt.Sprintf("127.0.0.1:%d", cfg.GRPCServerPort) + } + grpcConn, err = agent.NewGRPCConnection( + ctx, + cfg.TestkubeProTLSInsecure, + cfg.TestkubeProSkipVerify, + controlPlaneUrl, + cfg.TestkubeProCertFile, + cfg.TestkubeProKeyFile, + cfg.TestkubeProCAFile, //nolint + log.DefaultLogger, + ) + commons.ExitOnError("error creating gRPC connection", err) + + grpcClient = cloud.NewTestKubeCloudAPIClient(grpcConn) + + if mode == common.ModeAgent && cfg.WorkflowStorage == "control-plane" { + testWorkflowsClient = cloudtestworkflow.NewCloudTestWorkflowRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) + testWorkflowTemplatesClient = cloudtestworkflow.NewCloudTestWorkflowTemplateRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) + } else { + testWorkflowsClient = testworkflowsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) + testWorkflowTemplatesClient = testworkflowsclientv1.NewTestWorkflowTemplatesClient(kubeClient, cfg.TestkubeNamespace) + } - grpcClient = cloud.NewTestKubeCloudAPIClient(grpcConn) + testWorkflowResultsRepository := cloudtestworkflow.NewCloudRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) + var opts []cloudtestworkflow.Option + if cfg.StorageSkipVerify { + opts = append(opts, cloudtestworkflow.WithSkipVerify()) } + testWorkflowOutputRepository := cloudtestworkflow.NewCloudOutputRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey, opts...) + triggerLeaseBackend := triggers.NewAcquireAlwaysLeaseBackend() + artifactStorage := cloudartifacts.NewCloudArtifactsStorage(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - proContext := commons.ReadProContext(ctx, cfg, grpcClient) + nc := commons.MustCreateNATSConnection(cfg) + eventBus := bus.NewNATSBus(nc) + if cfg.Trace { + eventBus.TraceEvents() + } + eventsEmitter := event.NewEmitter(eventBus, cfg.TestkubeClusterName) // Check Pro/Enterprise subscription + proContext := commons.ReadProContext(ctx, cfg, grpcClient) subscriptionChecker, err := checktcl.NewSubscriptionChecker(ctx, proContext, grpcClient, grpcConn) commons.ExitOnError("Failed creating subscription checker", err) @@ -198,111 +189,32 @@ func main() { serviceAccountNames = schedulertcl.GetServiceAccountNamesFromConfig(serviceAccountNames, cfg.TestkubeExecutionNamespaces) } - if mode == common.ModeAgent && cfg.WorkflowStorage == "control-plane" { - testWorkflowsClient = cloudtestworkflow.NewCloudTestWorkflowRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - testWorkflowTemplatesClient = cloudtestworkflow.NewCloudTestWorkflowTemplateRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - } else { - testWorkflowsClient = testworkflowsclientv1.NewClient(kubeClient, cfg.TestkubeNamespace) - testWorkflowTemplatesClient = testworkflowsclientv1.NewTestWorkflowTemplatesClient(kubeClient, cfg.TestkubeNamespace) - } + metrics := metrics.NewMetrics() - if mode == common.ModeAgent { - deprecatedRepositories = commons.CreateDeprecatedRepositoriesForCloud(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - testWorkflowResultsRepository = cloudtestworkflow.NewCloudRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - var opts []cloudtestworkflow.Option - if cfg.StorageSkipVerify { - opts = append(opts, cloudtestworkflow.WithSkipVerify()) - } - testWorkflowOutputRepository = cloudtestworkflow.NewCloudOutputRepository(grpcClient, grpcConn, cfg.TestkubeProAPIKey, opts...) - triggerLeaseBackend = triggers.NewAcquireAlwaysLeaseBackend() - artifactStorage = cloudartifacts.NewCloudArtifactsStorage(grpcClient, grpcConn, cfg.TestkubeProAPIKey) - } else { - // Connect to storages - db := commons.MustGetMongoDatabase(ctx, cfg, secretClient, !cfg.DisableMongoMigrations) - storageClient = commons.MustGetMinioClient(cfg) - - // Build repositories - sequenceRepository := sequence.NewMongoRepository(db) - testWorkflowResultsRepository = testworkflow2.NewMongoRepository(db, cfg.APIMongoAllowDiskUse, - testworkflow2.WithMongoRepositorySequence(sequenceRepository)) - triggerLeaseBackend = triggers.NewMongoLeaseBackend(db) - testWorkflowOutputRepository = testworkflow2.NewMinioOutputRepository(storageClient, db.Collection(testworkflow2.CollectionName), cfg.LogsBucket) - artifactStorage = minio.NewMinIOArtifactClient(storageClient) - deprecatedRepositories = commons.CreateDeprecatedRepositoriesForMongo(ctx, cfg, db, logGrpcClient, storageClient, features) + var deprecatedSystem *services.DeprecatedSystem + if !cfg.DisableDeprecatedTests { + deprecatedSystem = services.CreateDeprecatedSystem( + ctx, + mode, + cfg, + features, + metrics, + configMapConfig, + secretConfig, + grpcClient, + grpcConn, + nc, + eventsEmitter, + eventBus, + inspector, + subscriptionChecker, + &proContext, + ) } - executor, err := client.NewJobExecutor( - deprecatedRepositories, - deprecatedClients, - images, - jobTemplates, - serviceAccountNames, - metrics, - eventsEmitter, - configMapConfig, - clientset, - cfg.TestkubeRegistry, - cfg.TestkubePodStartTimeout, - clusterId, - cfg.TestkubeDashboardURI, - fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), - cfg.NatsURI, - cfg.Debug, - logsStream, - features, - cfg.TestkubeDefaultStorageClassName, - cfg.WhitelistedContainers, - ) - commons.ExitOnError("Creating executor client", err) - - containerExecutor, err := containerexecutor.NewContainerExecutor( - deprecatedRepositories, - deprecatedClients, - images, - containerTemplates, - inspector, - serviceAccountNames, - metrics, - eventsEmitter, - configMapConfig, - cfg.TestkubeRegistry, - cfg.TestkubePodStartTimeout, - clusterId, - cfg.TestkubeDashboardURI, - fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), - cfg.NatsURI, - cfg.Debug, - logsStream, - features, - cfg.TestkubeDefaultStorageClassName, - cfg.WhitelistedContainers, - cfg.TestkubeImageCredentialsCacheTTL, - ) - commons.ExitOnError("Creating container executor", err) - - sched := scheduler.NewScheduler( - metrics, - executor, - containerExecutor, - deprecatedRepositories, - deprecatedClients, - secretClient, - eventsEmitter, - log.DefaultLogger, - configMapConfig, - configMapClient, - eventBus, - cfg.TestkubeDashboardURI, - features, - logsStream, - cfg.TestkubeNamespace, - cfg.TestkubeProTLSSecret, - cfg.TestkubeProRunnerCustomCASecret, - subscriptionChecker, - ) - // Build internal execution worker testWorkflowProcessor := presets.NewOpenSource(inspector) + // Pro edition only (tcl protected code) if mode == common.ModeAgent { testWorkflowProcessor = presets.NewPro(inspector) } @@ -329,11 +241,20 @@ func main() { return nil }) + var deprecatedClients commons.DeprecatedClients + var deprecatedRepositories commons.DeprecatedRepositories + if deprecatedSystem != nil { + deprecatedClients = deprecatedSystem.Clients + deprecatedRepositories = deprecatedSystem.Repositories + } + // Initialize event handlers websocketLoader := ws.NewWebsocketLoader() - eventsEmitter.Loader.Register(webhook.NewWebhookLoader(log.DefaultLogger, webhooksClient, deprecatedClients.Templates(), deprecatedRepositories.TestResults(), deprecatedRepositories.TestSuiteResults(), testWorkflowResultsRepository, metrics, &proContext, envs)) + if !cfg.DisableWebhooks { + eventsEmitter.Loader.Register(webhook.NewWebhookLoader(log.DefaultLogger, webhooksClient, deprecatedClients, deprecatedRepositories, testWorkflowResultsRepository, metrics, &proContext, envs)) + } eventsEmitter.Loader.Register(websocketLoader) - eventsEmitter.Loader.Register(slackLoader) + eventsEmitter.Loader.Register(commons.MustCreateSlackLoader(cfg, envs)) if cfg.CDEventsTarget != "" { cdeventLoader, err := cdevent.NewCDEventLoader(cfg.CDEventsTarget, clusterId, cfg.TestkubeNamespace, cfg.TestkubeDashboardURI, testkube.AllEventTypes) if err == nil { @@ -363,41 +284,9 @@ func main() { Scopes: cfg.TestkubeOAuthScopes, })) - storageParams := deprecatedapiv1.StorageParams{ - SSL: cfg.StorageSSL, - SkipVerify: cfg.StorageSkipVerify, - CertFile: cfg.StorageCertFile, - KeyFile: cfg.StorageKeyFile, - CAFile: cfg.StorageCAFile, - Endpoint: cfg.StorageEndpoint, - AccessKeyId: cfg.StorageAccessKeyID, - SecretAccessKey: cfg.StorageSecretAccessKey, - Region: cfg.StorageRegion, - Token: cfg.StorageToken, - Bucket: cfg.StorageBucket, + if deprecatedSystem != nil && deprecatedSystem.API != nil { + deprecatedSystem.API.Init(httpServer) } - deprecatedApi := deprecatedapiv1.NewDeprecatedTestkubeAPI( - deprecatedRepositories, - deprecatedClients, - cfg.TestkubeNamespace, - secretClient, - eventsEmitter, - executor, - containerExecutor, - metrics, - sched, - cfg.GraphqlPort, - artifactStorage, - mode, - eventBus, - secretConfig, - features, - logsStream, - logGrpcClient, - &proContext, - storageParams, - ) - deprecatedApi.Init(httpServer) api := apiv1.NewTestkubeAPI( deprecatedClients, @@ -427,45 +316,49 @@ func main() { ) api.Init(httpServer) - if mode == common.ModeAgent { - log.DefaultLogger.Info("starting agent service") - getTestWorkflowNotificationsStream := func(ctx context.Context, executionID string) (<-chan testkube.TestWorkflowExecutionNotification, error) { - execution, err := testWorkflowResultsRepository.Get(ctx, executionID) - if err != nil { - return nil, err - } - notifications := executionWorker.Notifications(ctx, execution.Id, executionworkertypes.NotificationsOptions{ - Hints: executionworkertypes.Hints{ - Namespace: execution.Namespace, - Signature: execution.Signature, - ScheduledAt: common.Ptr(execution.ScheduledAt), - }, - }) - if notifications.Err() != nil { - return nil, notifications.Err() - } - return notifications.Channel(), nil + log.DefaultLogger.Info("starting agent service") + getTestWorkflowNotificationsStream := func(ctx context.Context, executionID string) (<-chan testkube.TestWorkflowExecutionNotification, error) { + execution, err := testWorkflowResultsRepository.Get(ctx, executionID) + if err != nil { + return nil, err } - agentHandle, err := agent.NewAgent( - log.DefaultLogger, - httpServer.Mux.Handler(), - grpcClient, - deprecatedApi.GetLogsStream, - getTestWorkflowNotificationsStream, - clusterId, - cfg.TestkubeClusterName, - features, - &proContext, - cfg.TestkubeDockerImageVersion, - ) - commons.ExitOnError("Starting agent", err) - g.Go(func() error { - err = agentHandle.Run(ctx) - commons.ExitOnError("Running agent", err) - return nil + notifications := executionWorker.Notifications(ctx, execution.Id, executionworkertypes.NotificationsOptions{ + Hints: executionworkertypes.Hints{ + Namespace: execution.Namespace, + Signature: execution.Signature, + ScheduledAt: common.Ptr(execution.ScheduledAt), + }, }) - eventsEmitter.Loader.Register(agentHandle) + if notifications.Err() != nil { + return nil, notifications.Err() + } + return notifications.Channel(), nil } + getDeprecatedLogStream := func(ctx context.Context, executionID string) (chan output.Output, error) { + return nil, errors.New("deprecated features have been disabled") + } + if deprecatedSystem != nil && deprecatedSystem.StreamLogs != nil { + getDeprecatedLogStream = deprecatedSystem.StreamLogs + } + agentHandle, err := agent.NewAgent( + log.DefaultLogger, + httpServer.Mux.Handler(), + grpcClient, + getDeprecatedLogStream, + getTestWorkflowNotificationsStream, + clusterId, + cfg.TestkubeClusterName, + features, + &proContext, + cfg.TestkubeDockerImageVersion, + ) + commons.ExitOnError("Starting agent", err) + g.Go(func() error { + err = agentHandle.Run(ctx) + commons.ExitOnError("Running agent", err) + return nil + }) + eventsEmitter.Loader.Register(agentHandle) if !cfg.DisableTestTriggers { k8sCfg, err := k8sclient.GetK8sClientConfig() @@ -476,16 +369,13 @@ func main() { //testkubeClientset := testkubeclientset.New(clientset.RESTClient()) triggerService := triggers.NewService( - deprecatedRepositories, - deprecatedClients, - sched, + deprecatedSystem, clientset, testkubeClientset, testWorkflowsClient, triggerLeaseBackend, log.DefaultLogger, configMapConfig, - executor, eventBus, metrics, executionWorker, @@ -506,15 +396,6 @@ func main() { log.DefaultLogger.Info("test triggers are disabled") } - if !cfg.DisableReconciler { - reconcilerClient := reconciler.NewClient(clientset, deprecatedRepositories, deprecatedClients, log.DefaultLogger) - g.Go(func() error { - return reconcilerClient.Run(ctx) - }) - } else { - log.DefaultLogger.Info("reconciler is disabled") - } - // telemetry based functions g.Go(func() error { services.HandleTelemetryHeartbeat(ctx, clusterId, configMapConfig) @@ -542,9 +423,19 @@ func main() { return httpServer.Run(ctx) }) - g.Go(func() error { - return deprecatedApi.RunGraphQLServer(ctx) - }) + if deprecatedSystem != nil { + if deprecatedSystem.Reconciler != nil { + g.Go(func() error { + return deprecatedSystem.Reconciler.Run(ctx) + }) + } + + if deprecatedSystem.API != nil { + g.Go(func() error { + return deprecatedSystem.API.RunGraphQLServer(ctx) + }) + } + } if err := g.Wait(); err != nil { log.DefaultLogger.Fatalf("Testkube is shutting down: %v", err) diff --git a/cmd/api-server/services/controlplane.go b/cmd/api-server/services/controlplane.go new file mode 100644 index 00000000000..933d39f9317 --- /dev/null +++ b/cmd/api-server/services/controlplane.go @@ -0,0 +1,391 @@ +package services + +import ( + "context" + "strings" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/internal/config" + cloudartifacts "github.com/kubeshop/testkube/pkg/cloud/data/artifact" + cloudconfig "github.com/kubeshop/testkube/pkg/cloud/data/config" + cloudresult "github.com/kubeshop/testkube/pkg/cloud/data/result" + cloudtestresult "github.com/kubeshop/testkube/pkg/cloud/data/testresult" + cloudtestworkflow "github.com/kubeshop/testkube/pkg/cloud/data/testworkflow" + "github.com/kubeshop/testkube/pkg/controlplane" + "github.com/kubeshop/testkube/pkg/featureflags" + "github.com/kubeshop/testkube/pkg/k8sclient" + "github.com/kubeshop/testkube/pkg/log" + logsclient "github.com/kubeshop/testkube/pkg/logs/client" + configRepo "github.com/kubeshop/testkube/pkg/repository/config" + "github.com/kubeshop/testkube/pkg/repository/result" + "github.com/kubeshop/testkube/pkg/repository/sequence" + "github.com/kubeshop/testkube/pkg/repository/testresult" + "github.com/kubeshop/testkube/pkg/repository/testworkflow" + "github.com/kubeshop/testkube/pkg/secret" + "github.com/kubeshop/testkube/pkg/storage/minio" + "github.com/kubeshop/testkube/pkg/tcl/checktcl" +) + +func mapTestWorkflowFilters(s []*testworkflow.FilterImpl) []testworkflow.Filter { + v := make([]testworkflow.Filter, len(s)) + for i := range s { + v[i] = s[i] + } + return v +} + +func mapTestFilters(s []*result.FilterImpl) []result.Filter { + v := make([]result.Filter, len(s)) + for i := range s { + v[i] = s[i] + } + return v +} + +func mapTestSuiteFilters(s []*testresult.FilterImpl) []testresult.Filter { + v := make([]testresult.Filter, len(s)) + for i := range s { + v[i] = s[i] + } + return v +} + +func CreateControlPlane(ctx context.Context, cfg *config.Config, features featureflags.FeatureFlags, configMapClient configRepo.Repository) *controlplane.Server { + // Connect to the cluster + clientset, err := k8sclient.ConnectToK8s() + commons.ExitOnError("Creating k8s clientset", err) + + // Connect to storages + secretClient := secret.NewClientFor(clientset, cfg.TestkubeNamespace) + db := commons.MustGetMongoDatabase(ctx, cfg, secretClient, !cfg.DisableMongoMigrations) + storageClient := commons.MustGetMinioClient(cfg) + + var logGrpcClient logsclient.StreamGetter + if !cfg.DisableDeprecatedTests && features.LogsV2 { + logGrpcClient = commons.MustGetLogsV2Client(cfg) + commons.ExitOnError("Creating logs streaming client", err) + } + + // Build repositories + sequenceRepository := sequence.NewMongoRepository(db) + testWorkflowResultsRepository := testworkflow.NewMongoRepository(db, cfg.APIMongoAllowDiskUse, + testworkflow.WithMongoRepositorySequence(sequenceRepository)) + testWorkflowOutputRepository := testworkflow.NewMinioOutputRepository(storageClient, db.Collection(testworkflow.CollectionName), cfg.LogsBucket) + artifactStorage := minio.NewMinIOArtifactClient(storageClient) + deprecatedRepositories := commons.CreateDeprecatedRepositoriesForMongo(ctx, cfg, db, logGrpcClient, storageClient, features) + + // Set up "Config" commands + configCommands := controlplane.CommandHandlers{ + cloudconfig.CmdConfigGetOrganizationPlan: controlplane.Handler(func(ctx context.Context, data checktcl.GetOrganizationPlanRequest) (r checktcl.GetOrganizationPlanResponse, err error) { + return + }), + cloudconfig.CmdConfigGetUniqueClusterId: controlplane.Handler(func(ctx context.Context, data cloudconfig.GetUniqueClusterIdRequest) (r cloudconfig.GetUniqueClusterIdResponse, err error) { + r.ClusterID, err = configMapClient.GetUniqueClusterId(ctx) + return + }), + cloudconfig.CmdConfigGetTelemetryEnabled: controlplane.Handler(func(ctx context.Context, data cloudconfig.GetTelemetryEnabledRequest) (r cloudconfig.GetTelemetryEnabledResponse, err error) { + r.Enabled, err = configMapClient.GetTelemetryEnabled(ctx) + return + }), + } + + // Set up "Tests - Executions" commands + deprecatedTestExecutionsCommands := controlplane.CommandHandlers{ + cloudresult.CmdResultGet: controlplane.Handler(func(ctx context.Context, data cloudresult.GetRequest) (r cloudresult.GetResponse, err error) { + r.Execution, err = deprecatedRepositories.TestResults().Get(ctx, data.ID) + return + }), + cloudresult.CmdResultGetByNameAndTest: controlplane.Handler(func(ctx context.Context, data cloudresult.GetByNameAndTestRequest) (r cloudresult.GetByNameAndTestResponse, err error) { + r.Execution, err = deprecatedRepositories.TestResults().GetByNameAndTest(ctx, data.Name, data.TestName) + return + }), + cloudresult.CmdResultGetLatestByTest: controlplane.Handler(func(ctx context.Context, data cloudresult.GetLatestByTestRequest) (r cloudresult.GetLatestByTestResponse, err error) { + ex, err := deprecatedRepositories.TestResults().GetLatestByTest(ctx, data.TestName) + if ex != nil { + r.Execution = *ex + } + return + }), + cloudresult.CmdResultGetLatestByTests: controlplane.Handler(func(ctx context.Context, data cloudresult.GetLatestByTestsRequest) (r cloudresult.GetLatestByTestsResponse, err error) { + r.Executions, err = deprecatedRepositories.TestResults().GetLatestByTests(ctx, data.TestNames) + return + }), + cloudresult.CmdResultGetExecutionTotals: controlplane.Handler(func(ctx context.Context, data cloudresult.GetExecutionTotalsRequest) (r cloudresult.GetExecutionTotalsResponse, err error) { + r.Result, err = deprecatedRepositories.TestResults().GetExecutionTotals(ctx, data.Paging, mapTestFilters(data.Filter)...) + return + }), + cloudresult.CmdResultGetExecutions: controlplane.Handler(func(ctx context.Context, data cloudresult.GetExecutionsRequest) (r cloudresult.GetExecutionsResponse, err error) { + r.Executions, err = deprecatedRepositories.TestResults().GetExecutions(ctx, data.Filter) + return + }), + cloudresult.CmdResultGetPreviousFinishedState: controlplane.Handler(func(ctx context.Context, data cloudresult.GetPreviousFinishedStateRequest) (r cloudresult.GetPreviousFinishedStateResponse, err error) { + r.Result, err = deprecatedRepositories.TestResults().GetPreviousFinishedState(ctx, data.TestName, data.Date) + return + }), + cloudresult.CmdResultInsert: controlplane.Handler(func(ctx context.Context, data cloudresult.InsertRequest) (r cloudresult.InsertResponse, err error) { + return r, deprecatedRepositories.TestResults().Insert(ctx, data.Result) + }), + cloudresult.CmdResultUpdate: controlplane.Handler(func(ctx context.Context, data cloudresult.UpdateRequest) (r cloudresult.UpdateResponse, err error) { + return r, deprecatedRepositories.TestResults().Update(ctx, data.Result) + }), + cloudresult.CmdResultUpdateResult: controlplane.Handler(func(ctx context.Context, data cloudresult.UpdateResultInExecutionRequest) (r cloudresult.UpdateResultInExecutionResponse, err error) { + return r, deprecatedRepositories.TestResults().UpdateResult(ctx, data.ID, data.Execution) + }), + cloudresult.CmdResultStartExecution: controlplane.Handler(func(ctx context.Context, data cloudresult.StartExecutionRequest) (r cloudresult.StartExecutionResponse, err error) { + return r, deprecatedRepositories.TestResults().StartExecution(ctx, data.ID, data.StartTime) + }), + cloudresult.CmdResultEndExecution: controlplane.Handler(func(ctx context.Context, data cloudresult.EndExecutionRequest) (r cloudresult.EndExecutionResponse, err error) { + return r, deprecatedRepositories.TestResults().EndExecution(ctx, data.Execution) + }), + cloudresult.CmdResultGetLabels: controlplane.Handler(func(ctx context.Context, data cloudresult.GetLabelsRequest) (r cloudresult.GetLabelsResponse, err error) { + r.Labels, err = deprecatedRepositories.TestResults().GetLabels(ctx) + return + }), + cloudresult.CmdResultDeleteByTest: controlplane.Handler(func(ctx context.Context, data cloudresult.DeleteByTestRequest) (r cloudresult.DeleteByTestResponse, err error) { + return r, deprecatedRepositories.TestResults().DeleteByTest(ctx, data.TestName) + }), + cloudresult.CmdResultDeleteByTestSuite: controlplane.Handler(func(ctx context.Context, data cloudresult.DeleteByTestSuiteRequest) (r cloudresult.DeleteByTestSuiteResponse, err error) { + return r, deprecatedRepositories.TestResults().DeleteByTestSuite(ctx, data.TestSuiteName) + }), + cloudresult.CmdResultDeleteAll: controlplane.Handler(func(ctx context.Context, data cloudresult.DeleteAllRequest) (r cloudresult.DeleteAllResponse, err error) { + return r, deprecatedRepositories.TestResults().DeleteAll(ctx) + }), + cloudresult.CmdResultDeleteByTests: controlplane.Handler(func(ctx context.Context, data cloudresult.DeleteByTestsRequest) (r cloudresult.DeleteByTestsResponse, err error) { + return r, deprecatedRepositories.TestResults().DeleteByTests(ctx, data.TestNames) + }), + cloudresult.CmdResultDeleteByTestSuites: controlplane.Handler(func(ctx context.Context, data cloudresult.DeleteByTestSuitesRequest) (r cloudresult.DeleteByTestSuitesResponse, err error) { + return r, deprecatedRepositories.TestResults().DeleteByTestSuites(ctx, data.TestSuiteNames) + }), + cloudresult.CmdResultDeleteForAllTestSuites: controlplane.Handler(func(ctx context.Context, data cloudresult.DeleteForAllTestSuitesRequest) (r cloudresult.DeleteForAllTestSuitesResponse, err error) { + return r, deprecatedRepositories.TestResults().DeleteForAllTestSuites(ctx) + }), + cloudresult.CmdResultGetTestMetrics: controlplane.Handler(func(ctx context.Context, data cloudresult.GetTestMetricsRequest) (r cloudresult.GetTestMetricsResponse, err error) { + r.Metrics, err = deprecatedRepositories.TestResults().GetTestMetrics(ctx, data.Name, data.Limit, data.Last) + return + }), + cloudresult.CmdResultGetNextExecutionNumber: controlplane.Handler(func(ctx context.Context, data cloudresult.NextExecutionNumberRequest) (r cloudresult.NextExecutionNumberResponse, err error) { + r.TestNumber, err = deprecatedRepositories.TestResults().GetNextExecutionNumber(ctx, data.TestName) + return + }), + } + + // Set up "Test Suites - Executions" commands + deprecatedTestSuiteExecutionsCommands := controlplane.CommandHandlers{ + cloudtestresult.CmdTestResultGet: controlplane.Handler(func(ctx context.Context, data cloudtestresult.GetRequest) (r cloudtestresult.GetResponse, err error) { + r.TestSuiteExecution, err = deprecatedRepositories.TestSuiteResults().Get(ctx, data.ID) + return + }), + cloudtestresult.CmdTestResultGetByNameAndTestSuite: controlplane.Handler(func(ctx context.Context, data cloudtestresult.GetByNameAndTestSuiteRequest) (r cloudtestresult.GetByNameAndTestSuiteResponse, err error) { + r.TestSuiteExecution, err = deprecatedRepositories.TestSuiteResults().GetByNameAndTestSuite(ctx, data.Name, data.TestSuiteName) + return + }), + cloudtestresult.CmdTestResultGetLatestByTestSuite: controlplane.Handler(func(ctx context.Context, data cloudtestresult.GetLatestByTestSuiteRequest) (r cloudtestresult.GetLatestByTestSuiteResponse, err error) { + ex, err := deprecatedRepositories.TestSuiteResults().GetLatestByTestSuite(ctx, data.TestSuiteName) + if ex != nil { + r.TestSuiteExecution = *ex + } + return + }), + cloudtestresult.CmdTestResultGetLatestByTestSuites: controlplane.Handler(func(ctx context.Context, data cloudtestresult.GetLatestByTestSuitesRequest) (r cloudtestresult.GetLatestByTestSuitesResponse, err error) { + r.TestSuiteExecutions, err = deprecatedRepositories.TestSuiteResults().GetLatestByTestSuites(ctx, data.TestSuiteNames) + return + }), + cloudtestresult.CmdTestResultGetExecutionsTotals: controlplane.Handler(func(ctx context.Context, data cloudtestresult.GetExecutionsTotalsRequest) (r cloudtestresult.GetExecutionsTotalsResponse, err error) { + r.ExecutionsTotals, err = deprecatedRepositories.TestSuiteResults().GetExecutionsTotals(ctx, mapTestSuiteFilters(data.Filter)...) + return + }), + cloudtestresult.CmdTestResultGetExecutions: controlplane.Handler(func(ctx context.Context, data cloudtestresult.GetExecutionsRequest) (r cloudtestresult.GetExecutionsResponse, err error) { + r.TestSuiteExecutions, err = deprecatedRepositories.TestSuiteResults().GetExecutions(ctx, data.Filter) + return + }), + cloudtestresult.CmdTestResultGetPreviousFinishedState: controlplane.Handler(func(ctx context.Context, data cloudtestresult.GetPreviousFinishedStateRequest) (r cloudtestresult.GetPreviousFinishedStateResponse, err error) { + r.Result, err = deprecatedRepositories.TestSuiteResults().GetPreviousFinishedState(ctx, data.TestSuiteName, data.Date) + return + }), + cloudtestresult.CmdTestResultInsert: controlplane.Handler(func(ctx context.Context, data cloudtestresult.InsertRequest) (r cloudtestresult.InsertResponse, err error) { + return r, deprecatedRepositories.TestSuiteResults().Insert(ctx, data.TestSuiteExecution) + }), + cloudtestresult.CmdTestResultUpdate: controlplane.Handler(func(ctx context.Context, data cloudtestresult.UpdateRequest) (r cloudtestresult.UpdateResponse, err error) { + return r, deprecatedRepositories.TestSuiteResults().Update(ctx, data.TestSuiteExecution) + }), + cloudtestresult.CmdTestResultStartExecution: controlplane.Handler(func(ctx context.Context, data cloudtestresult.StartExecutionRequest) (r cloudtestresult.StartExecutionResponse, err error) { + return r, deprecatedRepositories.TestSuiteResults().StartExecution(ctx, data.ID, data.StartTime) + }), + cloudtestresult.CmdTestResultEndExecution: controlplane.Handler(func(ctx context.Context, data cloudtestresult.EndExecutionRequest) (r cloudtestresult.EndExecutionResponse, err error) { + return r, deprecatedRepositories.TestSuiteResults().EndExecution(ctx, data.Execution) + }), + cloudtestresult.CmdTestResultDeleteByTestSuite: controlplane.Handler(func(ctx context.Context, data cloudtestresult.DeleteByTestSuiteRequest) (r cloudtestresult.DeleteByTestSuiteResponse, err error) { + return r, deprecatedRepositories.TestSuiteResults().DeleteByTestSuite(ctx, data.TestSuiteName) + }), + cloudtestresult.CmdTestResultDeleteAll: controlplane.Handler(func(ctx context.Context, data cloudtestresult.DeleteAllTestResultsRequest) (r cloudtestresult.DeleteAllTestResultsResponse, err error) { + return r, deprecatedRepositories.TestSuiteResults().DeleteAll(ctx) + }), + cloudtestresult.CmdTestResultDeleteByTestSuites: controlplane.Handler(func(ctx context.Context, data cloudtestresult.DeleteByTestSuitesRequest) (r cloudtestresult.DeleteByTestSuitesResponse, err error) { + return r, deprecatedRepositories.TestSuiteResults().DeleteByTestSuites(ctx, data.TestSuiteNames) + }), + cloudtestresult.CmdTestResultGetTestSuiteMetrics: controlplane.Handler(func(ctx context.Context, data cloudtestresult.GetTestSuiteMetricsRequest) (r cloudtestresult.GetTestSuiteMetricsResponse, err error) { + r.Metrics, err = deprecatedRepositories.TestSuiteResults().GetTestSuiteMetrics(ctx, data.Name, data.Limit, data.Last) + return + }), + cloudtestresult.CmdTestResultGetNextExecutionNumber: controlplane.Handler(func(ctx context.Context, data cloudtestresult.NextExecutionNumberRequest) (r cloudtestresult.NextExecutionNumberResponse, err error) { + r.TestSuiteNumber, err = deprecatedRepositories.TestSuiteResults().GetNextExecutionNumber(ctx, data.TestSuiteName) + return + }), + } + + // Set up "Test Workflows - Executions" commands + testWorkflowExecutionsCommands := controlplane.CommandHandlers{ + cloudtestworkflow.CmdTestWorkflowExecutionGet: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetRequest) (r cloudtestworkflow.ExecutionGetResponse, err error) { + r.WorkflowExecution, err = testWorkflowResultsRepository.Get(ctx, data.ID) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetByNameAndWorkflow: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetByNameAndWorkflowRequest) (r cloudtestworkflow.ExecutionGetByNameAndWorkflowResponse, err error) { + r.WorkflowExecution, err = testWorkflowResultsRepository.GetByNameAndTestWorkflow(ctx, data.Name, data.WorkflowName) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetLatestByWorkflow: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetLatestByWorkflowRequest) (r cloudtestworkflow.ExecutionGetLatestByWorkflowResponse, err error) { + r.WorkflowExecution, err = testWorkflowResultsRepository.GetLatestByTestWorkflow(ctx, data.WorkflowName) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetRunning: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetRunningRequest) (r cloudtestworkflow.ExecutionGetRunningResponse, err error) { + r.WorkflowExecutions, err = testWorkflowResultsRepository.GetRunning(ctx) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetLatestByWorkflows: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetLatestByWorkflowsRequest) (r cloudtestworkflow.ExecutionGetLatestByWorkflowsResponse, err error) { + r.WorkflowExecutions, err = testWorkflowResultsRepository.GetLatestByTestWorkflows(ctx, data.WorkflowNames) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetExecutionTotals: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetExecutionTotalsRequest) (r cloudtestworkflow.ExecutionGetExecutionTotalsResponse, err error) { + r.Totals, err = testWorkflowResultsRepository.GetExecutionsTotals(ctx, mapTestWorkflowFilters(data.Filter)...) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetExecutions: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetExecutionsRequest) (r cloudtestworkflow.ExecutionGetExecutionsResponse, err error) { + r.WorkflowExecutions, err = testWorkflowResultsRepository.GetExecutions(ctx, data.Filter) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetExecutionsSummary: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetExecutionsSummaryRequest) (r cloudtestworkflow.ExecutionGetExecutionsSummaryResponse, err error) { + r.WorkflowExecutions, err = testWorkflowResultsRepository.GetExecutionsSummary(ctx, data.Filter) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetPreviousFinishedState: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetPreviousFinishedStateRequest) (r cloudtestworkflow.ExecutionGetPreviousFinishedStateResponse, err error) { + r.Result, err = testWorkflowResultsRepository.GetPreviousFinishedState(ctx, data.WorkflowName, data.Date) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionInsert: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionInsertRequest) (r cloudtestworkflow.ExecutionInsertResponse, err error) { + return r, testWorkflowResultsRepository.Insert(ctx, data.WorkflowExecution) + }), + cloudtestworkflow.CmdTestWorkflowExecutionUpdate: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionUpdateRequest) (r cloudtestworkflow.ExecutionUpdateResponse, err error) { + return r, testWorkflowResultsRepository.Update(ctx, data.WorkflowExecution) + }), + cloudtestworkflow.CmdTestWorkflowExecutionUpdateResult: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionUpdateResultRequest) (r cloudtestworkflow.ExecutionUpdateResultResponse, err error) { + return r, testWorkflowResultsRepository.UpdateResult(ctx, data.ID, data.Result) + }), + cloudtestworkflow.CmdTestWorkflowExecutionUpdateOutput: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionUpdateOutputRequest) (r cloudtestworkflow.ExecutionUpdateOutputResponse, err error) { + return r, testWorkflowResultsRepository.UpdateOutput(ctx, data.ID, data.Output) + }), + cloudtestworkflow.CmdTestWorkflowExecutionDeleteByWorkflow: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionDeleteByWorkflowRequest) (r cloudtestworkflow.ExecutionDeleteByWorkflowResponse, err error) { + return r, testWorkflowResultsRepository.DeleteByTestWorkflow(ctx, data.WorkflowName) + }), + cloudtestworkflow.CmdTestWorkflowExecutionDeleteAll: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionDeleteAllRequest) (r cloudtestworkflow.ExecutionDeleteAllResponse, err error) { + return r, testWorkflowResultsRepository.DeleteAll(ctx) + }), + cloudtestworkflow.CmdTestWorkflowExecutionDeleteByWorkflows: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionDeleteByWorkflowsRequest) (r cloudtestworkflow.ExecutionDeleteByWorkflowsResponse, err error) { + return r, testWorkflowResultsRepository.DeleteByTestWorkflows(ctx, data.WorkflowNames) + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetWorkflowMetrics: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetWorkflowMetricsRequest) (r cloudtestworkflow.ExecutionGetWorkflowMetricsResponse, err error) { + r.Metrics, err = testWorkflowResultsRepository.GetTestWorkflowMetrics(ctx, data.Name, data.Limit, data.Last) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionAddReport: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionsAddReportRequest) (r cloudtestworkflow.ExecutionsAddReportResponse, err error) { + return r, status.Error(codes.Unimplemented, "not supported in the standalone mode") + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetNextExecutionNumber: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetNextExecutionNumberRequest) (r cloudtestworkflow.ExecutionGetNextExecutionNumberResponse, err error) { + r.TestWorkflowNumber, err = testWorkflowResultsRepository.GetNextExecutionNumber(ctx, data.TestWorkflowName) + return + }), + cloudtestworkflow.CmdTestWorkflowExecutionGetExecutionTags: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionGetExecutionTagsRequest) (r cloudtestworkflow.ExecutionGetExecutionTagsResponse, err error) { + r.Tags, err = testWorkflowResultsRepository.GetExecutionTags(ctx, data.TestWorkflowName) + return + }), + } + + // Set up "Test Workflows - Output" commands + testWorkflowsOutputCommands := controlplane.CommandHandlers{ + cloudtestworkflow.CmdTestWorkflowOutputPresignSaveLog: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.OutputPresignSaveLogRequest) (r cloudtestworkflow.OutputPresignSaveLogResponse, err error) { + r.URL, err = testWorkflowOutputRepository.PresignSaveLog(ctx, data.ID, data.WorkflowName) + return + }), + cloudtestworkflow.CmdTestWorkflowOutputPresignReadLog: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.OutputPresignReadLogRequest) (r cloudtestworkflow.OutputPresignReadLogResponse, err error) { + r.URL, err = testWorkflowOutputRepository.PresignReadLog(ctx, data.ID, data.WorkflowName) + return + }), + cloudtestworkflow.CmdTestWorkflowOutputHasLog: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.OutputHasLogRequest) (r cloudtestworkflow.OutputHasLogResponse, err error) { + r.Has, err = testWorkflowOutputRepository.HasLog(ctx, data.ID, data.WorkflowName) + return + }), + cloudtestworkflow.CmdTestWorkflowOutputDeleteByTestWorkflow: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionDeleteOutputByWorkflowRequest) (r cloudtestworkflow.ExecutionDeleteOutputByWorkflowResponse, err error) { + return r, testWorkflowOutputRepository.DeleteOutputByTestWorkflow(ctx, data.WorkflowName) + }), + cloudtestworkflow.CmdTestworkflowOutputDeleteForTestWorkflows: controlplane.Handler(func(ctx context.Context, data cloudtestworkflow.ExecutionDeleteOutputForTestWorkflowsRequest) (r cloudtestworkflow.ExecutionDeleteOutputForTestWorkflowsResponse, err error) { + return r, testWorkflowOutputRepository.DeleteOutputForTestWorkflows(ctx, data.WorkflowNames) + }), + } + + // Set up "Artifacts" commands + // TODO: What about downloading artifacts archive? + // TODO: What about handling ArtifactRequest.OmitFolderPerExecution and ArtifactRequest.StorageBucket? + artifactsCommands := controlplane.CommandHandlers{ + cloudartifacts.CmdScraperPutObjectSignedURL: controlplane.Handler(func(ctx context.Context, data cloudartifacts.PutObjectSignedURLRequest) (r cloudartifacts.PutObjectSignedURLResponse, err error) { + r.URL, err = storageClient.PresignUploadFileToBucket(ctx, cfg.StorageBucket, data.ExecutionID, data.Object, 15*time.Minute) + return r, err + }), + cloudartifacts.CmdArtifactsListFiles: controlplane.Handler(func(ctx context.Context, data cloudartifacts.ListFilesRequest) (r cloudartifacts.ListFilesResponse, err error) { + r.Artifacts, err = artifactStorage.ListFiles(ctx, data.ExecutionID, data.TestName, data.TestSuiteName, data.TestWorkflowName) + return r, err + }), + cloudartifacts.CmdArtifactsDownloadFile: controlplane.Handler(func(ctx context.Context, data cloudartifacts.DownloadFileRequest) (r cloudartifacts.DownloadFileResponse, err error) { + r.URL, err = storageClient.PresignDownloadFileFromBucket(ctx, cfg.StorageBucket, data.ExecutionID, data.File, 15*time.Minute) + return r, err + }), + } + + // Select commands to use + commands := []controlplane.CommandHandlers{configCommands, testWorkflowExecutionsCommands, testWorkflowsOutputCommands, artifactsCommands} + if !cfg.DisableDeprecatedTests { + commands = append(commands, deprecatedTestExecutionsCommands, deprecatedTestSuiteExecutionsCommands) + } + + // Ensure the buckets exist + if cfg.StorageBucket != "" { + exists, err := storageClient.BucketExists(ctx, cfg.StorageBucket) + if err != nil { + log.DefaultLogger.Errorw("Failed to check if the storage bucket exists", "error", err) + } else if !exists { + err = storageClient.CreateBucket(ctx, cfg.StorageBucket) + if err != nil && !strings.Contains(err.Error(), "already exists") { + log.DefaultLogger.Errorw("Creating storage bucket", "error", err) + } + } + } + if cfg.LogsBucket != "" { + exists, err := storageClient.BucketExists(ctx, cfg.StorageBucket) + if err != nil { + log.DefaultLogger.Errorw("Failed to check if the storage bucket exists", "error", err) + } else if !exists { + err = storageClient.CreateBucket(ctx, cfg.LogsBucket) + if err != nil && !strings.Contains(err.Error(), "already exists") { + log.DefaultLogger.Errorw("Creating logs bucket", "error", err) + } + } + } + + return controlplane.New(controlplane.Config{ + Port: cfg.GRPCServerPort, + Logger: log.DefaultLogger, + Verbose: false, + }, commands...) +} diff --git a/cmd/api-server/services/deprecatedsystem.go b/cmd/api-server/services/deprecatedsystem.go new file mode 100644 index 00000000000..db0c2b85309 --- /dev/null +++ b/cmd/api-server/services/deprecatedsystem.go @@ -0,0 +1,238 @@ +package services + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go" + "google.golang.org/grpc" + + kubeclient "github.com/kubeshop/testkube-operator/pkg/client" + "github.com/kubeshop/testkube/cmd/api-server/commons" + deprecatedapiv1 "github.com/kubeshop/testkube/internal/app/api/deprecatedv1" + "github.com/kubeshop/testkube/internal/app/api/metrics" + "github.com/kubeshop/testkube/internal/common" + "github.com/kubeshop/testkube/internal/config" + parser "github.com/kubeshop/testkube/internal/template" + "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/cloud" + cloudartifacts "github.com/kubeshop/testkube/pkg/cloud/data/artifact" + "github.com/kubeshop/testkube/pkg/configmap" + "github.com/kubeshop/testkube/pkg/event" + "github.com/kubeshop/testkube/pkg/event/bus" + kubeexecutor "github.com/kubeshop/testkube/pkg/executor" + "github.com/kubeshop/testkube/pkg/executor/client" + "github.com/kubeshop/testkube/pkg/executor/containerexecutor" + "github.com/kubeshop/testkube/pkg/executor/output" + "github.com/kubeshop/testkube/pkg/featureflags" + "github.com/kubeshop/testkube/pkg/imageinspector" + "github.com/kubeshop/testkube/pkg/k8sclient" + "github.com/kubeshop/testkube/pkg/log" + logsclient "github.com/kubeshop/testkube/pkg/logs/client" + "github.com/kubeshop/testkube/pkg/reconciler" + configRepo "github.com/kubeshop/testkube/pkg/repository/config" + "github.com/kubeshop/testkube/pkg/scheduler" + "github.com/kubeshop/testkube/pkg/secret" + "github.com/kubeshop/testkube/pkg/storage" + "github.com/kubeshop/testkube/pkg/storage/minio" + "github.com/kubeshop/testkube/pkg/tcl/checktcl" + "github.com/kubeshop/testkube/pkg/tcl/schedulertcl" +) + +type DeprecatedSystem struct { + Clients commons.DeprecatedClients + Repositories commons.DeprecatedRepositories + Scheduler *scheduler.Scheduler + Reconciler *reconciler.Client + JobExecutor client.Executor + API *deprecatedapiv1.DeprecatedTestkubeAPI + StreamLogs func(ctx context.Context, executionID string) (chan output.Output, error) +} + +func CreateDeprecatedSystem( + ctx context.Context, + mode string, + cfg *config.Config, + features featureflags.FeatureFlags, + metrics metrics.Metrics, + configMapConfig configRepo.Repository, + secretConfig testkube.SecretConfig, + grpcClient cloud.TestKubeCloudAPIClient, + grpcConn *grpc.ClientConn, + natsConn *nats.EncodedConn, + eventsEmitter *event.Emitter, + eventBus *bus.NATSBus, + inspector imageinspector.Inspector, + subscriptionChecker checktcl.SubscriptionChecker, + proContext *config.ProContext, +) *DeprecatedSystem { + kubeClient, err := kubeclient.GetClient() + commons.ExitOnError("Getting kubernetes client", err) + clientset, err := k8sclient.ConnectToK8s() + commons.ExitOnError("Creating k8s clientset", err) + + secretClient := secret.NewClientFor(clientset, cfg.TestkubeNamespace) + configMapClient := configmap.NewClientFor(clientset, cfg.TestkubeNamespace) + + deprecatedClients := commons.CreateDeprecatedClients(kubeClient, cfg.TestkubeNamespace) + deprecatedRepositories := commons.CreateDeprecatedRepositoriesForCloud(grpcClient, grpcConn, cfg.TestkubeProAPIKey) + + defaultExecutors, images, err := commons.ReadDefaultExecutors(cfg) + commons.ExitOnError("Parsing default executors", err) + if !cfg.TestkubeReadonlyExecutors { + err := kubeexecutor.SyncDefaultExecutors(deprecatedClients.Executors(), cfg.TestkubeNamespace, defaultExecutors) + commons.ExitOnError("Sync default executors", err) + } + jobTemplates, err := parser.ParseJobTemplates(cfg) + commons.ExitOnError("Creating job templates", err) + containerTemplates, err := parser.ParseContainerTemplates(cfg) + commons.ExitOnError("Creating container job templates", err) + + serviceAccountNames := map[string]string{ + cfg.TestkubeNamespace: cfg.JobServiceAccountName, + } + // Pro edition only (tcl protected code) + if cfg.TestkubeExecutionNamespaces != "" { + err = subscriptionChecker.IsActiveOrgPlanEnterpriseForFeature("execution namespace") + commons.ExitOnError("Subscription checking", err) + serviceAccountNames = schedulertcl.GetServiceAccountNamesFromConfig(serviceAccountNames, cfg.TestkubeExecutionNamespaces) + } + + clusterId, _ := configMapConfig.GetUniqueClusterId(ctx) + + var logGrpcClient logsclient.StreamGetter + var logsStream logsclient.Stream + if features.LogsV2 { + logGrpcClient = commons.MustGetLogsV2Client(cfg) + logsStream, err = logsclient.NewNatsLogStream(natsConn.Conn) + commons.ExitOnError("Creating logs streaming client", err) + } + + executor, err := client.NewJobExecutor( + deprecatedRepositories, + deprecatedClients, + images, + jobTemplates, + serviceAccountNames, + metrics, + eventsEmitter, + configMapConfig, + clientset, + cfg.TestkubeRegistry, + cfg.TestkubePodStartTimeout, + clusterId, + cfg.TestkubeDashboardURI, + fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), + cfg.NatsURI, + cfg.Debug, + logsStream, + features, + cfg.TestkubeDefaultStorageClassName, + cfg.WhitelistedContainers, + ) + commons.ExitOnError("Creating executor client", err) + + containerExecutor, err := containerexecutor.NewContainerExecutor( + deprecatedRepositories, + deprecatedClients, + images, + containerTemplates, + inspector, + serviceAccountNames, + metrics, + eventsEmitter, + configMapConfig, + cfg.TestkubeRegistry, + cfg.TestkubePodStartTimeout, + clusterId, + cfg.TestkubeDashboardURI, + fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), + cfg.NatsURI, + cfg.Debug, + logsStream, + features, + cfg.TestkubeDefaultStorageClassName, + cfg.WhitelistedContainers, + cfg.TestkubeImageCredentialsCacheTTL, + ) + commons.ExitOnError("Creating container executor", err) + + sched := scheduler.NewScheduler( + metrics, + executor, + containerExecutor, + deprecatedRepositories, + deprecatedClients, + secretClient, + eventsEmitter, + log.DefaultLogger, + configMapConfig, + configMapClient, + eventBus, + cfg.TestkubeDashboardURI, + features, + logsStream, + cfg.TestkubeNamespace, + cfg.TestkubeProTLSSecret, + cfg.TestkubeProRunnerCustomCASecret, + subscriptionChecker, + ) + + storageParams := deprecatedapiv1.StorageParams{ + SSL: cfg.StorageSSL, + SkipVerify: cfg.StorageSkipVerify, + CertFile: cfg.StorageCertFile, + KeyFile: cfg.StorageKeyFile, + CAFile: cfg.StorageCAFile, + Endpoint: cfg.StorageEndpoint, + AccessKeyId: cfg.StorageAccessKeyID, + SecretAccessKey: cfg.StorageSecretAccessKey, + Region: cfg.StorageRegion, + Token: cfg.StorageToken, + Bucket: cfg.StorageBucket, + } + // Use direct MinIO artifact storage for deprecated API for backwards compatibility + var deprecatedArtifactStorage storage.ArtifactsStorage + if mode == common.ModeAgent { + deprecatedArtifactStorage = cloudartifacts.NewCloudArtifactsStorage(grpcClient, grpcConn, cfg.TestkubeProAPIKey) + } else { + deprecatedArtifactStorage = minio.NewMinIOArtifactClient(commons.MustGetMinioClient(cfg)) + } + deprecatedApi := deprecatedapiv1.NewDeprecatedTestkubeAPI( + deprecatedRepositories, + deprecatedClients, + cfg.TestkubeNamespace, + secretClient, + eventsEmitter, + executor, + containerExecutor, + metrics, + sched, + cfg.GraphqlPort, + deprecatedArtifactStorage, + mode, + eventBus, + secretConfig, + features, + logsStream, + logGrpcClient, + proContext, + storageParams, + ) + + var reconcilerClient *reconciler.Client + if !cfg.DisableReconciler { + reconcilerClient = reconciler.NewClient(clientset, deprecatedRepositories, deprecatedClients, log.DefaultLogger) + } else { + log.DefaultLogger.Info("reconciler is disabled") + } + + return &DeprecatedSystem{ + Clients: deprecatedClients, + Repositories: deprecatedRepositories, + Scheduler: sched, + Reconciler: reconcilerClient, + API: &deprecatedApi, + StreamLogs: deprecatedApi.GetLogsStream, + } +} diff --git a/cmd/api-server/services/executionworker.go b/cmd/api-server/services/executionworker.go index b78965689cd..4b5db6af44d 100644 --- a/cmd/api-server/services/executionworker.go +++ b/cmd/api-server/services/executionworker.go @@ -24,25 +24,6 @@ func CreateExecutionWorker( for n, s := range serviceAccountNames { namespacesConfig[n] = kubernetesworker.NamespaceConfig{DefaultServiceAccountName: s} } - cloudUrl := cfg.TestkubeProURL - cloudApiKey := cfg.TestkubeProAPIKey - objectStorageConfig := testworkflowconfig.ObjectStorageConfig{} - if cloudApiKey == "" { - cloudUrl = "" - objectStorageConfig = testworkflowconfig.ObjectStorageConfig{ - Endpoint: cfg.StorageEndpoint, - AccessKeyID: cfg.StorageAccessKeyID, - SecretAccessKey: cfg.StorageSecretAccessKey, - Region: cfg.StorageRegion, - Token: cfg.StorageToken, - Bucket: cfg.StorageBucket, - Ssl: cfg.StorageSSL, - SkipVerify: cfg.StorageSkipVerify, - CertFile: cfg.StorageCertFile, - KeyFile: cfg.StorageKeyFile, - CAFile: cfg.StorageCAFile, - } - } return executionworker.NewKubernetes(clientSet, processor, kubernetesworker.Config{ Cluster: kubernetesworker.ClusterConfig{ Id: clusterId, @@ -56,14 +37,13 @@ func CreateExecutionWorker( CacheTTL: cfg.TestkubeImageCredentialsCacheTTL, }, Connection: testworkflowconfig.WorkerConnectionConfig{ - Url: cloudUrl, - ApiKey: cloudApiKey, + Url: cfg.TestkubeProURL, + ApiKey: cfg.TestkubeProAPIKey, SkipVerify: cfg.TestkubeProSkipVerify, TlsInsecure: cfg.TestkubeProTLSInsecure, // TODO: Prepare ControlPlane interface for OSS, so we may unify the communication - LocalApiUrl: fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), - ObjectStorage: objectStorageConfig, + LocalApiUrl: fmt.Sprintf("http://%s:%d", cfg.APIServerFullname, cfg.APIServerPort), }, }) } diff --git a/cmd/testworkflow-toolkit/artifacts/direct_uploader.go b/cmd/testworkflow-toolkit/artifacts/direct_uploader.go deleted file mode 100644 index 66cd4b3c545..00000000000 --- a/cmd/testworkflow-toolkit/artifacts/direct_uploader.go +++ /dev/null @@ -1,105 +0,0 @@ -package artifacts - -import ( - "context" - "fmt" - "io" - "sync" - "sync/atomic" - - minio2 "github.com/minio/minio-go/v7" - - "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/env" - "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/env/config" - "github.com/kubeshop/testkube/pkg/storage/minio" - "github.com/kubeshop/testkube/pkg/ui" -) - -type PutObjectOptionsEnhancer = func(options *minio2.PutObjectOptions, path string, size int64) - -func NewDirectUploader(opts ...DirectUploaderOpt) Uploader { - uploader := &directUploader{ - parallelism: 1, - options: make([]PutObjectOptionsEnhancer, 0), - } - for _, opt := range opts { - opt(uploader) - } - return uploader -} - -type directUploader struct { - client *minio.Client - wg sync.WaitGroup - sema chan struct{} - parallelism int - error atomic.Bool - options []PutObjectOptionsEnhancer -} - -func (d *directUploader) Start() (err error) { - d.client, err = env.ObjectStorageClient() - d.sema = make(chan struct{}, d.parallelism) - return err -} - -func (d *directUploader) buildOptions(path string, size int64) (options minio2.PutObjectOptions) { - for _, enhance := range d.options { - enhance(&options, path, size) - } - if options.ContentType == "" { - options.ContentType = "application/octet-stream" - } - return options -} - -func (d *directUploader) upload(path string, file io.Reader, size int64) { - ns := config.ExecutionId() - opts := d.buildOptions(path, size) - err := d.client.SaveFileDirect(context.Background(), ns, path, file, size, opts) - - if err != nil { - d.error.Store(true) - ui.Errf("%s: failed: %s", path, err.Error()) - return - } -} - -func (d *directUploader) Add(path string, file io.Reader, size int64) error { - d.wg.Add(1) - d.sema <- struct{}{} - go func() { - d.upload(path, file, size) - if f, ok := file.(io.Closer); ok { - _ = f.Close() - } - d.wg.Done() - <-d.sema - }() - return nil -} - -func (d *directUploader) End() error { - d.wg.Wait() - if d.error.Load() { - return fmt.Errorf("upload failed") - } - return nil -} - -type DirectUploaderOpt = func(uploader *directUploader) - -func WithParallelism(parallelism int) DirectUploaderOpt { - return func(uploader *directUploader) { - if parallelism < 1 { - parallelism = 1 - } - uploader.parallelism = parallelism - } -} - -func WithMinioOptionsEnhancer(fn PutObjectOptionsEnhancer) DirectUploaderOpt { - return func(uploader *directUploader) { - uploader.options = append(uploader.options, fn) - } -} diff --git a/cmd/testworkflow-toolkit/artifacts/internalartifactstorage.go b/cmd/testworkflow-toolkit/artifacts/internalartifactstorage.go index 08c33cf2a08..1e5a8a753c8 100644 --- a/cmd/testworkflow-toolkit/artifacts/internalartifactstorage.go +++ b/cmd/testworkflow-toolkit/artifacts/internalartifactstorage.go @@ -31,13 +31,10 @@ type internalArtifactStorage struct { } func newArtifactUploader() Uploader { - if config.CloudEnabled() { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - client, _ := env.Cloud(ctx) - return NewCloudUploader(client, WithParallelismCloud(30), CloudDetectMimetype) - } - return NewDirectUploader(WithParallelism(30), DirectDetectMimetype) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + client, _ := env.Cloud(ctx) + return NewCloudUploader(client, WithParallelismCloud(30), CloudDetectMimetype) } func InternalStorage() InternalArtifactStorage { diff --git a/cmd/testworkflow-toolkit/artifacts/options.go b/cmd/testworkflow-toolkit/artifacts/options.go index 152679bf0d2..eb02f9458b9 100644 --- a/cmd/testworkflow-toolkit/artifacts/options.go +++ b/cmd/testworkflow-toolkit/artifacts/options.go @@ -2,8 +2,6 @@ package artifacts import ( "net/http" - - "github.com/minio/minio-go/v7" ) var CloudDetectMimetype = WithRequestEnhancerCloud(func(req *http.Request, path string, size int64) { @@ -17,9 +15,3 @@ var CloudDetectMimetype = WithRequestEnhancerCloud(func(req *http.Request, path } } }) - -var DirectDetectMimetype = WithMinioOptionsEnhancer(func(options *minio.PutObjectOptions, path string, size int64) { - if options.ContentType == "" { - options.ContentType = DetectMimetype(path) - } -}) diff --git a/cmd/testworkflow-toolkit/commands/artifacts.go b/cmd/testworkflow-toolkit/commands/artifacts.go index cfeb7d4a77e..657c3e22930 100644 --- a/cmd/testworkflow-toolkit/commands/artifacts.go +++ b/cmd/testworkflow-toolkit/commands/artifacts.go @@ -19,7 +19,6 @@ import ( "github.com/kubeshop/testkube/pkg/filesystem" - "github.com/minio/minio-go/v7" "github.com/spf13/cobra" "github.com/kubeshop/testkube/cmd/testworkflow-toolkit/artifacts" @@ -28,22 +27,6 @@ import ( "github.com/kubeshop/testkube/pkg/ui" ) -var directAddGzipEncoding = artifacts.WithMinioOptionsEnhancer(func(options *minio.PutObjectOptions, path string, size int64) { - options.ContentType = "application/gzip" - options.ContentEncoding = "gzip" -}) - -var directDisableMultipart = artifacts.WithMinioOptionsEnhancer(func(options *minio.PutObjectOptions, path string, size int64) { - options.DisableMultipart = true -}) - -var directUnpack = artifacts.WithMinioOptionsEnhancer(func(options *minio.PutObjectOptions, path string, size int64) { - options.UserMetadata = map[string]string{ - "X-Amz-Meta-Snowball-Auto-Extract": "true", - "X-Amz-Meta-Minio-Snowball-Prefix": config.WorkflowName() + "/" + config.ExecutionId(), - } -}) - var cloudAddGzipEncoding = artifacts.WithRequestEnhancerCloud(func(req *http.Request, path string, size int64) { req.Header.Set("Content-Type", "application/gzip") req.Header.Set("Content-Encoding", "gzip") @@ -99,48 +82,34 @@ func NewArtifactsCmd() *cobra.Command { var handlerOpts []artifacts.HandlerOpts // Archive - if config.CloudEnabled() { - ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second) - defer cancel() - ctx = agent.AddAPIKeyMeta(ctx, config.Config().Worker.Connection.ApiKey) - executor, client := env.Cloud(ctx) - proContext, err := client.GetProContext(ctx, &emptypb.Empty{}) - var supported []*cloud.Capability - if err != nil { - fmt.Printf("Warning: couldn't get capabilities: %s\n", err.Error()) - } - if proContext != nil { - supported = proContext.Capabilities - } - defer executor.Close() + ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second) + defer cancel() + ctx = agent.AddAPIKeyMeta(ctx, config.Config().Worker.Connection.ApiKey) + executor, client := env.Cloud(ctx) + proContext, err := client.GetProContext(ctx, &emptypb.Empty{}) + var supported []*cloud.Capability + if err != nil { + fmt.Printf("Warning: couldn't get capabilities: %s\n", err.Error()) + } + if proContext != nil { + supported = proContext.Capabilities + } + defer executor.Close() - if config.JUnitParserEnabled() || capabilities.Enabled(supported, capabilities.CapabilityJUnitReports) { - junitProcessor := artifacts.NewJUnitPostProcessor(filesystem.NewOSFileSystem(), executor, walker.Root(), config.Config().Resource.FsPrefix) - handlerOpts = append(handlerOpts, artifacts.WithPostProcessor(junitProcessor)) - } - if compress != "" { - processor = artifacts.NewTarCachedProcessor(compress, compressCachePath) - opts := []artifacts.CloudUploaderOpt{cloudAddGzipEncoding} - if unpack { - opts = append(opts, cloudUnpack) - } - uploader = artifacts.NewCloudUploader(executor, opts...) - } else { - processor = artifacts.NewDirectProcessor() - uploader = artifacts.NewCloudUploader(executor, artifacts.WithParallelismCloud(30), artifacts.CloudDetectMimetype) - } - } else if compress != "" && unpack { - processor = artifacts.NewTarCachedProcessor(compress, compressCachePath) - uploader = artifacts.NewDirectUploader(directAddGzipEncoding, directDisableMultipart, directUnpack) - } else if compress != "" && compressCachePath != "" { + if config.JUnitParserEnabled() || capabilities.Enabled(supported, capabilities.CapabilityJUnitReports) { + junitProcessor := artifacts.NewJUnitPostProcessor(filesystem.NewOSFileSystem(), executor, walker.Root(), config.Config().Resource.FsPrefix) + handlerOpts = append(handlerOpts, artifacts.WithPostProcessor(junitProcessor)) + } + if compress != "" { processor = artifacts.NewTarCachedProcessor(compress, compressCachePath) - uploader = artifacts.NewDirectUploader(directAddGzipEncoding, directDisableMultipart) - } else if compress != "" { - processor = artifacts.NewTarProcessor(compress) - uploader = artifacts.NewDirectUploader(directAddGzipEncoding) + opts := []artifacts.CloudUploaderOpt{cloudAddGzipEncoding} + if unpack { + opts = append(opts, cloudUnpack) + } + uploader = artifacts.NewCloudUploader(executor, opts...) } else { processor = artifacts.NewDirectProcessor() - uploader = artifacts.NewDirectUploader(artifacts.WithParallelism(30), artifacts.DirectDetectMimetype) + uploader = artifacts.NewCloudUploader(executor, artifacts.WithParallelismCloud(30), artifacts.CloudDetectMimetype) } // Isolate the files under specific prefix diff --git a/cmd/testworkflow-toolkit/env/client.go b/cmd/testworkflow-toolkit/env/client.go index 87af1041873..0f1b244af99 100644 --- a/cmd/testworkflow-toolkit/env/client.go +++ b/cmd/testworkflow-toolkit/env/client.go @@ -24,7 +24,6 @@ import ( "github.com/kubeshop/testkube/pkg/k8sclient" "github.com/kubeshop/testkube/pkg/log" "github.com/kubeshop/testkube/pkg/secret" - "github.com/kubeshop/testkube/pkg/storage/minio" "github.com/kubeshop/testkube/pkg/ui" ) @@ -77,13 +76,6 @@ func Testkube() client.Client { return client.NewDirectAPIClient(httpClient, sseClient, fmt.Sprintf("http://%s:%d", config.APIServerName, config.APIServerPort), "") } -func ObjectStorageClient() (*minio.Client, error) { - cfg := config2.Config().Worker.Connection.ObjectStorage - opts := minio.GetTLSOptions(cfg.Ssl, cfg.SkipVerify, cfg.CertFile, cfg.KeyFile, cfg.CAFile) - c := minio.NewClient(cfg.Endpoint, cfg.AccessKeyID, cfg.SecretAccessKey, cfg.Region, cfg.Token, cfg.Bucket, opts...) - return c, c.Connect() -} - func Cloud(ctx context.Context) (cloudexecutor.Executor, cloud.TestKubeCloudAPIClient) { cfg := config2.Config().Worker.Connection grpcConn, err := agent.NewGRPCConnection(ctx, cfg.TlsInsecure, cfg.SkipVerify, cfg.Url, "", "", "", log.DefaultLogger) diff --git a/cmd/testworkflow-toolkit/env/config/config.go b/cmd/testworkflow-toolkit/env/config/config.go index c016715310a..a9e16300ce3 100644 --- a/cmd/testworkflow-toolkit/env/config/config.go +++ b/cmd/testworkflow-toolkit/env/config/config.go @@ -50,10 +50,6 @@ func Debug() bool { return Config().Execution.Debug || os.Getenv("DEBUG") == "1" } -func CloudEnabled() bool { - return Config().Worker.Connection.ApiKey != "" -} - func UseProxy() bool { return UseProxyValue } diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 00000000000..d02d45661ba --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,46 @@ +variable "GOCACHE" { + default = "/go/pkg" +} +variable "GOMODCACHE" { + default = "/root/.cache/go-build" +} + +group "default" { + targets = ["agent-server", "testworkflow-init", "testworkflow-toolkit"] +} + +target "agent-server-meta" {} +target "agent-server" { + inherits = ["agent-server-meta"] + context="." + dockerfile = "build/_local/agent-server.Dockerfile" + platforms = ["linux/arm64"] + args = { + GOCACHE = "${GOCACHE}" + GOMODCACHE = "${GOMODCACHE}" + } +} + +target "testworkflow-init-meta" {} +target "testworkflow-init" { + inherits = ["testworkflow-init-meta"] + context="." + dockerfile = "build/_local/testworkflow-init.Dockerfile" + platforms = ["linux/arm64"] + args = { + GOCACHE = "${GOCACHE}" + GOMODCACHE = "${GOMODCACHE}" + } +} + +target "testworkflow-toolkit-meta" {} +target "testworkflow-toolkit" { + inherits = ["testworkflow-toolkit-meta"] + context="." + dockerfile = "build/_local/testworkflow-toolkit.Dockerfile" + platforms = ["linux/arm64"] + args = { + GOCACHE = "${GOCACHE}" + GOMODCACHE = "${GOMODCACHE}" + } +} diff --git a/go.mod b/go.mod index 11ac926defe..c413e8a7376 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/creasty/defaults v1.7.0 github.com/denisbrodbeck/machineid v1.0.1 - github.com/docker/docker v25.0.6+incompatible + github.com/docker/docker v27.3.1+incompatible github.com/dustin/go-humanize v1.0.1 github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 github.com/fasthttp/websocket v1.5.0 @@ -29,10 +29,12 @@ require ( github.com/google/uuid v1.6.0 github.com/gookit/color v1.5.4 github.com/gorilla/websocket v1.5.0 + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/h2non/filetype v1.1.3 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/itchyny/gojq v0.12.14 github.com/joshdk/go-junit v1.0.0 + github.com/json-iterator/go v1.1.12 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/kelseyhightower/envconfig v1.4.0 github.com/kubepug/kubepug v1.7.1 @@ -46,6 +48,7 @@ require ( github.com/olekukonko/tablewriter v0.0.6-0.20230925090304-df64c4bbad77 github.com/onsi/ginkgo/v2 v2.15.0 github.com/onsi/gomega v1.31.0 + github.com/opencontainers/image-spec v1.1.0-rc3 github.com/otiai10/copy v1.11.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 @@ -63,12 +66,12 @@ require ( go.mongodb.org/mongo-driver v1.11.3 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/oauth2 v0.21.0 + golang.org/x/oauth2 v0.22.0 golang.org/x/sync v0.8.0 - golang.org/x/text v0.18.0 + golang.org/x/text v0.19.0 google.golang.org/appengine v1.6.8 - google.golang.org/grpc v1.66.1 - google.golang.org/protobuf v1.34.2 + google.golang.org/grpc v1.67.1 + google.golang.org/protobuf v1.35.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.28.4 @@ -88,6 +91,7 @@ require ( github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/brotli v1.0.5 // indirect + github.com/avast/retry-go/v4 v4.6.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect @@ -140,7 +144,6 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/leodido/go-urn v1.2.1 // indirect @@ -159,6 +162,7 @@ require ( github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -169,7 +173,6 @@ require ( github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc3 // indirect github.com/package-url/packageurl-go v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -204,22 +207,22 @@ require ( github.com/yuin/goldmark v1.4.13 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/sdk v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.27.0 // indirect + golang.org/x/crypto v0.28.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/term v0.24.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index a9718e6e703..9ed62c0b3b0 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,11 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= +github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= +github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= @@ -156,8 +159,8 @@ github.com/docker/cli v24.0.0+incompatible h1:0+1VshNwBQzQAx9lOl+OYCTCEAD8fKs/qe github.com/docker/cli v24.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg= -github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= +github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -198,6 +201,8 @@ github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -224,6 +229,7 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= @@ -330,6 +336,8 @@ github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= @@ -381,6 +389,7 @@ github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuOb github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -440,6 +449,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= @@ -492,6 +503,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/otiai10/copy v1.11.0 h1:OKBD80J/mLBrwnzXqGtFCzprFSGioo30JcmR4APsNwc= github.com/otiai10/copy v1.11.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= @@ -499,6 +511,7 @@ github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAq github.com/package-url/packageurl-go v0.1.0 h1:efWBc98O/dBZRg1pw2xiDzovnlMjCa9NPnfaiBduh8I= github.com/package-url/packageurl-go v0.1.0/go.mod h1:C/ApiuWpmbpni4DIOECf6WCjFUZV7O1Fx7VAzrZHgBw= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -562,6 +575,7 @@ github.com/shurcooL/githubv4 v0.0.0-20220520033151-0b4e3294ff00 h1:fiFvD4lT0aWju github.com/shurcooL/githubv4 v0.0.0-20220520033151-0b4e3294ff00/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 h1:B1PEwpArrNp4dkQrfxh/abbBAOZBVp0ds+fBEOUOqOc= github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -576,6 +590,7 @@ github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRM github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= @@ -649,26 +664,30 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0/go.mod h1:4lVs6obhSVRb1EW5FhOuBTyiQhtRtAnnva9vD3yRfq8= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= -go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= -go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQvwAgSyisUghWoY20I7huthMk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -683,8 +702,8 @@ golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -765,8 +784,8 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -776,8 +795,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -799,6 +818,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -840,6 +860,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -859,16 +880,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -880,8 +901,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -902,6 +923,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -996,6 +1018,7 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= @@ -1012,10 +1035,10 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= -google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= -google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg= +google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1032,8 +1055,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM= -google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1046,8 +1069,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/config/config.go b/internal/config/config.go index d763de318a9..a4b1fd15975 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,6 +10,7 @@ type Config struct { APIServerPort int `envconfig:"APISERVER_PORT" default:"8088"` APIServerConfig string `envconfig:"APISERVER_CONFIG" default:""` APIServerFullname string `envconfig:"APISERVER_FULLNAME" default:"testkube-api-server"` + GRPCServerPort int `envconfig:"GRPCSERVER_PORT" default:"8089"` APIMongoDSN string `envconfig:"API_MONGO_DSN" default:"mongodb://localhost:27017"` APIMongoAllowTLS bool `envconfig:"API_MONGO_ALLOW_TLS" default:"false"` APIMongoSSLCert string `envconfig:"API_MONGO_SSL_CERT" default:""` @@ -117,6 +118,8 @@ type Config struct { GlobalWorkflowTemplateName string `envconfig:"TESTKUBE_GLOBAL_WORKFLOW_TEMPLATE_NAME" default:""` EnableK8sEvents bool `envconfig:"ENABLE_K8S_EVENTS" default:"true"` TestkubeDockerImageVersion string `envconfig:"TESTKUBE_DOCKER_IMAGE_VERSION" default:""` + DisableDeprecatedTests bool `envconfig:"DISABLE_DEPRECATED_TESTS" default:"false"` + DisableWebhooks bool `envconfig:"DISABLE_WEBHOOKS" default:"false"` // DEPRECATED: Use TestkubeProAPIKey instead TestkubeCloudAPIKey string `envconfig:"TESTKUBE_CLOUD_API_KEY" default:""` diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 575b618e401..116c7198127 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -308,7 +308,9 @@ func (ag *Agent) receiveCommand(ctx context.Context, stream cloud.TestKubeCloudA } func (ag *Agent) runCommandLoop(ctx context.Context) error { - ctx = AddAPIKeyMeta(ctx, ag.proContext.APIKey) + if ag.proContext.APIKey != "" { + ctx = AddAPIKeyMeta(ctx, ag.proContext.APIKey) + } ctx = metadata.AppendToOutgoingContext(ctx, clusterIDMeta, ag.clusterID) ctx = metadata.AppendToOutgoingContext(ctx, cloudMigrateMeta, ag.proContext.Migrate) diff --git a/pkg/agent/events.go b/pkg/agent/events.go index 2af87a793d2..957bfcdae94 100644 --- a/pkg/agent/events.go +++ b/pkg/agent/events.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" "google.golang.org/grpc" "google.golang.org/grpc/encoding/gzip" - "google.golang.org/grpc/metadata" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/cloud" @@ -62,8 +61,9 @@ func (ag *Agent) Notify(event testkube.Event) (result testkube.EventResult) { func (ag *Agent) runEventLoop(ctx context.Context) error { opts := []grpc.CallOption{grpc.UseCompressor(gzip.Name)} - md := metadata.Pairs(apiKeyMeta, ag.apiKey) - ctx = metadata.NewOutgoingContext(ctx, md) + if ag.apiKey != "" { + ctx = AddAPIKeyMeta(ctx, ag.apiKey) + } stream, err := ag.client.Send(ctx, opts...) if err != nil { diff --git a/pkg/cloud/client/rest.go b/pkg/cloud/client/rest.go index df2d2489f56..23f826d6ab9 100644 --- a/pkg/cloud/client/rest.go +++ b/pkg/cloud/client/rest.go @@ -112,3 +112,32 @@ func (c RESTClient[T]) Create(entity T, overridePath ...string) (e T, err error) return e, nil } + +func (c RESTClient[T]) Delete(id string, overridePath ...string) (err error) { + path := c.Path + "/" + id + if len(overridePath) == 1 { + path = overridePath[0] + } + + r, err := nethttp.NewRequest("DELETE", c.BaseUrl+path, nil) + if err != nil { + return err + } + r.Header.Add("Content-type", "application/json") + r.Header.Add("Authorization", "Bearer "+c.Token) + + resp, err := c.Client.Do(r) + if err != nil { + return err + } + + if resp.StatusCode >= 400 { + d, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("error deleting %s: can't read response: %s", c.Path, err) + } + return fmt.Errorf("error creating %s: %s", c.Path, d) + } + + return nil +} diff --git a/pkg/cloud/data/result/result_models.go b/pkg/cloud/data/result/result_models.go index 2c523a9ef4f..c1afe539093 100644 --- a/pkg/cloud/data/result/result_models.go +++ b/pkg/cloud/data/result/result_models.go @@ -113,6 +113,9 @@ type EndExecutionRequest struct { type EndExecutionResponse struct { } +type GetLabelsRequest struct { +} + type GetLabelsResponse struct { Labels map[string][]string `json:"labels"` } @@ -147,6 +150,9 @@ type DeleteByTestSuitesRequest struct { type DeleteByTestSuitesResponse struct{} +type DeleteForAllTestSuitesRequest struct { +} + type DeleteForAllTestSuitesResponse struct { } diff --git a/pkg/controlplane/errors.go b/pkg/controlplane/errors.go new file mode 100644 index 00000000000..880dc85867b --- /dev/null +++ b/pkg/controlplane/errors.go @@ -0,0 +1,63 @@ +package controlplane + +type Entity string + +func IsNotFoundErr(err error) bool { + _, ok := err.(*ErrNotFound) + return ok +} + +func NewNotFoundErr(entity Entity) *ErrNotFound { + return &ErrNotFound{ + entity: entity, + } +} + +type ErrNotFound struct { + entity Entity + id string + parentId string + err error +} + +func (e *ErrNotFound) Unwrap() error { + return e.err +} + +func (e *ErrNotFound) WithEntity(entity Entity) *ErrNotFound { + e.entity = entity + return e +} + +func (e *ErrNotFound) WithId(id string) *ErrNotFound { + e.id = id + return e +} + +func (e *ErrNotFound) WithParentId(id string) *ErrNotFound { + e.parentId = id + return e +} + +func (e *ErrNotFound) WithErr(err error) *ErrNotFound { + e.err = err + return e +} + +func (e ErrNotFound) Error() string { + msg := string(e.entity) + if e.id != "" { + msg += " id:" + e.id + } + + if e.parentId != "" { + msg += " for:" + e.parentId + } + + msg += " not found" + + if e.err != nil { + msg += " error:" + e.err.Error() + } + return msg +} diff --git a/pkg/controlplane/server.go b/pkg/controlplane/server.go new file mode 100644 index 00000000000..8589571a22c --- /dev/null +++ b/pkg/controlplane/server.go @@ -0,0 +1,264 @@ +package controlplane + +import ( + "context" + "fmt" + "math" + "net" + "time" + + "github.com/gofiber/fiber/v2/log" + grpczap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" + grpcrecovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" + grpcctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" + "github.com/pkg/errors" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/kubeshop/testkube/pkg/cloud" + "github.com/kubeshop/testkube/pkg/cloud/data/executor" +) + +const ( + KeepAliveTime = 10 * time.Second + KeepAliveTimeout = 5 * time.Second + HealthCheckInterval = 60 * time.Second + SendPingInterval = HealthCheckInterval / 2 +) + +type Server struct { + cloud.UnimplementedTestKubeCloudAPIServer + cfg Config + server *grpc.Server + commands map[executor.Command]CommandHandler +} + +type Config struct { + Port int + Verbose bool + Logger *zap.SugaredLogger +} + +func New(cfg Config, commandGroups ...CommandHandlers) *Server { + commands := make(map[executor.Command]CommandHandler) + for _, group := range commandGroups { + for cmd, handler := range group { + commands[cmd] = handler + } + } + return &Server{ + cfg: cfg, + commands: commands, + } +} + +func (s *Server) GetProContext(_ context.Context, _ *emptypb.Empty) (*cloud.ProContextResponse, error) { + return nil, status.Error(codes.Unimplemented, "not supported in the standalone version") +} + +func (s *Server) ExecuteAsync(srv cloud.TestKubeCloudAPI_ExecuteAsyncServer) error { + ctx, cancel := context.WithCancel(srv.Context()) + g, _ := errgroup.WithContext(ctx) + defer cancel() + + // Ignore all the messages + g.Go(func() error { + for { + select { + case <-ctx.Done(): + return nil + default: + srv.Recv() + } + } + }) + + g.Go(func() error { + ticker := time.NewTicker(HealthCheckInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + messageId := fmt.Sprintf("hs%d", time.Now().UnixNano()) + req := &cloud.ExecuteRequest{Url: "healthcheck", MessageId: messageId} + err := srv.Send(req) + if err != nil { + log.Errorw("failed to publish agent healthcheck", "error", err) + } + } + } + }) + + return g.Wait() +} + +// TODO: Consider deleting that +func (s *Server) GetTestWorkflowNotificationsStream(srv cloud.TestKubeCloudAPI_GetTestWorkflowNotificationsStreamServer) error { + ctx, cancel := context.WithCancel(srv.Context()) + defer cancel() + g, _ := errgroup.WithContext(ctx) + + ticker := time.NewTicker(SendPingInterval) + defer ticker.Stop() + + g.Go(func() error { + for { + select { + case <-ticker.C: + srv.Send(&cloud.TestWorkflowNotificationsRequest{ + StreamId: "ping", + ExecutionId: "ping", + RequestType: cloud.TestWorkflowNotificationsRequestType_WORKFLOW_STREAM_HEALTH_CHECK, + }) + case <-ctx.Done(): + return nil + } + } + }) + + // Ignore all the messages + g.Go(func() error { + for { + select { + case <-ctx.Done(): + return nil + default: + srv.Recv() + } + } + }) + + return g.Wait() +} + +// Send is called on agent client, returning from this method closes the connection +func (s *Server) Send(srv cloud.TestKubeCloudAPI_SendServer) error { + for { + if err := srv.Context().Err(); err != nil { + log.Info("agent websocket stream is canceled, agent client is disconnected") + return nil + } + + _, err := srv.Recv() + if err != nil { + errMsg := "failed to receive websocket message" + log.Errorw(errMsg, "error", err) + return errors.Wrap(err, errMsg) + } + } +} + +// TODO: Consider deleting that +func (s *Server) GetLogsStream(srv cloud.TestKubeCloudAPI_GetLogsStreamServer) error { + ctx, cancel := context.WithCancel(srv.Context()) + defer cancel() + g, _ := errgroup.WithContext(ctx) + + ticker := time.NewTicker(SendPingInterval) + defer ticker.Stop() + + g.Go(func() error { + for { + select { + case <-ticker.C: + srv.Send(&cloud.LogsStreamRequest{ + StreamId: "ping", + ExecutionId: "ping", + RequestType: cloud.LogsStreamRequestType_STREAM_HEALTH_CHECK, + }) + case <-ctx.Done(): + return nil + } + } + }) + + // Ignore all the messages + g.Go(func() error { + for { + select { + case <-ctx.Done(): + return nil + default: + srv.Recv() + } + } + }) + + return g.Wait() +} + +func (s *Server) Call(ctx context.Context, request *cloud.CommandRequest) (*cloud.CommandResponse, error) { + if cmd, ok := s.commands[executor.Command(request.Command)]; ok { + return cmd(ctx, request) + } + return nil, errors.Errorf("command not implemented: %s", request.Command) +} + +func (s *Server) Run(ctx context.Context) error { + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.Port)) + if err != nil { + return errors.Errorf("failed to listen for GraphQL server: %v", err) + } + var opts []grpc.ServerOption + + // Create a server, make sure we put the grpcctxtags context before everything else. + creds := insecure.NewCredentials() + + // default MaxRecvMsgSize is 4Mib, which causes trouble + opts = append(opts, + grpc.Creds(creds), + grpc.MaxRecvMsgSize(math.MaxInt32), + grpc.ChainUnaryInterceptor(grpcrecovery.UnaryServerInterceptor()), + grpc.ChainStreamInterceptor(grpcrecovery.StreamServerInterceptor()), + ) + if s.cfg.Verbose { + // Shared options for the logger, with a custom gRPC code to log level function. + logger := s.cfg.Logger.Desugar() + grpczap.ReplaceGrpcLoggerV2(logger) + opts = append( + opts, + grpc.ChainUnaryInterceptor( + grpcctxtags.UnaryServerInterceptor(grpcctxtags.WithFieldExtractor(grpcctxtags.CodeGenRequestFieldExtractor)), + grpczap.UnaryServerInterceptor(logger), + ), + ) + opts = append( + opts, + grpc.ChainStreamInterceptor( + grpcctxtags.StreamServerInterceptor(grpcctxtags.WithFieldExtractor(grpcctxtags.CodeGenRequestFieldExtractor)), + grpczap.StreamServerInterceptor(logger), + ), + ) + } + opts = append(opts, + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{PermitWithoutStream: true}), + grpc.KeepaliveParams(keepalive.ServerParameters{Time: KeepAliveTime, Timeout: KeepAliveTimeout})) + grpcServer := grpc.NewServer(opts...) + + cloud.RegisterTestKubeCloudAPIServer(grpcServer, s) + s.server = grpcServer + go func() { + <-ctx.Done() + s.Shutdown() + }() + err = grpcServer.Serve(ln) + if err != nil { + return errors.Wrap(err, "grpc server error") + } + return nil +} + +// TODO: Use this when context is down +func (s *Server) Shutdown() { + if s.server != nil { + s.server.GracefulStop() + } +} diff --git a/pkg/controlplane/utils.go b/pkg/controlplane/utils.go new file mode 100644 index 00000000000..52907166c46 --- /dev/null +++ b/pkg/controlplane/utils.go @@ -0,0 +1,70 @@ +package controlplane + +import ( + "context" + "encoding/json" + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/mongo" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/kubeshop/testkube/pkg/cloud" + "github.com/kubeshop/testkube/pkg/cloud/data/executor" + "github.com/kubeshop/testkube/pkg/log" +) + +type grpcstatus interface { + GRPCStatus() *status.Status +} + +type CommandHandler func(ctx context.Context, req *cloud.CommandRequest) (*cloud.CommandResponse, error) +type CommandHandlers map[executor.Command]CommandHandler + +func Handler[T any, U any](fn func(ctx context.Context, payload T) (U, error)) func(ctx context.Context, req *cloud.CommandRequest) (*cloud.CommandResponse, error) { + return func(ctx context.Context, req *cloud.CommandRequest) (*cloud.CommandResponse, error) { + data, _ := read[T](req.Payload) + value, err := fn(ctx, data) + if err != nil { + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, status.Error(codes.NotFound, NewNotFoundErr("").Error()) + } + if _, ok := err.(grpcstatus); ok { + return nil, err + } + log.DefaultLogger.Errorw(fmt.Sprintf("command %s failed", req.Command), "error", err) + return nil, status.Error(codes.Internal, err.Error()) + } + return marshal(value) + } +} + +func read[T any](payload *structpb.Struct) (v T, err error) { + err = cycleJSON(payload, &v) + if err != nil { + return v, status.Error(codes.Internal, "error unmarshalling payload") + } + return v, nil +} + +func marshal(response any) (*cloud.CommandResponse, error) { + jsonResponse, err := json.Marshal(response) + commandResponse := cloud.CommandResponse{Response: jsonResponse} + return &commandResponse, err +} + +func cycleJSON(src any, tgt any) error { + b, _ := toJSON(src) + return fromJSON(b, tgt) +} + +func toJSON(src any) (json.RawMessage, error) { + return jsoniter.Marshal(src) +} + +func fromJSON(msg json.RawMessage, tgt any) error { + return jsoniter.Unmarshal(msg, tgt) +} diff --git a/pkg/event/bus/nats.go b/pkg/event/bus/nats.go index 79e61d1ebff..f652882cbb2 100644 --- a/pkg/event/bus/nats.go +++ b/pkg/event/bus/nats.go @@ -6,15 +6,19 @@ import ( "sync" "time" + "github.com/avast/retry-go/v4" "github.com/nats-io/nats.go" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/event/kind/common" "github.com/kubeshop/testkube/pkg/log" + "github.com/kubeshop/testkube/pkg/utils" ) var ( _ Bus = (*NATSBus)(nil) + + NATS_RETRY_ATTEMPTS uint = 20 ) const ( @@ -80,7 +84,15 @@ func NewNATSEncodedConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats. func NewNATSConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats.Conn, error) { opts = append(opts, optsFromConfig(cfg)...) - nc, err := nats.Connect(cfg.NatsURI, opts...) + log.DefaultLogger.Infoln("Connecting to NATS") + nc, err := retry.DoWithData( + func() (*nats.Conn, error) { + return nats.Connect(cfg.NatsURI, opts...) + }, + retry.DelayType(retry.FixedDelay), + retry.Delay(utils.DefaultRetryDelay), + retry.Attempts(NATS_RETRY_ATTEMPTS), + ) if err != nil { log.DefaultLogger.Fatalw("error connecting to nats", "error", err) return nil, err diff --git a/pkg/event/kind/webhook/listener.go b/pkg/event/kind/webhook/listener.go index 605c5e2e086..41615aa62f1 100644 --- a/pkg/event/kind/webhook/listener.go +++ b/pkg/event/kind/webhook/listener.go @@ -12,14 +12,13 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" + "github.com/kubeshop/testkube/cmd/api-server/commons" v1 "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/internal/config" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/event/kind/common" thttp "github.com/kubeshop/testkube/pkg/http" "github.com/kubeshop/testkube/pkg/log" - "github.com/kubeshop/testkube/pkg/repository/result" - "github.com/kubeshop/testkube/pkg/repository/testresult" "github.com/kubeshop/testkube/pkg/repository/testworkflow" "github.com/kubeshop/testkube/pkg/utils" "github.com/kubeshop/testkube/pkg/utils/text" @@ -29,8 +28,7 @@ var _ common.Listener = (*WebhookListener)(nil) func NewWebhookListener(name, uri, selector string, events []testkube.EventType, payloadObjectField, payloadTemplate string, headers map[string]string, disabled bool, - testExecutionResults result.Repository, - testSuiteExecutionResults testresult.Repository, + deprecatedRepositories commons.DeprecatedRepositories, testWorkflowExecutionResults testworkflow.Repository, metrics v1.Metrics, proContext *config.ProContext, @@ -47,8 +45,7 @@ func NewWebhookListener(name, uri, selector string, events []testkube.EventType, payloadTemplate: payloadTemplate, headers: headers, disabled: disabled, - testExecutionResults: testExecutionResults, - testSuiteExecutionResults: testSuiteExecutionResults, + deprecatedRepositories: deprecatedRepositories, testWorkflowExecutionResults: testWorkflowExecutionResults, metrics: metrics, proContext: proContext, @@ -67,8 +64,7 @@ type WebhookListener struct { payloadTemplate string headers map[string]string disabled bool - testExecutionResults result.Repository - testSuiteExecutionResults testresult.Repository + deprecatedRepositories commons.DeprecatedRepositories testWorkflowExecutionResults testworkflow.Repository metrics v1.Metrics proContext *config.ProContext @@ -282,8 +278,8 @@ func (l *WebhookListener) processTemplate(field, body string, event testkube.Eve func (l *WebhookListener) hasBecomeState(event testkube.Event) (bool, error) { log := l.Log.With(event.Log()...) - if event.TestExecution != nil && event.Type_ != nil { - prevStatus, err := l.testExecutionResults.GetPreviousFinishedState(context.Background(), event.TestExecution.TestName, event.TestExecution.EndTime) + if l.deprecatedRepositories != nil && event.TestExecution != nil && event.Type_ != nil { + prevStatus, err := l.deprecatedRepositories.TestResults().GetPreviousFinishedState(context.Background(), event.TestExecution.TestName, event.TestExecution.EndTime) if err != nil { return false, err } @@ -296,8 +292,8 @@ func (l *WebhookListener) hasBecomeState(event testkube.Event) (bool, error) { return event.Type_.IsBecomeExecutionStatus(prevStatus), nil } - if event.TestSuiteExecution != nil && event.TestSuiteExecution.TestSuite != nil && event.Type_ != nil { - prevStatus, err := l.testSuiteExecutionResults.GetPreviousFinishedState(context.Background(), event.TestSuiteExecution.TestSuite.Name, event.TestSuiteExecution.EndTime) + if l.deprecatedRepositories != nil && event.TestSuiteExecution != nil && event.TestSuiteExecution.TestSuite != nil && event.Type_ != nil { + prevStatus, err := l.deprecatedRepositories.TestSuiteResults().GetPreviousFinishedState(context.Background(), event.TestSuiteExecution.TestSuite.Name, event.TestSuiteExecution.EndTime) if err != nil { return false, err } diff --git a/pkg/event/kind/webhook/listener_test.go b/pkg/event/kind/webhook/listener_test.go index 07e078006f3..386424cb748 100644 --- a/pkg/event/kind/webhook/listener_test.go +++ b/pkg/event/kind/webhook/listener_test.go @@ -34,7 +34,7 @@ func TestWebhookListener_Notify(t *testing.T) { svr := httptest.NewServer(testHandler) defer svr.Close() - l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "", nil, false, nil, nil, nil, v1.NewMetrics(), nil, nil) + l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "", nil, false, nil, nil, v1.NewMetrics(), nil, nil) // when r := l.Notify(testkube.Event{ @@ -56,7 +56,7 @@ func TestWebhookListener_Notify(t *testing.T) { svr := httptest.NewServer(testHandler) defer svr.Close() - l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "", nil, false, nil, nil, nil, v1.NewMetrics(), nil, nil) + l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "", nil, false, nil, nil, v1.NewMetrics(), nil, nil) // when r := l.Notify(testkube.Event{ @@ -73,7 +73,7 @@ func TestWebhookListener_Notify(t *testing.T) { t.Parallel() // given - s := NewWebhookListener("l1", "http://baduri.badbadbad", "", testEventTypes, "", "", nil, false, nil, nil, nil, v1.NewMetrics(), nil, nil) + s := NewWebhookListener("l1", "http://baduri.badbadbad", "", testEventTypes, "", "", nil, false, nil, nil, v1.NewMetrics(), nil, nil) // when r := s.Notify(testkube.Event{ @@ -106,7 +106,7 @@ func TestWebhookListener_Notify(t *testing.T) { svr := httptest.NewServer(testHandler) defer svr.Close() - l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "field", "", nil, false, nil, nil, nil, v1.NewMetrics(), nil, nil) + l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "field", "", nil, false, nil, nil, v1.NewMetrics(), nil, nil) // when r := l.Notify(testkube.Event{ @@ -133,7 +133,7 @@ func TestWebhookListener_Notify(t *testing.T) { defer svr.Close() l := NewWebhookListener("l1", svr.URL, "", testEventTypes, "", "{\"id\": \"{{ .Id }}\"}", - map[string]string{"Content-Type": "application/json"}, false, nil, nil, nil, v1.NewMetrics(), nil, nil) + map[string]string{"Content-Type": "application/json"}, false, nil, nil, v1.NewMetrics(), nil, nil) // when r := l.Notify(testkube.Event{ @@ -150,7 +150,7 @@ func TestWebhookListener_Notify(t *testing.T) { t.Parallel() // given - s := NewWebhookListener("l1", "http://baduri.badbadbad", "", testEventTypes, "", "", nil, true, nil, nil, nil, v1.NewMetrics(), nil, nil) + s := NewWebhookListener("l1", "http://baduri.badbadbad", "", testEventTypes, "", "", nil, true, nil, nil, v1.NewMetrics(), nil, nil) // when r := s.Notify(testkube.Event{ diff --git a/pkg/event/kind/webhook/loader.go b/pkg/event/kind/webhook/loader.go index e2682af54b3..0804ad3d77d 100644 --- a/pkg/event/kind/webhook/loader.go +++ b/pkg/event/kind/webhook/loader.go @@ -6,29 +6,26 @@ import ( "go.uber.org/zap" executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" - templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" + "github.com/kubeshop/testkube/cmd/api-server/commons" v1 "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/internal/config" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/event/kind/common" "github.com/kubeshop/testkube/pkg/mapper/webhooks" - "github.com/kubeshop/testkube/pkg/repository/result" - "github.com/kubeshop/testkube/pkg/repository/testresult" "github.com/kubeshop/testkube/pkg/repository/testworkflow" ) var _ common.ListenerLoader = (*WebhooksLoader)(nil) -func NewWebhookLoader(log *zap.SugaredLogger, webhooksClient executorsclientv1.WebhooksInterface, templatesClient templatesclientv1.Interface, - testExecutionResults result.Repository, testSuiteExecutionResults testresult.Repository, testWorkflowExecutionResults testworkflow.Repository, +func NewWebhookLoader(log *zap.SugaredLogger, webhooksClient executorsclientv1.WebhooksInterface, deprecatedClients commons.DeprecatedClients, + deprecatedRepositories commons.DeprecatedRepositories, testWorkflowExecutionResults testworkflow.Repository, metrics v1.Metrics, proContext *config.ProContext, envs map[string]string, ) *WebhooksLoader { return &WebhooksLoader{ log: log, WebhooksClient: webhooksClient, - templatesClient: templatesClient, - testExecutionResults: testExecutionResults, - testSuiteExecutionResults: testSuiteExecutionResults, + deprecatedClients: deprecatedClients, + deprecatedRepositories: deprecatedRepositories, testWorkflowExecutionResults: testWorkflowExecutionResults, metrics: metrics, proContext: proContext, @@ -39,9 +36,8 @@ func NewWebhookLoader(log *zap.SugaredLogger, webhooksClient executorsclientv1.W type WebhooksLoader struct { log *zap.SugaredLogger WebhooksClient executorsclientv1.WebhooksInterface - templatesClient templatesclientv1.Interface - testExecutionResults result.Repository - testSuiteExecutionResults testresult.Repository + deprecatedClients commons.DeprecatedClients + deprecatedRepositories commons.DeprecatedRepositories testWorkflowExecutionResults testworkflow.Repository metrics v1.Metrics proContext *config.ProContext @@ -63,7 +59,11 @@ func (r WebhooksLoader) Load() (listeners common.Listeners, err error) { for _, webhook := range webhookList.Items { payloadTemplate := "" if webhook.Spec.PayloadTemplateReference != "" { - template, err := r.templatesClient.Get(webhook.Spec.PayloadTemplateReference) + if r.deprecatedClients == nil { + r.log.Errorw("webhook using deprecated PayloadTemplateReference", "name", webhook.Name, "template", webhook.Spec.PayloadTemplateReference) + continue + } + template, err := r.deprecatedClients.Templates().Get(webhook.Spec.PayloadTemplateReference) if err != nil { return listeners, err } @@ -86,7 +86,7 @@ func (r WebhooksLoader) Load() (listeners common.Listeners, err error) { NewWebhookListener( name, webhook.Spec.Uri, webhook.Spec.Selector, types, webhook.Spec.PayloadObjectField, payloadTemplate, webhook.Spec.Headers, webhook.Spec.Disabled, - r.testExecutionResults, r.testSuiteExecutionResults, r.testWorkflowExecutionResults, + r.deprecatedRepositories, r.testWorkflowExecutionResults, r.metrics, r.proContext, r.envs, ), ) diff --git a/pkg/event/kind/webhook/loader_test.go b/pkg/event/kind/webhook/loader_test.go index 5261091506e..532d27aae81 100644 --- a/pkg/event/kind/webhook/loader_test.go +++ b/pkg/event/kind/webhook/loader_test.go @@ -10,6 +10,7 @@ import ( executorsv1 "github.com/kubeshop/testkube-operator/api/executor/v1" executorsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/executors/v1" templatesclientv1 "github.com/kubeshop/testkube-operator/pkg/client/templates/v1" + "github.com/kubeshop/testkube/cmd/api-server/commons" v1 "github.com/kubeshop/testkube/internal/app/api/metrics" ) @@ -26,8 +27,10 @@ func TestWebhookLoader(t *testing.T) { {Spec: executorsv1.WebhookSpec{Uri: "http://localhost:3333", Events: []executorsv1.EventType{"start-test"}, PayloadObjectField: "text", PayloadTemplate: "{{ .Id }}", Headers: map[string]string{"Content-Type": "application/xml"}}}, }, }, nil).AnyTimes() + mockDeprecatedClients := commons.NewMockDeprecatedClients(mockCtrl) + mockDeprecatedClients.EXPECT().Templates().Return(mockTemplatesClient).AnyTimes() - webhooksLoader := NewWebhookLoader(zap.NewNop().Sugar(), mockWebhooksClient, mockTemplatesClient, nil, nil, nil, v1.NewMetrics(), nil, nil) + webhooksLoader := NewWebhookLoader(zap.NewNop().Sugar(), mockWebhooksClient, mockDeprecatedClients, nil, nil, v1.NewMetrics(), nil, nil) listeners, err := webhooksLoader.Load() assert.Equal(t, 1, len(listeners)) diff --git a/pkg/storage/minio/minio.go b/pkg/storage/minio/minio.go index 644676bf945..4a53b818cb3 100644 --- a/pkg/storage/minio/minio.go +++ b/pkg/storage/minio/minio.go @@ -100,6 +100,14 @@ func (c *Client) CreateBucket(ctx context.Context, bucket string) error { return nil } +// BucketExists checks if the bucket exists +func (c *Client) BucketExists(ctx context.Context, bucket string) (bool, error) { + if err := c.Connect(); err != nil { + return false, err + } + return c.minioClient.BucketExists(ctx, bucket) +} + // DeleteBucket deletes bucket by name func (c *Client) DeleteBucket(ctx context.Context, bucket string, force bool) error { if err := c.Connect(); err != nil { @@ -648,7 +656,7 @@ func (c *Client) PresignDownloadFileFromBucket(ctx context.Context, bucket, buck file = strings.Trim(bucketFolder, "/") + "/" + file } c.Log.Debugw("presigning get object from minio", "file", file, "bucket", bucket) - url, err := c.minioClient.PresignedPutObject(ctx, bucket, file, expires) + url, err := c.minioClient.PresignedGetObject(ctx, bucket, file, expires, nil) if err != nil { return "", err } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 40e2695a396..586fe9608e2 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -34,6 +34,7 @@ type ClientImplicitBucket interface { type ClientBucket interface { CreateBucket(ctx context.Context, bucket string) error DeleteBucket(ctx context.Context, bucket string, force bool) error + BucketExists(ctx context.Context, bucket string) (bool, error) ListBuckets(ctx context.Context) ([]string, error) DownloadFileFromBucket(ctx context.Context, bucket, bucketFolder, file string) (io.Reader, minio.ObjectInfo, error) DownloadArchiveFromBucket(ctx context.Context, bucket, bucketFolder string, masks []string) (io.Reader, error) diff --git a/pkg/storage/storage_mock.go b/pkg/storage/storage_mock.go index 03583bc5548..3d635cfae38 100644 --- a/pkg/storage/storage_mock.go +++ b/pkg/storage/storage_mock.go @@ -38,6 +38,21 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { return m.recorder } +// BucketExists mocks base method. +func (m *MockClient) BucketExists(arg0 context.Context, arg1 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BucketExists", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BucketExists indicates an expected call of BucketExists. +func (mr *MockClientMockRecorder) BucketExists(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BucketExists", reflect.TypeOf((*MockClient)(nil).BucketExists), arg0, arg1) +} + // CreateBucket mocks base method. func (m *MockClient) CreateBucket(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() diff --git a/pkg/testworkflows/testworkflowconfig/config.go b/pkg/testworkflows/testworkflowconfig/config.go index 025e5ebcdd9..8f41f3c8cfe 100644 --- a/pkg/testworkflows/testworkflowconfig/config.go +++ b/pkg/testworkflows/testworkflowconfig/config.go @@ -61,21 +61,5 @@ type WorkerConnectionConfig struct { SkipVerify bool `json:"v,omitempty"` TlsInsecure bool `json:"i,omitempty"` - LocalApiUrl string `json:"A,omitempty"` // TODO: Avoid using internal API with Control Plane - ObjectStorage ObjectStorageConfig `json:"O,omitempty"` // TODO: Avoid using Object Storage only directly -} - -// TODO: Avoid using Object Storage directly -type ObjectStorageConfig struct { - Endpoint string `json:"e,omitempty"` - AccessKeyID string `json:"a,omitempty"` - SecretAccessKey string `json:"s,omitempty"` - Region string `json:"r,omitempty"` - Token string `json:"t,omitempty"` - Bucket string `json:"b,omitempty"` - Ssl bool `json:"S,omitempty"` - SkipVerify bool `json:"v,omitempty"` - CertFile string `json:"c,omitempty"` - KeyFile string `json:"k,omitempty"` - CAFile string `json:"C,omitempty"` + LocalApiUrl string `json:"A,omitempty"` // TODO: Avoid using internal API with Control Plane } diff --git a/pkg/testworkflows/testworkflowconfig/expressions.go b/pkg/testworkflows/testworkflowconfig/expressions.go index 6b25773cdd4..3cd5cd330b8 100644 --- a/pkg/testworkflows/testworkflowconfig/expressions.go +++ b/pkg/testworkflows/testworkflowconfig/expressions.go @@ -2,23 +2,6 @@ package testworkflowconfig import "github.com/kubeshop/testkube/pkg/expressions" -func CreateStorageMachine(cfg *ObjectStorageConfig) expressions.Machine { - return expressions.NewMachine(). - RegisterMap("internal", map[string]interface{}{ - "storage.url": cfg.Endpoint, - "storage.accessKey": cfg.AccessKeyID, - "storage.secretKey": cfg.SecretAccessKey, - "storage.region": cfg.Region, - "storage.bucket": cfg.Bucket, - "storage.token": cfg.Token, - "storage.ssl": cfg.Ssl, - "storage.skipVerify": cfg.SkipVerify, - "storage.certFile": cfg.CertFile, - "storage.keyFile": cfg.KeyFile, - "storage.caFile": cfg.CAFile, - }) -} - func CreateExecutionMachine(cfg *ExecutionConfig) expressions.Machine { return expressions.NewMachine(). Register("execution", map[string]interface{}{ @@ -92,5 +75,5 @@ func CreateWorkerMachine(cfg *WorkerConfig) expressions.Machine { "api.url": cfg.Connection.LocalApiUrl, // TODO: Delete }) - return expressions.CombinedMachines(machine, CreateStorageMachine(&cfg.Connection.ObjectStorage)) + return expressions.CombinedMachines(machine) } diff --git a/pkg/triggers/executor.go b/pkg/triggers/executor.go index d9b3e0be970..ebb2ed222a9 100644 --- a/pkg/triggers/executor.go +++ b/pkg/triggers/executor.go @@ -85,7 +85,7 @@ func (s *Service) execute(ctx context.Context, e *watcherEvent, t *testtriggersv "trigger service: executor component: scheduling test executions for trigger %s/%s", t.Namespace, t.Name, ) - go wp.SendRequests(s.scheduler.PrepareTestRequests(tests, request)) + go wp.SendRequests(s.deprecatedSystem.Scheduler.PrepareTestRequests(tests, request)) go wp.Run(ctx) }() @@ -120,7 +120,7 @@ func (s *Service) execute(ctx context.Context, e *watcherEvent, t *testtriggersv "trigger service: executor component: scheduling testsuite executions for trigger %s/%s", t.Namespace, t.Name, ) - go wp.SendRequests(s.scheduler.PrepareTestSuiteRequests(testSuites, request)) + go wp.SendRequests(s.deprecatedSystem.Scheduler.PrepareTestSuiteRequests(testSuites, request)) go wp.Run(ctx) }() @@ -196,7 +196,7 @@ func (s *Service) getTests(t *testtriggersv1.TestTrigger) ([]testsv3.Test, error var tests []testsv3.Test if t.Spec.TestSelector.Name != "" { s.logger.Debugf("trigger service: executor component: fetching testsv3.Test with name %s", t.Spec.TestSelector.Name) - test, err := s.deprecatedClients.Tests().Get(t.Spec.TestSelector.Name) + test, err := s.deprecatedSystem.Clients.Tests().Get(t.Spec.TestSelector.Name) if err != nil { return nil, err } @@ -205,7 +205,7 @@ func (s *Service) getTests(t *testtriggersv1.TestTrigger) ([]testsv3.Test, error if t.Spec.TestSelector.NameRegex != "" { s.logger.Debugf("trigger service: executor component: fetching testsv3.Test with name regex %s", t.Spec.TestSelector.NameRegex) - testList, err := s.deprecatedClients.Tests().List("") + testList, err := s.deprecatedSystem.Clients.Tests().List("") if err != nil { return nil, err } @@ -229,7 +229,7 @@ func (s *Service) getTests(t *testtriggersv1.TestTrigger) ([]testsv3.Test, error } stringifiedSelector := selector.String() s.logger.Debugf("trigger service: executor component: fetching testsv3.Test with labels %s", stringifiedSelector) - testList, err := s.deprecatedClients.Tests().List(stringifiedSelector) + testList, err := s.deprecatedSystem.Clients.Tests().List(stringifiedSelector) if err != nil { return nil, err } @@ -242,7 +242,7 @@ func (s *Service) getTestSuites(t *testtriggersv1.TestTrigger) ([]testsuitesv3.T var testSuites []testsuitesv3.TestSuite if t.Spec.TestSelector.Name != "" { s.logger.Debugf("trigger service: executor component: fetching testsuitesv3.TestSuite with name %s", t.Spec.TestSelector.Name) - testSuite, err := s.deprecatedClients.TestSuites().Get(t.Spec.TestSelector.Name) + testSuite, err := s.deprecatedSystem.Clients.TestSuites().Get(t.Spec.TestSelector.Name) if err != nil { return nil, err } @@ -251,7 +251,7 @@ func (s *Service) getTestSuites(t *testtriggersv1.TestTrigger) ([]testsuitesv3.T if t.Spec.TestSelector.NameRegex != "" { s.logger.Debugf("trigger service: executor component: fetching testsuitesv3.TestSuite with name regex %s", t.Spec.TestSelector.NameRegex) - testSuitesList, err := s.deprecatedClients.TestSuites().List("") + testSuitesList, err := s.deprecatedSystem.Clients.TestSuites().List("") if err != nil { return nil, err } @@ -275,7 +275,7 @@ func (s *Service) getTestSuites(t *testtriggersv1.TestTrigger) ([]testsuitesv3.T } stringifiedSelector := selector.String() s.logger.Debugf("trigger service: executor component: fetching testsuitesv3.TestSuite with label %s", stringifiedSelector) - testSuitesList, err := s.deprecatedClients.TestSuites().List(stringifiedSelector) + testSuitesList, err := s.deprecatedSystem.Clients.TestSuites().List(stringifiedSelector) if err != nil { return nil, err } diff --git a/pkg/triggers/executor_test.go b/pkg/triggers/executor_test.go index ea4d2b3d84b..9c1e7fda26c 100644 --- a/pkg/triggers/executor_test.go +++ b/pkg/triggers/executor_test.go @@ -19,6 +19,7 @@ import ( testsuitesv3 "github.com/kubeshop/testkube-operator/pkg/client/testsuites/v3" testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/cmd/api-server/services" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/configmap" @@ -143,11 +144,13 @@ func TestExecute(t *testing.T) { checktcl.SubscriptionChecker{}, ) s := &Service{ - triggerStatus: make(map[statusKey]*triggerStatus), - scheduler: sched, - deprecatedRepositories: mockDeprecatedRepositories, - deprecatedClients: mockDeprecatedClients, - logger: log.DefaultLogger, + triggerStatus: make(map[statusKey]*triggerStatus), + deprecatedSystem: &services.DeprecatedSystem{ + Scheduler: sched, + Repositories: mockDeprecatedRepositories, + Clients: mockDeprecatedClients, + }, + logger: log.DefaultLogger, } status := testtriggersv1.TRUE_TestTriggerConditionStatuses diff --git a/pkg/triggers/matcher.go b/pkg/triggers/matcher.go index 3fb8125a04c..6bb7abc2322 100644 --- a/pkg/triggers/matcher.go +++ b/pkg/triggers/matcher.go @@ -34,6 +34,9 @@ func (s *Service) match(ctx context.Context, e *watcherEvent) error { if t.Spec.Disabled { continue } + if s.deprecatedSystem == nil && (t.Spec.Execution == ExecutionTest || t.Spec.Execution == ExecutionTestSuite) { + continue + } if t.Spec.Resource != testtriggersv1.TestTriggerResource(e.resource) { continue diff --git a/pkg/triggers/matcher_test.go b/pkg/triggers/matcher_test.go index bb9d54f651e..3c61d10eedd 100644 --- a/pkg/triggers/matcher_test.go +++ b/pkg/triggers/matcher_test.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testtriggersv1 "github.com/kubeshop/testkube-operator/api/testtriggers/v1" + "github.com/kubeshop/testkube/cmd/api-server/services" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/log" ) @@ -82,6 +83,7 @@ func TestService_matchConditionsRetry(t *testing.T) { statusKey1 := newStatusKey(testTrigger1.Namespace, testTrigger1.Name) triggerStatus1 := &triggerStatus{testTrigger: testTrigger1} s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, defaultConditionsCheckBackoff: defaultConditionsCheckBackoff, defaultConditionsCheckTimeout: defaultConditionsCheckTimeout, triggerExecutor: func(ctx context.Context, e *watcherEvent, trigger *testtriggersv1.TestTrigger) error { @@ -160,6 +162,7 @@ func TestService_matchConditionsTimeout(t *testing.T) { statusKey1 := newStatusKey(testTrigger1.Namespace, testTrigger1.Name) triggerStatus1 := &triggerStatus{testTrigger: testTrigger1} s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, defaultConditionsCheckBackoff: defaultConditionsCheckBackoff, defaultConditionsCheckTimeout: defaultConditionsCheckTimeout, triggerExecutor: func(ctx context.Context, e *watcherEvent, trigger *testtriggersv1.TestTrigger) error { @@ -234,6 +237,7 @@ func TestService_matchProbesMultiple(t *testing.T) { statusKey1 := newStatusKey(testTrigger1.Namespace, testTrigger1.Name) triggerStatus1 := &triggerStatus{testTrigger: testTrigger1} s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, defaultProbesCheckBackoff: defaultProbesCheckBackoff, defaultProbesCheckTimeout: defaultProbesCheckTimeout, triggerExecutor: func(ctx context.Context, e *watcherEvent, trigger *testtriggersv1.TestTrigger) error { @@ -302,6 +306,7 @@ func TestService_matchProbesTimeout(t *testing.T) { statusKey1 := newStatusKey(testTrigger1.Namespace, testTrigger1.Name) triggerStatus1 := &triggerStatus{testTrigger: testTrigger1} s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, defaultProbesCheckBackoff: defaultProbesCheckBackoff, defaultProbesCheckTimeout: defaultProbesCheckTimeout, triggerExecutor: func(ctx context.Context, e *watcherEvent, trigger *testtriggersv1.TestTrigger) error { @@ -407,6 +412,7 @@ func TestService_match(t *testing.T) { statusKey1 := newStatusKey(testTrigger1.Namespace, testTrigger1.Name) triggerStatus1 := &triggerStatus{testTrigger: testTrigger1} s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, defaultConditionsCheckBackoff: defaultConditionsCheckBackoff, defaultConditionsCheckTimeout: defaultConditionsCheckTimeout, defaultProbesCheckBackoff: defaultProbesCheckBackoff, @@ -459,6 +465,7 @@ func TestService_matchRegex(t *testing.T) { statusKey1 := newStatusKey(testTrigger1.Namespace, testTrigger1.Name) triggerStatus1 := &triggerStatus{testTrigger: testTrigger1} s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, defaultConditionsCheckBackoff: defaultConditionsCheckBackoff, defaultConditionsCheckTimeout: defaultConditionsCheckTimeout, defaultProbesCheckBackoff: defaultProbesCheckBackoff, @@ -511,10 +518,11 @@ func TestService_noMatch(t *testing.T) { return nil } s := &Service{ - triggerExecutor: testExecutorF, - triggerStatus: map[statusKey]*triggerStatus{statusKey1: triggerStatus1}, - logger: log.DefaultLogger, - metrics: metrics.NewMetrics(), + deprecatedSystem: &services.DeprecatedSystem{}, + triggerExecutor: testExecutorF, + triggerStatus: map[statusKey]*triggerStatus{statusKey1: triggerStatus1}, + logger: log.DefaultLogger, + metrics: metrics.NewMetrics(), } err := s.match(context.Background(), e) diff --git a/pkg/triggers/scraper.go b/pkg/triggers/scraper.go index 26be0cd904e..43c6f5d4e83 100644 --- a/pkg/triggers/scraper.go +++ b/pkg/triggers/scraper.go @@ -24,8 +24,10 @@ func (s *Service) runExecutionScraper(ctx context.Context) { s.logger.Debugf("trigger service: execution scraper component: starting new ticker iteration") for triggerName, status := range s.triggerStatus { if status.hasActiveTests() { - s.checkForRunningTestExecutions(ctx, status) - s.checkForRunningTestSuiteExecutions(ctx, status) + if s.deprecatedSystem != nil { + s.checkForRunningTestExecutions(ctx, status) + s.checkForRunningTestSuiteExecutions(ctx, status) + } s.checkForRunningTestWorkflowExecutions(ctx, status) if !status.hasActiveTests() { s.logger.Debugf("marking status as finished for testtrigger %s", triggerName) @@ -41,7 +43,7 @@ func (s *Service) checkForRunningTestExecutions(ctx context.Context, status *tri testExecutionIDs := status.getExecutionIDs() for _, id := range testExecutionIDs { - execution, err := s.deprecatedRepositories.TestResults().Get(ctx, id) + execution, err := s.deprecatedSystem.Repositories.TestResults().Get(ctx, id) if err == mongo.ErrNoDocuments { s.logger.Warnf("trigger service: execution scraper component: no test execution found for id %s", id) status.removeExecutionID(id) @@ -61,7 +63,7 @@ func (s *Service) checkForRunningTestSuiteExecutions(ctx context.Context, status testSuiteExecutionIDs := status.getTestSuiteExecutionIDs() for _, id := range testSuiteExecutionIDs { - execution, err := s.deprecatedRepositories.TestSuiteResults().Get(ctx, id) + execution, err := s.deprecatedSystem.Repositories.TestSuiteResults().Get(ctx, id) if err == mongo.ErrNoDocuments { s.logger.Warnf("trigger service: execution scraper component: no testsuite execution found for id %s", id) status.removeTestSuiteExecutionID(id) @@ -100,8 +102,10 @@ func (s *Service) checkForRunningTestWorkflowExecutions(ctx context.Context, sta func (s *Service) abortExecutions(ctx context.Context, testTriggerName string, status *triggerStatus) { s.logger.Debugf("trigger service: abort executions") - s.abortRunningTestExecutions(ctx, status) - s.abortRunningTestSuiteExecutions(ctx, status) + if s.deprecatedSystem != nil { + s.abortRunningTestExecutions(ctx, status) + s.abortRunningTestSuiteExecutions(ctx, status) + } s.abortRunningTestWorkflowExecutions(ctx, status) if !status.hasActiveTests() { s.logger.Debugf("marking status as finished for testtrigger %s", testTriggerName) @@ -113,7 +117,7 @@ func (s *Service) abortRunningTestExecutions(ctx context.Context, status *trigge testExecutionIDs := status.getExecutionIDs() for _, id := range testExecutionIDs { - execution, err := s.deprecatedRepositories.TestResults().Get(ctx, id) + execution, err := s.deprecatedSystem.Repositories.TestResults().Get(ctx, id) if err == mongo.ErrNoDocuments { s.logger.Warnf("trigger service: execution scraper component: no test execution found for id %s", id) status.removeExecutionID(id) @@ -123,7 +127,7 @@ func (s *Service) abortRunningTestExecutions(ctx context.Context, status *trigge continue } if execution.IsRunning() || execution.IsQueued() { - res, err := s.testExecutor.Abort(ctx, &execution) + res, err := s.deprecatedSystem.JobExecutor.Abort(ctx, &execution) if err != nil { s.logger.Errorf("trigger service: execution scraper component: error aborting test execution: %v", err) continue @@ -140,7 +144,7 @@ func (s *Service) abortRunningTestSuiteExecutions(ctx context.Context, status *t testSuiteExecutionIDs := status.getTestSuiteExecutionIDs() for _, id := range testSuiteExecutionIDs { - execution, err := s.deprecatedRepositories.TestSuiteResults().Get(ctx, id) + execution, err := s.deprecatedSystem.Repositories.TestSuiteResults().Get(ctx, id) if err == mongo.ErrNoDocuments { s.logger.Warnf("trigger service: execution scraper component: no testsuite execution found for id %s", id) status.removeTestSuiteExecutionID(id) diff --git a/pkg/triggers/scraper_test.go b/pkg/triggers/scraper_test.go index 496fe36365b..c1d50cd581a 100644 --- a/pkg/triggers/scraper_test.go +++ b/pkg/triggers/scraper_test.go @@ -10,6 +10,7 @@ import ( "go.mongodb.org/mongo-driver/mongo" "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/cmd/api-server/services" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/log" "github.com/kubeshop/testkube/pkg/repository/result" @@ -57,8 +58,10 @@ func TestService_runExecutionScraper(t *testing.T) { statusKey3: triggerStatus3, } s := &Service{ - triggerStatus: triggerStatusMap, - deprecatedRepositories: mockDeprecatedRepositories, + triggerStatus: triggerStatusMap, + deprecatedSystem: &services.DeprecatedSystem{ + Repositories: mockDeprecatedRepositories, + }, testWorkflowResultsRepository: mockTestWorkflowResultsRepository, scraperInterval: 100 * time.Millisecond, logger: log.DefaultLogger, @@ -104,8 +107,10 @@ func TestService_runExecutionScraper(t *testing.T) { statusKey2: triggerStatus2, } s := &Service{ - triggerStatus: triggerStatusMap, - deprecatedRepositories: mockDeprecatedRepositories, + triggerStatus: triggerStatusMap, + deprecatedSystem: &services.DeprecatedSystem{ + Repositories: mockDeprecatedRepositories, + }, testWorkflowResultsRepository: mockTestWorkflowResultsRepository, scraperInterval: 100 * time.Millisecond, logger: log.DefaultLogger, diff --git a/pkg/triggers/service.go b/pkg/triggers/service.go index cff8fe41d6a..b049ff8de37 100644 --- a/pkg/triggers/service.go +++ b/pkg/triggers/service.go @@ -16,16 +16,14 @@ import ( testtriggersv1 "github.com/kubeshop/testkube-operator/api/testtriggers/v1" testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" testkubeclientsetv1 "github.com/kubeshop/testkube-operator/pkg/clientset/versioned" - "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/cmd/api-server/services" "github.com/kubeshop/testkube/internal/app/api/metrics" intconfig "github.com/kubeshop/testkube/internal/config" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/event/bus" - "github.com/kubeshop/testkube/pkg/executor/client" "github.com/kubeshop/testkube/pkg/http" "github.com/kubeshop/testkube/pkg/repository/config" "github.com/kubeshop/testkube/pkg/repository/testworkflow" - "github.com/kubeshop/testkube/pkg/scheduler" "github.com/kubeshop/testkube/pkg/telemetry" "github.com/kubeshop/testkube/pkg/testworkflows/executionworker/executionworkertypes" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowexecutor" @@ -60,16 +58,12 @@ type Service struct { defaultProbesCheckBackoff time.Duration watchFromDate time.Time triggerStatus map[statusKey]*triggerStatus - scheduler *scheduler.Scheduler clientset kubernetes.Interface testKubeClientset testkubeclientsetv1.Interface - deprecatedRepositories commons.DeprecatedRepositories - deprecatedClients commons.DeprecatedClients testWorkflowsClient testworkflowsclientv1.Interface logger *zap.SugaredLogger configMap config.Repository httpClient http.HttpClient - testExecutor client.Executor eventsBus bus.Bus metrics metrics.Metrics executionWorkerClient executionworkertypes.Worker @@ -78,22 +72,20 @@ type Service struct { testkubeNamespace string watcherNamespaces []string disableSecretCreation bool + deprecatedSystem *services.DeprecatedSystem proContext *intconfig.ProContext } type Option func(*Service) func NewService( - deprecatedRepositories commons.DeprecatedRepositories, - deprecatedClients commons.DeprecatedClients, - scheduler *scheduler.Scheduler, + deprecatedSystem *services.DeprecatedSystem, clientset kubernetes.Interface, testKubeClientset testkubeclientsetv1.Interface, testWorkflowsClient testworkflowsclientv1.Interface, leaseBackend LeaseBackend, logger *zap.SugaredLogger, configMap config.Repository, - testExecutor client.Executor, eventsBus bus.Bus, metrics metrics.Metrics, executionWorkerClient executionworkertypes.Worker, @@ -112,16 +104,12 @@ func NewService( defaultConditionsCheckBackoff: defaultConditionsCheckBackoff, defaultProbesCheckTimeout: defaultProbesCheckTimeout, defaultProbesCheckBackoff: defaultProbesCheckBackoff, - scheduler: scheduler, clientset: clientset, testKubeClientset: testKubeClientset, - deprecatedRepositories: deprecatedRepositories, - deprecatedClients: deprecatedClients, testWorkflowsClient: testWorkflowsClient, leaseBackend: leaseBackend, logger: logger, configMap: configMap, - testExecutor: testExecutor, eventsBus: eventsBus, metrics: metrics, executionWorkerClient: executionWorkerClient, @@ -130,6 +118,7 @@ func NewService( httpClient: http.NewClient(), watchFromDate: time.Now(), triggerStatus: make(map[statusKey]*triggerStatus), + deprecatedSystem: deprecatedSystem, } if s.triggerExecutor == nil { s.triggerExecutor = s.execute @@ -302,14 +291,14 @@ func (s *Service) addTest(test *testsv3.Test) { } test.Labels[testkube.TestLabelTestType] = utils.SanitizeName(test.Spec.Type_) - executorCR, err := s.deprecatedClients.Executors().GetByType(test.Spec.Type_) + executorCR, err := s.deprecatedSystem.Clients.Executors().GetByType(test.Spec.Type_) if err == nil { test.Labels[testkube.TestLabelExecutor] = executorCR.Name } else { s.logger.Debugw("can't get executor spec", "error", err) } - if _, err = s.deprecatedClients.Tests().Update(test, s.disableSecretCreation); err != nil { + if _, err = s.deprecatedSystem.Clients.Tests().Update(test, s.disableSecretCreation); err != nil { s.logger.Debugw("can't update test spec", "error", err) } } @@ -326,7 +315,7 @@ func (s *Service) updateTest(test *testsv3.Test) { changed = true } - executorCR, err := s.deprecatedClients.Executors().GetByType(test.Spec.Type_) + executorCR, err := s.deprecatedSystem.Clients.Executors().GetByType(test.Spec.Type_) if err == nil { if test.Labels[testkube.TestLabelExecutor] != executorCR.Name { test.Labels[testkube.TestLabelExecutor] = executorCR.Name @@ -337,7 +326,7 @@ func (s *Service) updateTest(test *testsv3.Test) { } if changed { - if _, err = s.deprecatedClients.Tests().Update(test, s.disableSecretCreation); err != nil { + if _, err = s.deprecatedSystem.Clients.Tests().Update(test, s.disableSecretCreation); err != nil { s.logger.Debugw("can't update test spec", "error", err) } } diff --git a/pkg/triggers/service_test.go b/pkg/triggers/service_test.go index 44ef14ab67d..d3037e07a67 100644 --- a/pkg/triggers/service_test.go +++ b/pkg/triggers/service_test.go @@ -22,6 +22,7 @@ import ( testworkflowsclientv1 "github.com/kubeshop/testkube-operator/pkg/client/testworkflows/v1" faketestkube "github.com/kubeshop/testkube-operator/pkg/clientset/versioned/fake" "github.com/kubeshop/testkube/cmd/api-server/commons" + "github.com/kubeshop/testkube/cmd/api-server/services" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/configmap" @@ -166,21 +167,25 @@ func TestService_Run(t *testing.T) { testIdentifier := "test-host-1" mockLeaseBackend.EXPECT().TryAcquire(gomock.Any(), testIdentifier, testClusterID).Return(true, nil).AnyTimes() + mockDeprecatedSystem := services.DeprecatedSystem{ + Clients: mockDeprecatedClients, + Repositories: mockDeprecatedRepositories, + JobExecutor: mockExecutor, + Scheduler: sched, + } + fakeTestkubeClientset := faketestkube.NewSimpleClientset() fakeClientset := fake.NewSimpleClientset() eventBus := bus.NewEventBusMock() metrics := metrics.NewMetrics() s := NewService( - mockDeprecatedRepositories, - mockDeprecatedClients, - sched, + &mockDeprecatedSystem, fakeClientset, fakeTestkubeClientset, mockTestWorkflowsClient, mockLeaseBackend, testLogger, configMapConfig, - mockExecutor, eventBus, metrics, mockExecutionWorkerClient, diff --git a/pkg/triggers/watcher.go b/pkg/triggers/watcher.go index b3420388759..7e99d1390d1 100644 --- a/pkg/triggers/watcher.go +++ b/pkg/triggers/watcher.go @@ -48,11 +48,16 @@ type k8sInformers struct { configMapInformers []coreinformerv1.ConfigMapInformer testTriggerInformer testkubeinformerv1.TestTriggerInformer - testSuiteInformer testkubeinformerv3.TestSuiteInformer - testInformer testkubeinformerv3.TestInformer - executorInformer testkubeexecutorinformerv1.ExecutorInformer webhookInformer testkubeexecutorinformerv1.WebhookInformer - testSourceInformer testkubeinformerv1.TestSourceInformer + + deprecated deprecatedK8sInformers +} + +type deprecatedK8sInformers struct { + testSuiteInformer testkubeinformerv3.TestSuiteInformer + testInformer testkubeinformerv3.TestInformer + executorInformer testkubeexecutorinformerv1.ExecutorInformer + testSourceInformer testkubeinformerv1.TestSourceInformer } func newK8sInformers(clientset kubernetes.Interface, testKubeClientset versioned.Interface, @@ -77,11 +82,12 @@ func newK8sInformers(clientset kubernetes.Interface, testKubeClientset versioned testkubeInformerFactory := externalversions.NewSharedInformerFactoryWithOptions( testKubeClientset, 0, externalversions.WithNamespace(testkubeNamespace)) k8sInformers.testTriggerInformer = testkubeInformerFactory.Tests().V1().TestTriggers() - k8sInformers.testSuiteInformer = testkubeInformerFactory.Tests().V3().TestSuites() - k8sInformers.testInformer = testkubeInformerFactory.Tests().V3().Tests() - k8sInformers.executorInformer = testkubeInformerFactory.Executor().V1().Executor() k8sInformers.webhookInformer = testkubeInformerFactory.Executor().V1().Webhook() - k8sInformers.testSourceInformer = testkubeInformerFactory.Tests().V1().TestSource() + + k8sInformers.deprecated.testSuiteInformer = testkubeInformerFactory.Tests().V3().TestSuites() + k8sInformers.deprecated.testInformer = testkubeInformerFactory.Tests().V3().Tests() + k8sInformers.deprecated.executorInformer = testkubeInformerFactory.Executor().V1().Executor() + k8sInformers.deprecated.testSourceInformer = testkubeInformerFactory.Tests().V1().TestSource() return &k8sInformers } @@ -127,6 +133,9 @@ func (s *Service) runInformers(ctx context.Context, stop <-chan struct{}) { for i := range s.informers.podInformers { s.informers.podInformers[i].Informer().AddEventHandler(s.podEventHandler(ctx)) + if s.deprecatedSystem != nil { + s.informers.podInformers[i].Informer().AddEventHandler(s.deprecatedPodEventHandler(ctx)) + } } for i := range s.informers.deploymentInformers { @@ -158,11 +167,7 @@ func (s *Service) runInformers(ctx context.Context, stop <-chan struct{}) { } s.informers.testTriggerInformer.Informer().AddEventHandler(s.testTriggerEventHandler()) - s.informers.testSuiteInformer.Informer().AddEventHandler(s.testSuiteEventHandler()) - s.informers.testInformer.Informer().AddEventHandler(s.testEventHandler()) - s.informers.executorInformer.Informer().AddEventHandler(s.executorEventHandler()) s.informers.webhookInformer.Informer().AddEventHandler(s.webhookEventHandler()) - s.informers.testSourceInformer.Informer().AddEventHandler(s.testSourceEventHandler()) s.logger.Debugf("trigger service: starting pod informers") for i := range s.informers.podInformers { @@ -206,16 +211,24 @@ func (s *Service) runInformers(ctx context.Context, stop <-chan struct{}) { s.logger.Debugf("trigger service: starting test trigger informer") go s.informers.testTriggerInformer.Informer().Run(stop) - s.logger.Debugf("trigger service: starting test suite informer") - go s.informers.testSuiteInformer.Informer().Run(stop) - s.logger.Debugf("trigger service: starting test informer") - go s.informers.testInformer.Informer().Run(stop) - s.logger.Debugf("trigger service: starting executor informer") - go s.informers.executorInformer.Informer().Run(stop) s.logger.Debugf("trigger service: starting webhook informer") go s.informers.webhookInformer.Informer().Run(stop) - s.logger.Debugf("trigger service: starting test source informer") - go s.informers.testSourceInformer.Informer().Run(stop) + + if s.deprecatedSystem != nil { + s.informers.deprecated.testSuiteInformer.Informer().AddEventHandler(s.testSuiteEventHandler()) + s.informers.deprecated.testInformer.Informer().AddEventHandler(s.testEventHandler()) + s.informers.deprecated.executorInformer.Informer().AddEventHandler(s.executorEventHandler()) + s.informers.deprecated.testSourceInformer.Informer().AddEventHandler(s.testSourceEventHandler()) + + s.logger.Debugf("trigger service: starting test suite informer") + go s.informers.deprecated.testSuiteInformer.Informer().Run(stop) + s.logger.Debugf("trigger service: starting test informer") + go s.informers.deprecated.testInformer.Informer().Run(stop) + s.logger.Debugf("trigger service: starting executor informer") + go s.informers.deprecated.executorInformer.Informer().Run(stop) + s.logger.Debugf("trigger service: starting test source informer") + go s.informers.deprecated.testSourceInformer.Informer().Run(stop) + } } func (s *Service) podEventHandler(ctx context.Context) cache.ResourceEventHandlerFuncs { @@ -224,7 +237,7 @@ func (s *Service) podEventHandler(ctx context.Context) cache.ResourceEventHandle return getPodConditions(ctx, s.clientset, object) } } - getAddrress := func(object metav1.Object) func(c context.Context, delay time.Duration) (string, error) { + getAddress := func(object metav1.Object) func(c context.Context, delay time.Duration) (string, error) { return func(c context.Context, delay time.Duration) (string, error) { return getPodAdress(c, s.clientset, object, delay) } @@ -245,12 +258,29 @@ func (s *Service) podEventHandler(ctx context.Context) cache.ResourceEventHandle } s.logger.Debugf("trigger service: watcher component: emiting event: pod %s/%s created", pod.Namespace, pod.Name) event := newWatcherEvent(testtrigger.EventCreated, pod, testtrigger.ResourcePod, - withConditionsGetter(getConditions(pod)), withAddressGetter(getAddrress(pod))) + withConditionsGetter(getConditions(pod)), withAddressGetter(getAddress(pod))) if err := s.match(ctx, event); err != nil { s.logger.Errorf("event matcher returned an error while matching create pod event: %v", err) } - }, + DeleteFunc: func(obj interface{}) { + pod, ok := obj.(*corev1.Pod) + if !ok { + s.logger.Errorf("failed to process delete pod event due to it being an unexpected type, received type %+v", obj) + return + } + s.logger.Debugf("trigger service: watcher component: emiting event: pod %s/%s deleted", pod.Namespace, pod.Name) + event := newWatcherEvent(testtrigger.EventDeleted, pod, testtrigger.ResourcePod, + withConditionsGetter(getConditions(pod)), withAddressGetter(getAddress(pod))) + if err := s.match(ctx, event); err != nil { + s.logger.Errorf("event matcher returned an error while matching delete pod event: %v", err) + } + }, + } +} + +func (s *Service) deprecatedPodEventHandler(ctx context.Context) cache.ResourceEventHandlerFuncs { + return cache.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj, newObj any) { oldPod, ok := oldObj.(*corev1.Pod) if !ok { @@ -282,35 +312,28 @@ func (s *Service) podEventHandler(ctx context.Context) cache.ResourceEventHandle !(strings.HasSuffix(oldPod.Name, cexecutor.ScraperPodSuffix) || strings.HasSuffix(newPod.Name, cexecutor.ScraperPodSuffix)) && oldPod.Labels["job-name"] == newPod.Labels["job-name"] { s.metrics.IncTestTriggerEventCount("", string(testtrigger.ResourcePod), string(testtrigger.CauseEventUpdated), nil) - s.checkExecutionPodStatus(ctx, oldPod.Labels["job-name"], []*corev1.Pod{oldPod, newPod}) + s.deprecatedCheckExecutionPodStatus(ctx, oldPod.Labels["job-name"], []*corev1.Pod{oldPod, newPod}) } }, DeleteFunc: func(obj interface{}) { pod, ok := obj.(*corev1.Pod) if !ok { - s.logger.Errorf("failed to process delete pod event due to it being an unexpected type, received type %+v", obj) return } - s.logger.Debugf("trigger service: watcher component: emiting event: pod %s/%s deleted", pod.Namespace, pod.Name) if pod.Namespace == s.testkubeNamespace && pod.Labels["job-name"] != "" && !strings.HasSuffix(pod.Name, cexecutor.ScraperPodSuffix) && pod.Labels[testkube.TestLabelTestName] != "" { - s.checkExecutionPodStatus(ctx, pod.Labels["job-name"], []*corev1.Pod{pod}) - } - event := newWatcherEvent(testtrigger.EventDeleted, pod, testtrigger.ResourcePod, - withConditionsGetter(getConditions(pod)), withAddressGetter(getAddrress(pod))) - if err := s.match(ctx, event); err != nil { - s.logger.Errorf("event matcher returned an error while matching delete pod event: %v", err) + s.deprecatedCheckExecutionPodStatus(ctx, pod.Labels["job-name"], []*corev1.Pod{pod}) } }, } } -func (s *Service) checkExecutionPodStatus(ctx context.Context, executionID string, pods []*corev1.Pod) error { +func (s *Service) deprecatedCheckExecutionPodStatus(ctx context.Context, executionID string, pods []*corev1.Pod) error { if len(pods) > 0 && pods[0].Labels[constants.ResourceIdLabelName] != "" { return nil } - execution, err := s.deprecatedRepositories.TestResults().Get(ctx, executionID) + execution, err := s.deprecatedSystem.Repositories.TestResults().Get(ctx, executionID) if err != nil { s.logger.Errorf("get execution returned an error %v while looking for execution id: %s", err, executionID) return err @@ -333,7 +356,7 @@ func (s *Service) checkExecutionPodStatus(ctx context.Context, executionID strin } execution.ExecutionResult.ErrorMessage += errorMessage - test, err := s.deprecatedClients.Tests().Get(execution.TestName) + test, err := s.deprecatedSystem.Clients.Tests().Get(execution.TestName) if err != nil { s.logger.Errorf("get test returned an error %v while looking for test name: %s", err, execution.TestName) return err @@ -345,7 +368,7 @@ func (s *Service) checkExecutionPodStatus(ctx context.Context, executionID strin execution.ExecutionResult.ErrorMessage = "" } - err = s.deprecatedRepositories.TestResults().UpdateResult(ctx, executionID, execution) + err = s.deprecatedSystem.Repositories.TestResults().UpdateResult(ctx, executionID, execution) if err != nil { s.logger.Errorf("update execution result returned an error %v while storing for execution id: %s", err, executionID) return err diff --git a/pkg/triggers/watcher_test.go b/pkg/triggers/watcher_test.go index a9e3915332f..7c982973a6f 100644 --- a/pkg/triggers/watcher_test.go +++ b/pkg/triggers/watcher_test.go @@ -12,6 +12,7 @@ import ( testtriggersv1 "github.com/kubeshop/testkube-operator/api/testtriggers/v1" faketestkube "github.com/kubeshop/testkube-operator/pkg/clientset/versioned/fake" + "github.com/kubeshop/testkube/cmd/api-server/services" "github.com/kubeshop/testkube/internal/app/api/metrics" "github.com/kubeshop/testkube/pkg/event/bus" "github.com/kubeshop/testkube/pkg/log" @@ -30,6 +31,7 @@ func TestService_runWatcher_lease(t *testing.T) { defer cancel() s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, triggerStatus: make(map[statusKey]*triggerStatus), clientset: clientset, testKubeClientset: testKubeClientset, @@ -87,6 +89,7 @@ func TestService_runWatcher_lease(t *testing.T) { return nil } s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, triggerExecutor: testExecutorF, identifier: "testkube-api", clusterID: "testkube", @@ -152,6 +155,7 @@ func TestService_runWatcher_noLease(t *testing.T) { defer cancel() s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, triggerStatus: make(map[statusKey]*triggerStatus), identifier: "testkube-api", clusterID: "testkube", @@ -194,6 +198,7 @@ func TestService_runWatcher_noLease(t *testing.T) { defer cancel() s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, triggerStatus: make(map[statusKey]*triggerStatus), identifier: "testkube-api", clusterID: "testkube", @@ -238,6 +243,7 @@ func TestService_runWatcher_noLease(t *testing.T) { defer cancel() s := &Service{ + deprecatedSystem: &services.DeprecatedSystem{}, triggerStatus: make(map[statusKey]*triggerStatus), identifier: "testkube-api", clusterID: "testkube", diff --git a/pkg/utils/consts.go b/pkg/utils/consts.go index 043bb8500c0..890c7cfac5b 100644 --- a/pkg/utils/consts.go +++ b/pkg/utils/consts.go @@ -1,6 +1,9 @@ package utils +import "time" + const ( // DefaultDockerRegistry is the default registry used when no registry is specified in the image name. DefaultDockerRegistry = "https://index.docker.io/v1/" + DefaultRetryDelay = time.Second * 3 ) diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 00000000000..0918d56d41c --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,58 @@ +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: testkube-agent +build: + local: + concurrency: 1 # for testing, should be possible to run all concurrently. + artifacts: + - image: docker.io/testkube-agent-server + context: . + custom: + buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set agent-server.tags="$IMAGE" --set agent-server.target="debug" agent-server + dependencies: + dockerfile: + path: build/_local/agent-server.Dockerfile + - image: docker.io/testworkflow-init + context: . + custom: + buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-init.tags="$IMAGE" testworkflow-init + dependencies: + dockerfile: + path: build/_local/testworkflow-init.Dockerfile + - image: docker.io/testworkflow-toolkit + context: . + custom: + buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-toolkit.tags="$IMAGE" --set testworkflow-toolkit.target="debug" testworkflow-toolkit + dependencies: + dockerfile: + path: build/_local/testworkflow-toolkit.Dockerfile +deploy: + helm: + # see https://skaffold.dev/docs/renderers/helm/#skaffoldyaml-configuration + releases: + - name: testkube-agent + repo: https://kubeshop.github.io/helm-charts + remoteChart: testkube + # Alternative: Local chart - useful for when you are actively making changes to the chart. +# chartPath: /Users/you/path/to/helm-charts/charts/testkube + upgradeOnChange: true +# skipBuildDependencies: true # This implies that you need to build dependencies yourself when you make local chart changes! + namespace: tk-dev + wait: true + createNamespace: true + valuesFiles: ['build/_local/values.dev.yaml'] # IMPORTANT: You will have to copy the values.dev.tpl.yaml template to get started! + setValueTemplates: + testkube-api.image.registry: '{{.IMAGE_DOMAIN_docker_io_testkube_agent_server}}' + testkube-api.image.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testkube_agent_server}}' + testkube-api.image.tag: '{{.IMAGE_TAG_docker_io_testkube_agent_server}}@{{.IMAGE_DIGEST_docker_io_testkube_agent_server}}' + testkube-api.imageTwInit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_init}}' + testkube-api.imageTwInit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_init}}' + testkube-api.imageTwInit.tag: '{{.IMAGE_TAG_docker_io_testworkflow_init}}@{{.IMAGE_DIGEST_docker_io_testworkflow_init}}' + testkube-api.imageTwToolkit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_toolkit}}' + testkube-api.imageTwToolkit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_toolkit}}' + testkube-api.imageTwToolkit.tag: '{{.IMAGE_TAG_docker_io_testworkflow_toolkit}}@{{.IMAGE_DIGEST_docker_io_testworkflow_toolkit}}' + flags: + upgrade: ["--no-hooks"] + statusCheckDeadlineSeconds: 300 + tolerateFailuresUntilDeadline: true