test: refactor E2E examples tests and fix Makefile (argoproj#14094)

Signed-off-by: Mason Malone <[email protected]>

.github/workflows/ci-build.yaml

@@ -61,7 +61,6 @@ jobs:
               - sdks/**
               # example test suite
               - examples/**
-              - hack/test-examples.sh
             codegen:
               - *common
               # generated files

Makefile

@@ -51,7 +51,7 @@ endif
 # -- install & run options
 PROFILE               ?= minimal
 KUBE_NAMESPACE        ?= argo # namespace where Kubernetes resources/RBAC will be installed
-PLUGINS               ?= $(shell [ $PROFILE = plugins ] && echo false || echo true)
+PLUGINS               ?= $(shell [ $(PROFILE) = plugins ] && echo true || echo false)
 UI                    ?= false # start the UI with HTTP
 UI_SECURE             ?= false # start the UI with HTTPS
 API                   ?= $(UI) # start the Argo Server
@@ -608,10 +608,6 @@ test-cli: ./dist/argo
 test-%:
 	E2E_WAIT_TIMEOUT=$(E2E_WAIT_TIMEOUT) go test -failfast -v -timeout $(E2E_SUITE_TIMEOUT) -count 1 --tags $* -parallel $(E2E_PARALLEL) ./test/e2e
 
-.PHONY: test-examples
-test-examples:
-	./hack/test-examples.sh
-
 .PHONY: test-%-sdk
 test-%-sdk:
 	make --directory sdks/$* install test -B

examples/webhdfs-input-output-artifacts.yaml

@@ -49,7 +49,6 @@ spec:
       artifacts:
       - name: my-art2
         path: /my-artifact
-        overwrite: true
         http:
           # below is an example on how to use authentication via certificates
           # clientCert.clientCertSecret: points to a kubernetes secret named cert-sec with a data entry of "certificate.pem"

test/e2e/cli_test.go

@@ -1392,7 +1392,6 @@ func (s *CLISuite) TestCronCommands() {
 			assert.Contains(t, output, "5 5 5 * *")
 			assert.Contains(t, output, "Replace")
 			assert.Contains(t, output, "whalesay")
-			assert.NotContains(t, output, "argosay")
 		})
 	})
 	s.Run("Create Parameter Override", func() {

test/e2e/cron/basic-update-template.yaml

@@ -26,6 +26,5 @@ spec:
           parameters:
             - name: message
         container:
-          image: python:alpine3.6
-          command: [ "sh", -c ]
-          args: [ "echo {{inputs.parameters.message}}" ]
+          image: argoproj/argosay:v2
+          args: [ "echo", "{{inputs.parameters.message}}" ]

test/e2e/cron/cron-and-malformed-template.yaml

@@ -18,9 +18,8 @@ spec:
     templates:
       - name: whalesay
         container:
-          image: python:alpine3.6
-          command: ["sh", -c]
-          args: ["echo hello"]
+          image: argoproj/argosay:v2
+          args: ["echo", "hello"]
 
 ---
 

test/e2e/cron/param.yaml

@@ -25,6 +25,5 @@ spec:
           parameters:
             - name: message
         container:
-          image: python:alpine3.6
-          command: ["sh", -c]
-          args: ["echo {{inputs.parameters.message}}"]
+          image: argoproj/argosay:v2
+          args: ["echo", "{{inputs.parameters.message}}"]

test/e2e/cron_test.go

@@ -454,8 +454,7 @@ spec:
 }
 
 func (s *CronSuite) TestMalformedCronWorkflow() {
-	s.Given().
-		Exec("kubectl", []string{"apply", "-f", "testdata/malformed/malformed-cronworkflow.yaml"}, fixtures.ErrorOutput("unknown field \"spec.workflowSpec.arguments.parameters.someParam\""))
+	s.Given().KubectlApply("testdata/malformed/malformed-cronworkflow.yaml", fixtures.ErrorOutput(".spec.workflowSpec.arguments.parameters: expected list"))
 }
 
 func TestCronSuite(t *testing.T) {

test/e2e/examples_test.go

@@ -0,0 +1,47 @@
+//go:build examples
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/argoproj/argo-workflows/v3/test/e2e/fixtures"
+	fileutil "github.com/argoproj/argo-workflows/v3/util/file"
+	"github.com/argoproj/argo-workflows/v3/workflow/common"
+)
+
+type ExamplesSuite struct {
+	fixtures.E2ESuite
+}
+
+func (s *ExamplesSuite) BeforeTest(suiteName, testName string) {
+	s.E2ESuite.BeforeTest(suiteName, testName)
+	s.Given().KubectlApply("../../examples/configmaps/simple-parameters-configmap.yaml", fixtures.NoError)
+}
+
+func (s *ExamplesSuite) TestExampleWorkflows() {
+	err := fileutil.WalkManifests("../../examples", func(path string, data []byte) error {
+		wfs, err := common.SplitWorkflowYAMLFile(data, true)
+		if err != nil {
+			s.T().Fatalf("Error parsing %s: %v", path, err)
+		}
+		for _, wf := range wfs {
+			if _, ok := wf.GetLabels()["workflows.argoproj.io/test"]; ok {
+				s.T().Logf("Found example workflow at %s with test label\n", path)
+				s.Given().
+					ExampleWorkflow(&wf).
+					When().
+					SubmitWorkflow().
+					WaitForWorkflow(fixtures.ToBeSucceeded)
+			}
+		}
+		return nil
+	})
+	s.CheckError(err)
+}
+
+func TestExamplesSuite(t *testing.T) {
+	suite.Run(t, new(ExamplesSuite))
+}

test/e2e/expectedfailures/exit-handler-fail-missing-output.yaml

@@ -26,14 +26,13 @@ spec:
             valueFrom:
               path: /tmp/hello_world.txt
       container:
-        image: alpine:latest
+        image: argoproj/argosay:v2
         command: ["sh", "-c"]
         args: ["echo intentional failure; exit 1"]
     - name: lifecycle-hook
       inputs:
         parameters:
           - name: hello-param
       container:
-        image: busybox
-        command: [echo]
-        args: ["Hello param: {{inputs.parameters.hello-param}}"]
+        image: argoproj/argosay:v2
+        args: ["echo", "Hello param: {{inputs.parameters.hello-param}}"]

test/e2e/expectedfailures/parallelism-dag-fail-fast.yaml

@@ -16,11 +16,9 @@ spec:
             template: sleep
     - name: fail
       container:
-        image: alpine:latest
-        command: [ sh, -c ]
-        args: ["exit 1"]
+        image: argoproj/argosay:v2
+        args: ["exit", "1"]
     - name: sleep
       container:
-        image: alpine:latest
-        command: [sh, -c]
-        args: ["sleep 5"]
+        image: argoproj/argosay:v2
+        args: ["sleep", "5"]

test/e2e/expectedfailures/parallelism-step-fail-fast.yaml

@@ -15,11 +15,9 @@ spec:
             template: sleep
     - name: fail
       container:
-        image: alpine:latest
-        command: [sh, -c]
-        args: ["exit 1"]
+        image: argoproj/argosay:v2
+        command: ["exit", "1"]
     - name: sleep
       container:
-        image: alpine:latest
-        command: [ sh, -c ]
-        args: [ "sleep 5" ]
+        image: argoproj/argosay:v2
+        command: ["sleep", "5"]

test/e2e/fixtures/given.go

@@ -1,13 +1,11 @@
 package fixtures
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
-	"github.com/TwiN/go-color"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -45,7 +43,16 @@ func (g *Given) Workflow(text string) *Given {
 	g.t.Helper()
 	g.wf = &wfv1.Workflow{}
 	g.readResource(text, g.wf)
-	g.checkImages(g.wf)
+	g.checkImages(g.wf, false)
+	return g
+}
+
+// Load parsed Workflow that's assumed to be from the "examples/" directory
+func (g *Given) ExampleWorkflow(wf *wfv1.Workflow) *Given {
+	g.t.Helper()
+	g.wf = wf
+	g.checkLabels(wf)
+	g.checkImages(g.wf, true)
 	return g
 }
 
@@ -83,38 +90,40 @@ func (g *Given) readResource(text string, v metav1.Object) {
 	}
 }
 
-func (g *Given) checkImages(wf interface{}) {
+// Check if given Workflow, WorkflowTemplate, or CronWorkflow uses forbidden images.
+// Using an arbitrary image will result in slow and flakey tests as we can't really predict when they'll be
+// downloaded or evicted. To keep tests fast and reliable you must use allowed images.
+// Workflows from the examples/ folder are given special treatment and allowed to use a wider range of images.
+func (g *Given) checkImages(wf interface{}, isExample bool) {
 	g.t.Helper()
 	var defaultImage string
 	var templates []wfv1.Template
 	switch baseTemplate := wf.(type) {
 	case *wfv1.Workflow:
+		templates = baseTemplate.Spec.Templates
 		if baseTemplate.Spec.TemplateDefaults != nil && baseTemplate.Spec.TemplateDefaults.Container != nil && baseTemplate.Spec.TemplateDefaults.Container.Image != "" {
 			defaultImage = baseTemplate.Spec.TemplateDefaults.Container.Image
-			templates = baseTemplate.Spec.Templates
 		}
 	case *wfv1.WorkflowTemplate:
+		templates = baseTemplate.Spec.Templates
 		if baseTemplate.Spec.TemplateDefaults != nil && baseTemplate.Spec.TemplateDefaults.Container != nil && baseTemplate.Spec.TemplateDefaults.Container.Image != "" {
 			defaultImage = baseTemplate.Spec.TemplateDefaults.Container.Image
-			templates = baseTemplate.Spec.Templates
 		}
 	case *wfv1.CronWorkflow:
+		templates = baseTemplate.Spec.WorkflowSpec.Templates
 		if baseTemplate.Spec.WorkflowSpec.TemplateDefaults != nil && baseTemplate.Spec.WorkflowSpec.TemplateDefaults.Container != nil && baseTemplate.Spec.WorkflowSpec.TemplateDefaults.Container.Image != "" {
 			defaultImage = baseTemplate.Spec.WorkflowSpec.TemplateDefaults.Container.Image
-			templates = baseTemplate.Spec.WorkflowSpec.Templates
 		}
 	default:
 		g.t.Fatalf("Unsupported checkImage workflow type: %s", wf)
 	}
 
-	// discouraged
-	discouraged := func(image string) bool {
-		return image == "python:alpine3.6"
-	}
-	// Using an arbitrary image will result in slow and flakey tests as we can't really predict when they'll be
-	// downloaded or evicted. To keep tests fast and reliable you must use allowed images.
 	allowed := func(image string) bool {
-		return strings.Contains(image, "argoexec:") || image == "argoproj/argosay:v1" || image == "argoproj/argosay:v2" || discouraged(image)
+		return strings.Contains(image, "argoexec:") ||
+			image == "argoproj/argosay:v1" ||
+			image == "argoproj/argosay:v2" ||
+			image == "quay.io/argoproj/argocli:latest" ||
+			(isExample && (image == "busybox" || image == "python:alpine3.6"))
 	}
 	for _, t := range templates {
 		container := t.Container
@@ -131,9 +140,6 @@ func (g *Given) checkImages(wf interface{}) {
 			// (⎈ |docker-desktop:argo)➜  ~ time docker run --rm argoproj/argosay:v2
 			// docker run --rm argoproj/argosay˜:v2  0.21s user 0.10s system 16% cpu 1.912 total
 			// docker run --rm argoproj/argosay:v1  0.17s user 0.08s system 31% cpu 0.784 total
-			if discouraged(image) {
-				_, _ = fmt.Println(color.Ize(color.Yellow, "DISCOURAGED IMAGE: "+g.t.Name()+" is using "+image))
-			}
 		}
 	}
 }
@@ -168,7 +174,7 @@ func (g *Given) WorkflowTemplate(text string) *Given {
 	g.t.Helper()
 	wfTemplate := &wfv1.WorkflowTemplate{}
 	g.readResource(text, wfTemplate)
-	g.checkImages(wfTemplate)
+	g.checkImages(wfTemplate, false)
 	g.wfTemplates = append(g.wfTemplates, wfTemplate)
 	return g
 }
@@ -177,7 +183,7 @@ func (g *Given) CronWorkflow(text string) *Given {
 	g.t.Helper()
 	g.cronWf = &wfv1.CronWorkflow{}
 	g.readResource(text, g.cronWf)
-	g.checkImages(g.cronWf)
+	g.checkImages(g.cronWf, false)
 	return g
 }
 
@@ -209,6 +215,12 @@ func (g *Given) Exec(name string, args []string, block func(t *testing.T, output
 	return g
 }
 
+// Use Kubectl to server-side apply the given file
+func (g *Given) KubectlApply(file string, block func(t *testing.T, output string, err error)) *Given {
+	g.t.Helper()
+	return g.Exec("kubectl", append([]string{"-n", Namespace, "apply", "--server-side", "-f"}, file), block)
+}
+
 func (g *Given) RunCli(args []string, block func(t *testing.T, output string, err error)) *Given {
 	return g.Exec("../../dist/argo", append([]string{"-n", Namespace}, args...), block)
 }

test/e2e/functional/dag-empty-param.yaml

@@ -22,12 +22,8 @@ spec:
 
   - name: gen-number-list
     script:
-      image: python:alpine3.6
-      command: [python]
-      source: |
-        import json
-        import sys
-        json.dump([i for i in range(0, -1)], sys.stdout)
+      image: argoproj/argosay:v2
+      args: ["echo", "[]"]
 
   - name: sleep-n-sec
     inputs:

test/e2e/functional/entrypointName-template.yaml

@@ -11,6 +11,5 @@ spec:
         - name: test
           value: "{{ workflow.mainEntrypoint }}"
     container:
-      image: alpine
-      command: [echo]
-      args: ["We got here!"]
+      image: argoproj/argosay:v2
+      args: [echo, "We got here!"]

test/e2e/functional/loops-empty-param.yaml

@@ -19,12 +19,8 @@ spec:
 
   - name: gen-number-list
     script:
-      image: python:alpine3.6
-      command: [python]
-      source: |
-        import json
-        import sys
-        json.dump([i for i in range(0, -1)], sys.stdout)
+      image: argoproj/argosay:v2
+      args: ["echo", "[]"]
 
   - name: sleep-n-sec
     inputs: