diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d9d121c1a7..f338c62ba4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: branches: [main, release-*] paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**'] pull_request: - branches: [ main, release-* ] + branches: [main, release-*] paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**'] schedule: - cron: '0 4 * * *' diff --git a/.github/workflows/compatibility-e2e-with-client-go.yml b/.github/workflows/compatibility-e2e-v1.yml similarity index 96% rename from .github/workflows/compatibility-e2e-with-client-go.yml rename to .github/workflows/compatibility-e2e-v1.yml index cf463ba0f12..7944d822812 100644 --- a/.github/workflows/compatibility-e2e-with-client-go.yml +++ b/.github/workflows/compatibility-e2e-v1.yml @@ -1,4 +1,4 @@ -name: Compatibility E2E Test(Golang Client) +name: Compatibility E2E Test(API v1 - Golang Client) on: push: @@ -21,8 +21,7 @@ env: DRAGONFLY_PROXY_SERVER_PATH: test/testdata/k8s/proxy.yaml jobs: - compatibility_e2e_tests_with_client_go: - name: e2e_tests_with_${{ matrix.module }} + compatibility_e2e_tests: timeout-minutes: 60 runs-on: ubuntu-latest strategy: @@ -162,7 +161,9 @@ jobs: DRAGONFLY_COMPATIBILITY_E2E_TEST_MODE: ${{ matrix.module }} DRAGONFLY_COMPATIBILITY_E2E_TEST_IMAGE: ${{ matrix.image }} DRAGONFLY_STABLE_IMAGE_TAG: ${{ matrix.image-tag }} - run: make actions-e2e-test-coverage + run: | + ginkgo -v -r --race --fail-fast --cover --trace --show-node-events test/e2e/v1 + cat coverprofile.out >> coverage.txt - name: Move cache run: | diff --git a/.github/workflows/e2e-with-client-go.yml b/.github/workflows/e2e-v1.yml similarity index 98% rename from .github/workflows/e2e-with-client-go.yml rename to .github/workflows/e2e-v1.yml index 5fd9c17ce17..0d38272b496 100644 --- a/.github/workflows/e2e-with-client-go.yml +++ b/.github/workflows/e2e-v1.yml @@ -1,4 +1,4 @@ -name: E2E Test(Golang Client) +name: E2E Test(API v1 - Golang Client) on: push: @@ -21,7 +21,7 @@ env: DRAGONFLY_MINIO_SERVER_PATH: test/testdata/k8s/minio.yaml jobs: - e2e_tests_with_client_go: + e2e_tests: runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -184,7 +184,7 @@ jobs: make build-e2e-download-grpc-test # generate an empty file docker exec kind-control-plane touch /tmp/empty-file - ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e -- \ + ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e/v1 -- \ --feature-gates=dfget-range=true,dfget-open-range=true,dfget-empty-file=true,dfget-recursive=true cat coverprofile.out >> coverage.txt diff --git a/.github/workflows/e2e-with-client-rs.yml b/.github/workflows/e2e-v2.yml similarity index 76% rename from .github/workflows/e2e-with-client-rs.yml rename to .github/workflows/e2e-v2.yml index c67b6eea014..157979f06cf 100644 --- a/.github/workflows/e2e-with-client-rs.yml +++ b/.github/workflows/e2e-v2.yml @@ -1,4 +1,4 @@ -name: E2E Test(Rust Client) +name: E2E Test(API v2 - Rust Client) on: push: @@ -13,12 +13,12 @@ on: env: KIND_VERSION: v0.12.0 CONTAINERD_VERSION: v1.5.2 - KIND_CONFIG_PATH: test/testdata/kind/config-rs.yaml + KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml jobs: - e2e_tests_with_client_rs: + e2e_tests: runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -27,7 +27,7 @@ jobs: - "normal" include: - module: normal - charts-config: test/testdata/charts/config-rs.yaml + charts-config: test/testdata/charts/config-v2.yaml skip: "" steps: - name: Free Disk Space (Ubuntu) @@ -74,11 +74,11 @@ jobs: - name: Pull Rust Client Image run: | cd client-rs - # TODO Use the latest tag. - # CLIENT_TAG=$(git describe --tags $(git rev-list --tags --max-count=1)) - CLIENT_TAG=latest + CLIENT_TAG=$(git describe --tags $(git rev-parse HEAD)) docker pull dragonflyoss/client:$CLIENT_TAG docker tag dragonflyoss/client:$CLIENT_TAG dragonflyoss/client:latest + docker pull dragonflyoss/dfinit:$CLIENT_TAG + docker tag dragonflyoss/dfinit:$CLIENT_TAG dragonflyoss/dfinit:latest - name: Build Scheduler Image uses: docker/build-push-action@v5 @@ -114,20 +114,19 @@ jobs: kind load docker-image dragonflyoss/manager:latest kind load docker-image dragonflyoss/scheduler:latest kind load docker-image dragonflyoss/client:latest + kind load docker-image dragonflyoss/dfinit:latest - # TODO Setup dragonfly - # - name: Setup dragonfly - # run: | - # helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} - # kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} - # kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m - - # TODO Run tests - # - name: Run E2E test - # run: | - # ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e -- \ - # --feature-gates=dfget-range=true,dfget-open-range=true,dfget-empty-file=true,dfget-recursive=true - # cat coverprofile.out >> coverage.txt + - name: Setup dragonfly + run: | + helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} + kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} + kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m + kubectl get po -n dragonfly-system + + - name: Run E2E test + run: | + ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e/v2 + cat coverprofile.out >> coverage.txt - name: Move cache run: | diff --git a/.gitignore b/.gitignore index 5a5484928ab..0e7597d09ec 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,5 @@ artifacts .run deploy/docker-compose/log deploy/docker-compose/config +test/e2e/v1/*.log +test/e2e/v2/*.log diff --git a/client-rs b/client-rs index cc8032aea2c..2ec8002355a 160000 --- a/client-rs +++ b/client-rs @@ -1 +1 @@ -Subproject commit cc8032aea2c1ddb3289c2be6707dec6d7cf6c119 +Subproject commit 2ec8002355ac3a82f432858e899f864b2818c766 diff --git a/test/e2e/concurrency_test.go b/test/e2e/v1/concurrency_test.go similarity index 98% rename from test/e2e/concurrency_test.go rename to test/e2e/v1/concurrency_test.go index a326fab76db..f8b461f4c52 100644 --- a/test/e2e/concurrency_test.go +++ b/test/e2e/v1/concurrency_test.go @@ -22,7 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint . "github.com/onsi/gomega" //nolint - "d7y.io/dragonfly/v2/test/e2e/util" + "d7y.io/dragonfly/v2/test/e2e/v1/util" ) var _ = Describe("Download concurrency", func() { diff --git a/test/e2e/constants.go b/test/e2e/v1/constants.go similarity index 100% rename from test/e2e/constants.go rename to test/e2e/v1/constants.go diff --git a/test/e2e/containerd_test.go b/test/e2e/v1/containerd_test.go similarity index 97% rename from test/e2e/containerd_test.go rename to test/e2e/v1/containerd_test.go index 5045132e449..664c64d2d70 100644 --- a/test/e2e/containerd_test.go +++ b/test/e2e/v1/containerd_test.go @@ -22,7 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint . "github.com/onsi/gomega" //nolint - "d7y.io/dragonfly/v2/test/e2e/util" + "d7y.io/dragonfly/v2/test/e2e/v1/util" ) var _ = Describe("Containerd with CRI support", func() { diff --git a/test/e2e/dfget_test.go b/test/e2e/v1/dfget_test.go similarity index 99% rename from test/e2e/dfget_test.go rename to test/e2e/v1/dfget_test.go index 818bf2d9d6d..e5da79c061e 100644 --- a/test/e2e/dfget_test.go +++ b/test/e2e/v1/dfget_test.go @@ -27,7 +27,7 @@ import ( . "github.com/onsi/gomega" //nolint "d7y.io/dragonfly/v2/pkg/net/http" - "d7y.io/dragonfly/v2/test/e2e/util" + "d7y.io/dragonfly/v2/test/e2e/v1/util" ) var _ = Describe("Download with dfget and proxy", func() { diff --git a/test/e2e/e2e_test.go b/test/e2e/v1/e2e_test.go similarity index 98% rename from test/e2e/e2e_test.go rename to test/e2e/v1/e2e_test.go index d822814c505..fc06ce7fc95 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/v1/e2e_test.go @@ -28,8 +28,8 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint . "github.com/onsi/gomega" //nolint - _ "d7y.io/dragonfly/v2/test/e2e/manager" - "d7y.io/dragonfly/v2/test/e2e/util" + _ "d7y.io/dragonfly/v2/test/e2e/v1/manager" + "d7y.io/dragonfly/v2/test/e2e/v1/util" ) var featureGatesFlag string diff --git a/test/e2e/manager/constants.go b/test/e2e/v1/manager/constants.go similarity index 100% rename from test/e2e/manager/constants.go rename to test/e2e/v1/manager/constants.go diff --git a/test/e2e/manager/preheat.go b/test/e2e/v1/manager/preheat.go similarity index 69% rename from test/e2e/manager/preheat.go rename to test/e2e/v1/manager/preheat.go index 5f310d01db8..e31416534cb 100644 --- a/test/e2e/manager/preheat.go +++ b/test/e2e/v1/manager/preheat.go @@ -35,13 +35,18 @@ import ( "d7y.io/dragonfly/v2/manager/types" "d7y.io/dragonfly/v2/pkg/idgen" "d7y.io/dragonfly/v2/pkg/structure" - "d7y.io/dragonfly/v2/test/e2e/util" + "d7y.io/dragonfly/v2/test/e2e/v1/util" ) +type taskMetadata struct { + ID string + Sha256 string +} + var _ = Describe("Preheat with manager", func() { Context("preheat", func() { It("preheat files should be ok", Label("preheat", "file"), func() { - var seedPeerPods [3]*util.PodExec + seedPeerPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedPeerPods[i] = getSeedPeerExec(i) } @@ -55,7 +60,7 @@ var _ = Describe("Preheat with manager", func() { out, err := util.DockerCommand("sha256sum", v).CombinedOutput() fmt.Println("original sha256sum: " + string(out)) Expect(err).NotTo(HaveOccurred()) - sha256sum1 := strings.Split(string(out), " ")[0] + taskSha256 := strings.Split(string(out), " ")[0] // preheat file req, err := structure.StructToMap(types.CreatePreheatJobRequest{ @@ -80,12 +85,12 @@ var _ = Describe("Preheat with manager", func() { Expect(done).Should(BeTrue()) // generate task_id, also the filename - seedPeerTaskID := idgen.TaskIDV1(url, &commonv1.UrlMeta{}) - fmt.Println(seedPeerTaskID) + taskID := idgen.TaskIDV1(url, &commonv1.UrlMeta{}) + fmt.Println(taskID) - sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID) + sha256sum, err := calculateSha256ByTaskID(seedPeerPods, taskID) Expect(err).NotTo(HaveOccurred()) - Expect(sha256sum1).To(Equal(sha256sum)) + Expect(taskSha256).To(Equal(sha256sum)) } }) @@ -93,18 +98,18 @@ var _ = Describe("Preheat with manager", func() { url := "https://index.docker.io/v2/dragonflyoss/busybox/manifests/1.35.0" fmt.Println("download image: " + url) - var ( - seedPeerTaskIDs = []string{ - "b6922209dc9616f8736a860e93c3cd7288a4e801517f88eec3df514606d18cdf", - "c0dfae864ae65c285676063eb148d0a0064d5c6c39367fee0bcc1f3700c39c31", - } - sha256sum1 = []string{ - "a711f05d33845e2e9deffcfcc5adf082d7c6e97e3e3a881d193d9aae38f092a8", - "f643e116a03d9604c344edb345d7592c48cc00f2a4848aaf773411f4fb30d2f5", - } - ) - - var seedPeerPods [3]*util.PodExec + taskMetadatas := []taskMetadata{ + { + ID: "b6922209dc9616f8736a860e93c3cd7288a4e801517f88eec3df514606d18cdf", + Sha256: "a711f05d33845e2e9deffcfcc5adf082d7c6e97e3e3a881d193d9aae38f092a8", + }, + { + ID: "c0dfae864ae65c285676063eb148d0a0064d5c6c39367fee0bcc1f3700c39c31", + Sha256: "f643e116a03d9604c344edb345d7592c48cc00f2a4848aaf773411f4fb30d2f5", + }, + } + + seedPeerPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedPeerPods[i] = getSeedPeerExec(i) } @@ -132,10 +137,10 @@ var _ = Describe("Preheat with manager", func() { done := waitForDone(job, fsPod) Expect(done).Should(BeTrue()) - for i, seedPeerTaskID := range seedPeerTaskIDs { - sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID) + for _, taskMetadata := range taskMetadatas { + sha256sum, err := calculateSha256ByTaskID(seedPeerPods, taskMetadata.ID) Expect(err).NotTo(HaveOccurred()) - Expect(sha256sum1[i]).To(Equal(sha256sum)) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) } }) @@ -148,24 +153,30 @@ var _ = Describe("Preheat with manager", func() { url := "https://index.docker.io/v2/dragonflyoss/scheduler/manifests/v2.1.0" fmt.Println("download image: " + url) - var ( - seedPeerTaskIDs = []string{ - "c8ca6a17354d3a79397eef26803e5af84d00a3fd64b0f823922086a31ebdee18", - "b8de5865e2ebf537279683adfbdb5f858b0c7212e5744a1df233086496c245d7", - "e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806", - "7da0721fd078dd46a63298747ffde8fcbe12b53378f282c9def693615ac7993e", - "3639c8c5712e77acd3751142c83150c0a12284a54fa41224a1c7acc0e343020d", - } - sha256sum1 = []string{ - "f1f1039835051ecc04909f939530e86a20f02d2ce5ad7a81c0fa3616f7303944", - "c1d6d1b2d5a367259e6e51a7f4d1ccd66a28cc9940d6599d8a8ea9544dd4b4a8", - "871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7", - "f1a1d290795d904815786e41d39a41dc1af5de68a9e9020baba8bd83b32d8f95", - "f1ffc4b5459e82dc8e7ddd1d1a2ec469e85a1f076090c22851a1f2ce6f71e1a6", - } - ) - - var seedPeerPods [3]*util.PodExec + taskMetadatas := []taskMetadata{ + { + ID: "c8ca6a17354d3a79397eef26803e5af84d00a3fd64b0f823922086a31ebdee18", + Sha256: "f1f1039835051ecc04909f939530e86a20f02d2ce5ad7a81c0fa3616f7303944", + }, + { + ID: "b8de5865e2ebf537279683adfbdb5f858b0c7212e5744a1df233086496c245d7", + Sha256: "c1d6d1b2d5a367259e6e51a7f4d1ccd66a28cc9940d6599d8a8ea9544dd4b4a8", + }, + { + ID: "e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806", + Sha256: "871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7", + }, + { + ID: "7da0721fd078dd46a63298747ffde8fcbe12b53378f282c9def693615ac7993e", + Sha256: "f1a1d290795d904815786e41d39a41dc1af5de68a9e9020baba8bd83b32d8f95", + }, + { + ID: "3639c8c5712e77acd3751142c83150c0a12284a54fa41224a1c7acc0e343020d", + Sha256: "f1ffc4b5459e82dc8e7ddd1d1a2ec469e85a1f076090c22851a1f2ce6f71e1a6", + }, + } + + seedPeerPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedPeerPods[i] = getSeedPeerExec(i) } @@ -194,10 +205,10 @@ var _ = Describe("Preheat with manager", func() { done := waitForDone(job, fsPod) Expect(done).Should(BeTrue()) - for i, seedPeerTaskID := range seedPeerTaskIDs { - sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID) + for _, taskMetadata := range taskMetadatas { + sha256sum, err := calculateSha256ByTaskID(seedPeerPods, taskMetadata.ID) Expect(err).NotTo(HaveOccurred()) - Expect(sha256sum1[i]).To(Equal(sha256sum)) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) } }) @@ -210,24 +221,30 @@ var _ = Describe("Preheat with manager", func() { url := "https://index.docker.io/v2/dragonflyoss/scheduler/manifests/v2.1.0" fmt.Println("download image: " + url) - var ( - seedPeerTaskIDs = []string{ - "9869dbb01ac214e90e4ae667e42d50210c2ff1e63292d73b14f0a7a2226c0320", - "ab049caee13f77d91568d954a5d32f5d2354497cab098887a8a663656daa9840", - "e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806", - "a26e1ac8b70926f45766fcf886f23a833793c39c62237bcda9ffeb158131c0d6", - "7376f665077e91cd0dc410c00242ab88775e3eae19eca4b7b3a29ded14fc3754", - } - sha256sum1 = []string{ - "a0d7a8f11f7e25ca59f0bf470187dd9aa27e7ca951cf67a53c750deea5d3b076", - "a880266d3b77f75696023df2da1ef66c3c565e0f70596242395c9e68de955c7c", - "871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7", - "9b5952218d7711195c6c6fbddbef2780507d20851ca68845d180397d1348f0d8", - "889f4c960ac4ff70774e9c4cfa64efc4823ade0702d0f96c20ff0054ffbbe504", - } - ) - - var seedPeerPods [3]*util.PodExec + taskMetadatas := []taskMetadata{ + { + ID: "9869dbb01ac214e90e4ae667e42d50210c2ff1e63292d73b14f0a7a2226c0320", + Sha256: "a0d7a8f11f7e25ca59f0bf470187dd9aa27e7ca951cf67a53c750deea5d3b076", + }, + { + ID: "ab049caee13f77d91568d954a5d32f5d2354497cab098887a8a663656daa9840", + Sha256: "a880266d3b77f75696023df2da1ef66c3c565e0f70596242395c9e68de955c7c", + }, + { + ID: "e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806", + Sha256: "871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7", + }, + { + ID: "a26e1ac8b70926f45766fcf886f23a833793c39c62237bcda9ffeb158131c0d6", + Sha256: "9b5952218d7711195c6c6fbddbef2780507d20851ca68845d180397d1348f0d8", + }, + { + ID: "7376f665077e91cd0dc410c00242ab88775e3eae19eca4b7b3a29ded14fc3754", + Sha256: "889f4c960ac4ff70774e9c4cfa64efc4823ade0702d0f96c20ff0054ffbbe504", + }, + } + + seedPeerPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { seedPeerPods[i] = getSeedPeerExec(i) } @@ -256,10 +273,10 @@ var _ = Describe("Preheat with manager", func() { done := waitForDone(job, fsPod) Expect(done).Should(BeTrue()) - for i, seedPeerTaskID := range seedPeerTaskIDs { - sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID) + for _, taskMetadata := range taskMetadatas { + sha256sum, err := calculateSha256ByTaskID(seedPeerPods, taskMetadata.ID) Expect(err).NotTo(HaveOccurred()) - Expect(sha256sum1[i]).To(Equal(sha256sum)) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) } }) }) @@ -294,18 +311,18 @@ func waitForDone(preheat *models.Job, pod *util.PodExec) bool { } } -func checkPreheatResult(seedPeerPods [3]*util.PodExec, seedPeerTaskID string) (string, error) { +func calculateSha256ByTaskID(pods []*util.PodExec, taskID string) (string, error) { var sha256sum string - for _, seedPeer := range seedPeerPods { - taskDir := fmt.Sprintf("%s/%s", seedPeerDataPath, seedPeerTaskID) - if _, err := seedPeer.Command("ls", taskDir).CombinedOutput(); err != nil { + for _, pod := range pods { + taskDir := fmt.Sprintf("%s/%s", seedPeerDataPath, taskID) + if _, err := pod.Command("ls", taskDir).CombinedOutput(); err != nil { // if the directory does not exist, skip this seed peer fmt.Printf("directory %s does not exist: %s\n", taskDir, err.Error()) continue } // calculate digest of downloaded file - out, err := seedPeer.Command("sh", "-c", fmt.Sprintf("sha256sum %s/*/%s", taskDir, "data")).CombinedOutput() + out, err := pod.Command("sh", "-c", fmt.Sprintf("sha256sum %s/*/%s", taskDir, "data")).CombinedOutput() fmt.Println("preheat sha256sum: " + string(out)) Expect(err).NotTo(HaveOccurred()) sha256sum = strings.Split(string(out), " ")[0] diff --git a/test/e2e/network_topology_test.go b/test/e2e/v1/network_topology_test.go similarity index 99% rename from test/e2e/network_topology_test.go rename to test/e2e/v1/network_topology_test.go index 760f98e1a86..0fbd50b4e00 100644 --- a/test/e2e/network_topology_test.go +++ b/test/e2e/v1/network_topology_test.go @@ -27,7 +27,7 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint . "github.com/onsi/gomega" //nolint - "d7y.io/dragonfly/v2/test/e2e/util" + "d7y.io/dragonfly/v2/test/e2e/v1/util" ) var _ = Describe("Evaluator with networkTopology", func() { diff --git a/test/e2e/util/artifact.go b/test/e2e/v1/util/artifact.go similarity index 100% rename from test/e2e/util/artifact.go rename to test/e2e/v1/util/artifact.go diff --git a/test/e2e/util/exec.go b/test/e2e/v1/util/exec.go similarity index 100% rename from test/e2e/util/exec.go rename to test/e2e/v1/util/exec.go diff --git a/test/e2e/util/feature_gate.go b/test/e2e/v1/util/feature_gate.go similarity index 100% rename from test/e2e/util/feature_gate.go rename to test/e2e/v1/util/feature_gate.go diff --git a/test/e2e/util/file_server.go b/test/e2e/v1/util/file_server.go similarity index 100% rename from test/e2e/util/file_server.go rename to test/e2e/v1/util/file_server.go diff --git a/test/e2e/v2/containerd_test.go b/test/e2e/v2/containerd_test.go new file mode 100644 index 00000000000..81105a11c79 --- /dev/null +++ b/test/e2e/v2/containerd_test.go @@ -0,0 +1,259 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" //nolint + . "github.com/onsi/gomega" //nolint + + "d7y.io/dragonfly/v2/test/e2e/v2/util" +) + +var _ = Describe("Containerd with CRI support", func() { + Context("dragonflyoss/manager:v2.0.0 image", func() { + It("pull should be ok", Label("containerd", "pull"), func() { + out, err := util.CriCtlCommand("pull", "dragonflyoss/manager:v2.1.0").CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + + taskMetadatas := []util.TaskMetadata{ + { + ID: "ff679754c7951659b21b84fedc15c82e285909146329eae5114a762459b49221", + Sha256: "ca51217de9012bffe54390f1a91365af22a06279a3f2b3e57d4d2dc99b989588", + }, + { + ID: "cfe7b96cdd7af51b0433507da8c4239adb1985d27c8a62136b7ca4c2b0796d63", + Sha256: "0d816dfc0753b877a04e3df93557bd3597fc7d0e308726655b14401c22a3b92a", + }, + { + ID: "43ab6b508e9e57e990d38322572a39fd628fd5ec34d83ecd66aee5d199265f84", + Sha256: "b5941d5a445040d3a792e5be361ca42989d97fc30ff53031f3004ccea8e44520", + }, + { + ID: "88c81ef2a014504dd493eeb4ad140dbfe05e12a53038fea2f4f04e10fd4bcf38", + Sha256: "c1d6d1b2d5a367259e6e51a7f4d1ccd66a28cc9940d6599d8a8ea9544dd4b4a8", + }, + { + ID: "adad87ebcac1fc92d13051a8f15fc29f33f733a93ad322119e00764cbbbcb501", + Sha256: "2a1bc4e0f20bb5ed9a2197ecffde7eace4a9b9179048614205d025df73ba97c7", + }, + { + ID: "1dcf30a9df83b64fd8ad85288ed01def4d8758ee9c433861b86ceced46b2c97d", + Sha256: "078ea4eebc352a499d7bb6ff65fab1325226e524acac89a9db922ad91cab88f1", + }, + } + + clientPod, err := util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + for _, taskMetadata := range taskMetadatas { + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, taskMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) + } + + seedClientPods := make([]*util.PodExec, 3) + for i := 0; i < 3; i++ { + seedClientPods[i], err = util.SeedClientExec(i) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + } + for _, taskMetadata := range taskMetadatas { + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, taskMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) + } + }) + + It("rmi should be ok", Label("containerd", "rmi"), func() { + out, err := util.CriCtlCommand("rmi", "dragonflyoss/manager:v2.1.0").CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("dragonflyoss/scheduler:v2.0.0 image", func() { + It("pull should be ok", Label("containerd", "pull"), func() { + out, err := util.CriCtlCommand("pull", "dragonflyoss/scheduler:v2.0.0").CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + + taskMetadatas := []util.TaskMetadata{ + { + ID: "69fa645056298e4809c88cf86c3fb559ffa62bb4d1f986cacf1aa81933e3d030", + Sha256: "0f4277a6444fbaf4eb5a7f39103e281dd57969953c7425edc7c8d4aa419347eb", + }, + { + ID: "23eaaf799cb4191256c352aec0a207089a26e105e779e7f93f3726598c975165", + Sha256: "e55b67c1d5660c34dcb0d8e6923d0a50695a4f0d94f858353069bae17d0bfdea", + }, + { + ID: "4dbaa1695f5410bcd59560cf79408c1ae69d27c8cfd94259359ff69e026ebdaa", + Sha256: "8572bc8fb8a32061648dd183b2c0451c82be1bd053a4ea8fae991436b92faebb", + }, + { + ID: "c65dac943d3f294a1e0005618ac197a4f8d38eb93176a631eba7e34913cb5747", + Sha256: "88bfc12bad0cc91b2d47de4c7a755f6547b750256cc4c8b284e07aae13e4e041", + }, + } + + clientPod, err := util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + for _, taskMetadata := range taskMetadatas { + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, taskMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) + } + + seedClientPods := make([]*util.PodExec, 3) + for i := 0; i < 3; i++ { + seedClientPods[i], err = util.SeedClientExec(i) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + } + for _, taskMetadata := range taskMetadatas { + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, taskMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) + } + }) + + It("rmi should be ok", Label("containerd", "rmi"), func() { + out, err := util.CriCtlCommand("rmi", "dragonflyoss/scheduler:v2.0.0").CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("dragonflyoss/client:v0.1.30 image", func() { + It("pull should be ok", Label("containerd", "pull"), func() { + out, err := util.CriCtlCommand("pull", "dragonflyoss/client:v0.1.30").CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + + taskMetadatas := []util.TaskMetadata{ + { + ID: "d6998ea9e6f6d0fe8dfa5efe1098d4bc29234a298a1bf857c3129d534c064dd5", + Sha256: "c8071d0de0f5bb17fde217dafdc9d2813ce9db77e60f6233bcd32f1c8888b121", + }, + { + ID: "9764b091db2dca99e2a0c09c776be3915e913f02def63e794435e048d2cb7ad7", + Sha256: "e964513726885fa2f977425fc889eabbe25c9fa47e7a4b0ec5e2baef96290f47", + }, + { + ID: "e376fd3b1d92d8db1a29fc7aa307ae243eaa229a84ffc4f4f84247046fd75850", + Sha256: "0e304933d7eae4674e05b3bc409f236c65077e2b7055119bbd66ff613fe5e1ad", + }, + { + ID: "77fb97fa428921c5a5a72be510c69b1208db540479dda3a1ff765a248ed23287", + Sha256: "53b01ef3d5d676a8514ded6b469932e33d84738e5e00932ca124382a8567c44b", + }, + { + ID: "b58219de7051f520e222ce29c90e2d61b49649d714bd2d9e6ebcebbc83a15f2b", + Sha256: "c9d959fc168ad8bdc9a021066eb9c1dd4de8e860c03619a88d8ba0ff5479d9ea", + }, + { + ID: "4bceb8758fe687ac33610c23f503dc13d9050c15be3f20f9141b94a450070d9f", + Sha256: "b6acfae843b58bf14369ebbeafa96af5352cde9a89f8255ca51f92b233a6e405", + }, + } + + clientPod, err := util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + for _, taskMetadata := range taskMetadatas { + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, taskMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) + } + + seedClientPods := make([]*util.PodExec, 3) + for i := 0; i < 3; i++ { + seedClientPods[i], err = util.SeedClientExec(i) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + } + for _, taskMetadata := range taskMetadatas { + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, taskMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) + } + }) + + It("rmi should be ok", Label("containerd", "rmi"), func() { + out, err := util.CriCtlCommand("rmi", "dragonflyoss/client:v0.1.30").CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("dragonflyoss/dfinit:v0.1.30 image", func() { + It("pull should be ok", Label("containerd", "pull"), func() { + out, err := util.CriCtlCommand("pull", "dragonflyoss/dfinit:v0.1.30").CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + + taskMetadatas := []util.TaskMetadata{ + { + ID: "5f04fd58b821860c616ad3f8cfe096c53969086a99ec9d63a7b6d75e643cd33f", + Sha256: "c58d97dd21c3b3121f262a1fbb5a278f77ab85dba7a02b819e710f34683cf746", + }, + { + ID: "67948562926bcaafea035f5e7ae8007f1b367c5fb050ed57fc70faa1b95d73af", + Sha256: "2ff0ae26fa61a2b0f88f470a8e50f7623ea48b224eb072a5878a20d663d5307d", + }, + { + ID: "ea70b1749ce2e542271512ba2a50b01787b4a17edd42a2c1450097c60845a10c", + Sha256: "b1826117441e607acd3b98c93cdb16759c2cc2240852055b8a2b5860f3204f1e", + }, + } + + clientPod, err := util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + for _, taskMetadata := range taskMetadatas { + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, taskMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) + } + + seedClientPods := make([]*util.PodExec, 3) + for i := 0; i < 3; i++ { + seedClientPods[i], err = util.SeedClientExec(i) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + } + for _, taskMetadata := range taskMetadatas { + sha256sum, err := util.CalculateSha256ByTaskID(seedClientPods, taskMetadata.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(taskMetadata.Sha256).To(Equal(sha256sum)) + } + }) + + It("rmi should be ok", Label("containerd", "rmi"), func() { + out, err := util.CriCtlCommand("rmi", "dragonflyoss/dfinit:v0.1.30").CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/test/e2e/v2/e2e_test.go b/test/e2e/v2/e2e_test.go new file mode 100644 index 00000000000..f36065d0715 --- /dev/null +++ b/test/e2e/v2/e2e_test.go @@ -0,0 +1,95 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "fmt" + "strconv" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" //nolint + . "github.com/onsi/gomega" //nolint + + // _ "d7y.io/dragonfly/v2/test/e2e/v2/manager" + "d7y.io/dragonfly/v2/test/e2e/v2/util" +) + +var _ = AfterSuite(func() { + for _, server := range util.Servers { + for i := 0; i < server.Replicas; i++ { + fmt.Printf("\n------------------------------ Get %s-%d Artifact Started ------------------------------\n", server.Name, i) + + out, err := util.KubeCtlCommand("-n", server.Namespace, "get", "pod", "-l", fmt.Sprintf("component=%s", server.Name), + "-o", fmt.Sprintf("jsonpath='{.items[%d].metadata.name}'", i)).CombinedOutput() + if err != nil { + fmt.Printf("get pod error: %s, output: %s\n", err, string(out)) + continue + } + podName := strings.Trim(string(out), "'") + pod := util.NewPodExec(server.Namespace, podName, server.Name) + + countOut, err := util.KubeCtlCommand("-n", server.Namespace, "get", "pod", "-l", fmt.Sprintf("component=%s", server.Name), + "-o", fmt.Sprintf("jsonpath='{.items[%d].status.containerStatuses[0].restartCount}'", i)).CombinedOutput() + if err != nil { + fmt.Printf("get pod %s restart count error: %s, output: %s\n", podName, err, string(countOut)) + continue + } + rawCount := strings.Trim(string(countOut), "'") + count, err := strconv.Atoi(rawCount) + if err != nil { + fmt.Printf("atoi error: %s\n", err) + continue + } + fmt.Printf("pod %s restart count: %d\n", podName, count) + + out, err = pod.Command("sh", "-c", fmt.Sprintf(` + set -x + cp -r /var/log/dragonfly/%s /tmp/artifact/%s-%d + find /tmp/artifact -type d -exec chmod 777 {} \; + `, server.LogDirName, server.Name, i)).CombinedOutput() + if err != nil { + fmt.Printf("copy log output: %q, error: %s\n", string(out), err) + } + + if count > 0 { + if err := util.UploadArtifactPrevStdout(server.Namespace, podName, fmt.Sprintf("%s-%d", server.Name, i), server.Name); err != nil { + fmt.Printf("upload pod %s artifact prev stdout file error: %v\n", podName, err) + } + } + + if err := util.UploadArtifactStdout(server.Namespace, podName, fmt.Sprintf("%s-%d", server.Name, i), server.Name); err != nil { + fmt.Printf("upload pod %s artifact stdout file error: %v\n", podName, err) + } + + fmt.Printf("------------------------------ Get %s-%d Artifact Finished ------------------------------\n", server.Name, i) + } + } +}) + +var _ = BeforeSuite(func() { + rawGitCommit, err := util.GitCommand("rev-parse", "--short", "HEAD").CombinedOutput() + Expect(err).NotTo(HaveOccurred()) + gitCommit := strings.Fields(string(rawGitCommit))[0] + fmt.Printf("git merge commit: %s\n", gitCommit) +}) + +// TestE2E is the root of e2e test function +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "dragonfly e2e test suite") +} diff --git a/test/e2e/v2/util/artifact.go b/test/e2e/v2/util/artifact.go new file mode 100644 index 00000000000..40ef5c2806d --- /dev/null +++ b/test/e2e/v2/util/artifact.go @@ -0,0 +1,78 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +const ( + defaultFileMode = os.FileMode(0664) +) + +func UploadArtifactStdout(namespace, podName, logDirName, logPrefix string) error { + out, err := KubeCtlCommand("-n", namespace, "logs", podName).CombinedOutput() + if err != nil { + return err + } + + logFileName := fmt.Sprintf("%s-stdout.log", logPrefix) + logDirname := fmt.Sprintf("/tmp/artifact/%s/", logDirName) + + if err := os.MkdirAll(logDirname, defaultFileMode); err != nil { + return err + } + + if err := os.WriteFile(logFileName, out, defaultFileMode); err != nil { + return err + } + + if out, err := exec.Command("mv", logFileName, filepath.Join(logDirname, logFileName)).CombinedOutput(); err != nil { + fmt.Printf("move %s error: %s\n", logFileName, out) + return err + } + + return nil +} + +func UploadArtifactPrevStdout(namespace, podName, logDirName, logPrefix string) error { + out, err := KubeCtlCommand("-n", namespace, "logs", podName, "-p").CombinedOutput() + if err != nil { + return err + } + + logFileName := fmt.Sprintf("%s-prev-stdout.log", logPrefix) + logDirname := fmt.Sprintf("/tmp/artifact/%s/", logDirName) + + if err := os.MkdirAll(logDirname, defaultFileMode); err != nil { + return err + } + + if err := os.WriteFile(logFileName, out, defaultFileMode); err != nil { + return err + } + + if out, err := exec.Command("mv", logFileName, filepath.Join(logDirname, logFileName)).CombinedOutput(); err != nil { + fmt.Printf("move %s error: %s\n", logFileName, out) + return err + } + + return nil +} diff --git a/test/e2e/v2/util/constants.go b/test/e2e/v2/util/constants.go new file mode 100644 index 00000000000..1e220300c36 --- /dev/null +++ b/test/e2e/v2/util/constants.go @@ -0,0 +1,62 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +const ( + DragonflyNamespace = "dragonfly-system" +) + +const ( + ManagerServerName = "manager" + SchedulerServerName = "scheduler" + SeedClientServerName = "seed-client" + ClientServerName = "client" +) + +type server struct { + Name string + Namespace string + LogDirName string + Replicas int +} + +var Servers = map[string]server{ + ManagerServerName: { + Name: ManagerServerName, + Namespace: DragonflyNamespace, + LogDirName: ManagerServerName, + Replicas: 1, + }, + SchedulerServerName: { + Name: SchedulerServerName, + Namespace: DragonflyNamespace, + LogDirName: SchedulerServerName, + Replicas: 3, + }, + SeedClientServerName: { + Name: SeedClientServerName, + Namespace: DragonflyNamespace, + LogDirName: "dfdaemon", + Replicas: 3, + }, + ClientServerName: { + Name: ClientServerName, + Namespace: DragonflyNamespace, + LogDirName: "dfdaemon", + Replicas: 1, + }, +} diff --git a/test/e2e/v2/util/exec.go b/test/e2e/v2/util/exec.go new file mode 100644 index 00000000000..678fb3fd014 --- /dev/null +++ b/test/e2e/v2/util/exec.go @@ -0,0 +1,131 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "encoding/json" + "fmt" + "os/exec" + "strings" +) + +const ( + kindDockerContainer = "kind-control-plane" +) + +func DockerCommand(arg ...string) *exec.Cmd { + container := kindDockerContainer + extArgs := []string{"exec", "-i", container} + extArgs = append(extArgs, arg...) + fmt.Println(fmt.Sprintf(`docker %s exec: "%s"`, container, strings.Join(arg, `" "`))) + return exec.Command("docker", extArgs...) +} + +func DockerCopy(dst, src string) *exec.Cmd { + container := kindDockerContainer + args := []string{"cp", src, fmt.Sprintf("%s:%s", container, dst)} + fmt.Println(fmt.Sprintf(`docker cp %s to %s:%s"`, src, container, dst)) + return exec.Command("docker", args...) +} + +func CriCtlCommand(arg ...string) *exec.Cmd { + extArgs := []string{"/usr/local/bin/crictl"} + extArgs = append(extArgs, arg...) + return DockerCommand(extArgs...) +} + +func KubeCtlCommand(arg ...string) *exec.Cmd { + fmt.Println(fmt.Sprintf(`kubectl command: "kubectl" "%s"`, strings.Join(arg, `" "`))) + return exec.Command("kubectl", arg...) +} + +func ABCommand(arg ...string) *exec.Cmd { + return exec.Command("ab", arg...) +} + +func GitCommand(arg ...string) *exec.Cmd { + return exec.Command("git", arg...) +} + +type PodExec struct { + namespace string + name string + container string +} + +func NewPodExec(namespace string, name string, container string) *PodExec { + return &PodExec{ + namespace: namespace, + name: name, + container: container, + } +} + +func (p *PodExec) Command(arg ...string) *exec.Cmd { + extArgs := []string{"-n", p.namespace, "exec", p.name, "--"} + if p.container != "" { + extArgs = []string{"-n", p.namespace, "exec", "-c", p.container, p.name, "--"} + } + extArgs = append(extArgs, arg...) + fmt.Println(fmt.Sprintf(`pod %s/%s exec: "%s"`, p.namespace, p.name, strings.Join(arg, `" "`))) + return KubeCtlCommand(extArgs...) +} + +func (p *PodExec) CurlCommand(method string, header map[string]string, data map[string]any, target string) *exec.Cmd { + extArgs := []string{"/usr/bin/curl", target, "-s"} + if method != "" { + extArgs = append(extArgs, "-X", method) + } + if header != nil { + for k, v := range header { + extArgs = append(extArgs, "-H", fmt.Sprintf("%s:%s", k, v)) + } + } + if data != nil { + b, _ := json.Marshal(data) + extArgs = append(extArgs, "-d", string(b)) + } + return p.Command(extArgs...) +} + +func KubeCtlCopyCommand(ns, pod, source, target string) *exec.Cmd { + return KubeCtlCommand("-n", ns, "cp", pod+":"+source, target) +} + +func ClientExec() (*PodExec, error) { + out, err := KubeCtlCommand("-n", DragonflyNamespace, "get", "pod", "-l", "component=client", + "-o", fmt.Sprintf("jsonpath='{range .items[0]}{.metadata.name}{end}'")).CombinedOutput() + if err != nil { + return nil, err + } + + podName := strings.Trim(string(out), "'") + fmt.Println(podName) + return NewPodExec(DragonflyNamespace, podName, "client"), nil +} + +func SeedClientExec(n int) (*PodExec, error) { + out, err := KubeCtlCommand("-n", DragonflyNamespace, "get", "pod", "-l", "component=seed-client", + "-o", fmt.Sprintf("jsonpath='{range .items[%d]}{.metadata.name}{end}'", n)).CombinedOutput() + if err != nil { + return nil, err + } + + podName := strings.Trim(string(out), "'") + fmt.Println(podName) + return NewPodExec(DragonflyNamespace, podName, "seed-client"), nil +} diff --git a/test/e2e/v2/util/file_server.go b/test/e2e/v2/util/file_server.go new file mode 100644 index 00000000000..80af28651be --- /dev/null +++ b/test/e2e/v2/util/file_server.go @@ -0,0 +1,47 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import "fmt" + +func GetFileList() []string { + return []string{ + "/etc/containerd/config.toml", + "/etc/fstab", + "/etc/hostname", + "/usr/bin/kubectl", + "/usr/bin/systemctl", + "/usr/local/bin/containerd-shim", + "/usr/local/bin/clean-install", + "/usr/local/bin/entrypoint", + "/usr/local/bin/containerd-shim-runc-v2", + "/usr/local/bin/ctr", + "/usr/local/bin/containerd", + "/usr/local/bin/create-kubelet-cgroup-v2", + "/usr/local/bin/crictl", + } +} + +func GetFileURL(filePath string) string { + baseURL := "http://file-server.dragonfly-e2e.svc/kind" + return fmt.Sprintf("%s%s", baseURL, filePath) +} + +func GetNoContentLengthFileURL(filePath string) string { + baseURL := "http://file-server-no-content-length.dragonfly-e2e.svc/kind" + return fmt.Sprintf("%s%s", baseURL, filePath) +} diff --git a/test/e2e/v2/util/task.go b/test/e2e/v2/util/task.go new file mode 100644 index 00000000000..a5c81e577d7 --- /dev/null +++ b/test/e2e/v2/util/task.go @@ -0,0 +1,60 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "errors" + "fmt" + "strings" +) + +const ( + clientContentDir = "/var/lib/dragonfly/content" +) + +type TaskMetadata struct { + ID string + Sha256 string +} + +func CalculateSha256ByTaskID(pods []*PodExec, taskID string) (string, error) { + var sha256sum string + for _, pod := range pods { + contentPath := fmt.Sprintf("%s/%s", clientContentDir, taskID) + if _, err := pod.Command("ls", contentPath).CombinedOutput(); err != nil { + // If the path does not exist, skip this client. + fmt.Printf("path %s does not exist: %s\n", contentPath, err.Error()) + continue + } + + // Calculate sha256sum of the task content. + out, err := pod.Command("sh", "-c", fmt.Sprintf("sha256sum %s", contentPath)).CombinedOutput() + if err != nil { + return "", fmt.Errorf("calculate sha256sum of %s failed: %s", contentPath, err.Error()) + } + + fmt.Println("sha256sum: " + string(out)) + sha256sum = strings.Split(string(out), " ")[0] + break + } + + if sha256sum == "" { + return "", errors.New("can not found sha256sum") + } + + return sha256sum, nil +} diff --git a/test/testdata/charts/config-rs.yaml b/test/testdata/charts/config-v2.yaml similarity index 77% rename from test/testdata/charts/config-rs.yaml rename to test/testdata/charts/config-v2.yaml index d284c8b81bd..79a124fe4df 100644 --- a/test/testdata/charts/config-rs.yaml +++ b/test/testdata/charts/config-v2.yaml @@ -81,9 +81,9 @@ seedClient: path: /tmp/artifact config: dynconfig: - refreshInterval: 1s + refreshInterval: 2s scheduler: - announceInterval: 1s + announceInterval: 2s log: level: info @@ -110,11 +110,27 @@ client: - name: artifact hostPath: path: /tmp/artifact + dfinit: + enable: true + image: + repository: dragonflyoss/dfinit + tag: latest + config: + containerRuntime: + containerd: + configPath: /etc/containerd/config.toml + registries: + - hostNamespace: docker.io + serverAddr: https://index.docker.io + capabilities: ["pull", "resolve"] + - hostNamespace: ghcr.io + serverAddr: https://ghcr.io + capabilities: ["pull", "resolve"] config: dynconfig: - refreshInterval: 1s + refreshInterval: 2s scheduler: - announceInterval: 1s + announceInterval: 2s log: level: info diff --git a/test/testdata/containerd/config-rs.toml b/test/testdata/containerd/config-v2.toml similarity index 73% rename from test/testdata/containerd/config-rs.toml rename to test/testdata/containerd/config-v2.toml index 6f70ce3efaa..980316bdf79 100644 --- a/test/testdata/containerd/config-rs.toml +++ b/test/testdata/containerd/config-v2.toml @@ -28,10 +28,3 @@ version = 2 tolerate_missing_hugepages_controller = true # restrict_oom_score_adj needs to be true when running inside UserNS (rootless) restrict_oom_score_adj = false - -# Mirrors use d7y.io instead of docker.io. -# when using docker.io to generate an endpoint, https://index.docker.io will be returned by default. -# https://github.com/containerd/containerd/blob/main/pkg/cri/server/image_pull.go#L456 -# https://github.com/containerd/containerd/blob/main/remotes/docker/resolver.go#L121 -[plugins."io.containerd.grpc.v1.cri".registry.mirrors."d7y.io"] - endpoint = ["http://127.0.0.1:4001"] diff --git a/test/testdata/k8s/file-server.yaml b/test/testdata/k8s/file-server.yaml index f13d2b48a92..f6123eeb659 100644 --- a/test/testdata/k8s/file-server.yaml +++ b/test/testdata/k8s/file-server.yaml @@ -42,8 +42,7 @@ spec: spec: containers: - name: nginx - # Pull image with dragonfly - image: d7y.io/library/nginx:1.21.1-alpine + image: nginx:1.21.1-alpine imagePullPolicy: "IfNotPresent" ports: - containerPort: 80 diff --git a/test/testdata/kind/config-rs.yaml b/test/testdata/kind/config-v2.yaml similarity index 78% rename from test/testdata/kind/config-rs.yaml rename to test/testdata/kind/config-v2.yaml index f10b075dd46..6b2c5f0c6f9 100644 --- a/test/testdata/kind/config-rs.yaml +++ b/test/testdata/kind/config-v2.yaml @@ -10,8 +10,6 @@ nodes: hostPort: 4001 protocol: TCP extraMounts: - - hostPath: ./test/testdata/containerd/config.toml - containerPath: /etc/containerd/config.toml - hostPath: /tmp/artifact containerPath: /tmp/artifact - hostPath: /dev/fuse