chore: support multi daemons e2e test (dragonflyoss#896)

Signed-off-by: Jim Ma <[email protected]>

.github/workflows/compatibility-e2e.yml

@@ -17,6 +17,7 @@ env:
   DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly
   DRAGONFLY_CHARTS_CONFIG_PATH: test/testdata/charts/config.yaml
   DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml
+  DRAGONFLY_PROXY_SERVER_PATH: test/testdata/k8s/proxy.yaml
 
 jobs:
   compatibility_e2e_tests:
@@ -51,7 +52,7 @@ jobs:
 
       - name: Build images
         run: |
-          make docker-build
+          make docker-build docker-build-testing-tools
           docker pull dragonflyoss/${{ matrix.module }}:${{ env.DRAGONFLY_STABLE_IMAGE_TAG }}
 
       - name: Prepare kind environment
@@ -63,7 +64,12 @@ jobs:
         run: |
           helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system --set ${{ matrix.module }}.tag=${{ env.DRAGONFLY_STABLE_IMAGE_TAG }} --set ${{ matrix.module }}.image=dragonflyoss/${{ matrix.module }} -f ${{ env.DRAGONFLY_CHARTS_CONFIG_PATH }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }}
           kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }}
+          kubectl apply -f ${{ env.DRAGONFLY_PROXY_SERVER_PATH }}
           kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl wait po file-server-no-content-length-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl wait po proxy-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl wait po proxy-1 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl wait po proxy-2 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
 
       - name: Run Compatibility E2E test
         env:

.github/workflows/e2e.yml

@@ -16,6 +16,7 @@ env:
   DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly
   DRAGONFLY_CHARTS_CONFIG_PATH: test/testdata/charts/config.yaml
   DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml
+  DRAGONFLY_PROXY_SERVER_PATH: test/testdata/k8s/proxy.yaml
 
 jobs:
   e2e_tests:
@@ -45,7 +46,7 @@ jobs:
           config: ${{ env.KIND_CONFIG_PATH }}
 
       - name: Build images
-        run: make docker-build
+        run: make docker-build docker-build-testing-tools
 
       - name: Prepare kind environment
         run: make kind-load
@@ -54,7 +55,12 @@ jobs:
         run: |
           helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ env.DRAGONFLY_CHARTS_CONFIG_PATH }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }}
           kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }}
+          kubectl apply -f ${{ env.DRAGONFLY_PROXY_SERVER_PATH }}
           kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl wait po file-server-no-content-length-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl wait po proxy-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl wait po proxy-1 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl wait po proxy-2 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
 
       - name: Run E2E test
         run: make actions-e2e-test-coverage

Makefile

@@ -62,6 +62,12 @@ docker-build-manager:
 	./hack/docker-build.sh manager
 .PHONY: docker-build-manager
 
+# Build testing tools image
+docker-build-testing-tools: build-dirs
+	@echo "Begin to testing tools image."
+	./test/tools/no-content-length/build.sh
+.PHONY: docker-build-testing-tools
+
 # Push cdn image
 docker-push-cdn: docker-build-cdn
 	@echo "Begin to push cdn docker image."
@@ -218,9 +224,9 @@ clean-e2e-test:
 .PHONY: clean-e2e-test
 
 # Kind load dragonlfy
-kind-load: kind-load-cdn kind-load-scheduler kind-load-dfdaemon kind-load-manager
+kind-load: kind-load-cdn kind-load-scheduler kind-load-dfdaemon kind-load-manager kind-load-testing-tools
 	@echo "Kind load image done."
-.PHONY: docker-build
+.PHONY: kind-load
 
 # Run kind load docker-image cdn
 kind-load-cdn:
@@ -242,6 +248,11 @@ kind-load-manager:
 	@./hack/kind-load.sh manager
 .PHONY: kind-load-manager
 
+# Run kind load docker testing tools
+kind-load-testing-tools:
+	@./hack/kind-load.sh no-content-length
+.PHONY: kind-load-testing-tools
+
 # Run code lint
 lint: markdownlint
 	@echo "Begin to golangci-lint."
@@ -302,11 +313,12 @@ help:
 	@echo "make e2e-test                       run e2e tests"
 	@echo "make e2e-test-coverage              run e2e tests with coverage"
 	@echo "make clean-e2e-test                 clean e2e tests"
-	@echo "make kind-load-image                kind load docker image"
+	@echo "make kind-load                      kind load docker image"
 	@echo "make kind-load-cdn                  kind load cdn docker image"
 	@echo "make kind-load-scheduler            kind load scheduler docker image"
 	@echo "make kind-load-dfdaemon             kind load dfdaemon docker image"
 	@echo "make kind-load-manager              kind load manager docker image"
+	@echo "make kind-load-testing-tools        kind load testing tools docker image"
 	@echo "make lint                           run code lint"
 	@echo "make markdownlint                   run markdown lint"
 	@echo "make swag                           generate swagger api docs"

client/daemon/peer/peertask_stream.go

@@ -379,15 +379,7 @@ func (s *streamPeerTask) backSource() {
 	defer backSourceSpan.End()
 	s.contentLength.Store(-1)
 	_ = s.callback.Init(s)
-	reportPieceCtx, reportPieceSpan := tracer.Start(backSourceCtx, config.SpanReportPieceResult)
-	defer reportPieceSpan.End()
-	if peerPacketStream, err := s.schedulerClient.ReportPieceResult(reportPieceCtx, s.taskID, s.request); err != nil {
-		logger.Errorf("step 2: peer %s report piece failed: err", s.request.PeerId, err)
-	} else {
-		s.peerPacketStream = peerPacketStream
-	}
-	logger.Infof("step 2: start report peer %s back source piece result", s.request.PeerId)
-	err := s.pieceManager.DownloadSource(s.ctx, s, s.request)
+	err := s.pieceManager.DownloadSource(backSourceCtx, s, s.request)
 	if err != nil {
 		s.Errorf("download from source error: %s", err)
 		s.failedReason = err.Error()

hack/install-e2e-test.sh

@@ -7,6 +7,7 @@ set -o pipefail
 KIND_CONFIG_PATH="test/testdata/kind/config.yaml"
 CHARTS_CONFIG_PATH="test/testdata/charts/config.yaml"
 FILE_SERVER_CONFIG_PATH="test/testdata/k8s/file-server.yaml"
+PROXY_SERVER_CONFIG_PATH="test/testdata/k8s/proxy.yaml"
 CHARTS_PATH="deploy/helm-charts/charts/dragonfly"
 NAMESPACE="dragonfly-system"
 E2E_NAMESPACE="dragonfly-e2e"
@@ -39,8 +40,24 @@ install-helm() {
 install-file-server() {
   kubectl apply -f ${FILE_SERVER_CONFIG_PATH}
   kubectl wait --namespace ${E2E_NAMESPACE} \
-  --for=condition=ready pod ${FILE_SERVER_NAME} \
-  --timeout=10m
+    --for=condition=ready pod ${FILE_SERVER_NAME} \
+    --timeout=10m
+  kubectl wait --namespace ${E2E_NAMESPACE} \
+    --for=condition=ready pod file-server-no-content-length-0 \
+    --timeout=10m
+}
+
+install-proxy-server() {
+  kubectl apply -f ${PROXY_SERVER_CONFIG_PATH}
+  kubectl wait --namespace ${E2E_NAMESPACE} \
+    --for=condition=ready pod proxy-0 \
+    --timeout=10m
+  kubectl wait --namespace ${E2E_NAMESPACE} \
+    --for=condition=ready pod proxy-1 \
+    --timeout=10m
+  kubectl wait --namespace ${E2E_NAMESPACE} \
+    --for=condition=ready pod proxy-2 \
+    --timeout=10m
 }
 
 install-ginkgo() {
@@ -72,7 +89,7 @@ main() {
   install-kind
 
   print_step_info "start building docker images"
-  make docker-build
+  make docker-build docker-build-testing-tools
 
   print_step_info "start loading image for kind"
   make kind-load
@@ -83,6 +100,9 @@ main() {
   print_step_info "start install file server"
   install-file-server
 
+  print_step_info "start install proxy server"
+  install-proxy-server
+
   print_step_info "start install ginkgo"
   install-ginkgo
 

hack/kind-load.sh

@@ -24,6 +24,9 @@ main() {
         ;;
     manager)
         kind-load manager
+        ;;
+    no-content-length)
+        kind-load no-content-length
     esac
 }
 

test/e2e/dfget_test.go

@@ -19,47 +19,96 @@ package e2e
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo" //nolint
 	. "github.com/onsi/gomega" //nolint
 
 	"d7y.io/dragonfly/v2/test/e2e/e2eutil"
 )
 
-var _ = Describe("Download with dfget", func() {
+var _ = Describe("Download with dfget and proxy", func() {
 	Context("dfget", func() {
-		It("dfget download should be ok", func() {
-			out, err := e2eutil.KubeCtlCommand("-n", dragonflyNamespace, "get", "pod", "-l", "component=dfdaemon",
-				"-o", "jsonpath='{range .items[*]}{.metadata.name}{end}'").CombinedOutput()
-			podName := strings.Trim(string(out), "'")
-			Expect(err).NotTo(HaveOccurred())
-			fmt.Println(podName)
-			Expect(strings.HasPrefix(podName, "dragonfly-dfdaemon-")).Should(BeTrue())
-			pod := e2eutil.NewPodExec(dragonflyNamespace, podName, "dfdaemon")
+		singleDfgetTest("dfget daemon download should be ok",
+			dragonflyNamespace, "component=dfdaemon",
+			"dragonfly-dfdaemon-", "dfdaemon")
+		for i := 0; i < 3; i++ {
+			singleDfgetTest(
+				fmt.Sprintf("dfget daemon proxy-%d should be ok", i),
+				dragonflyE2ENamespace,
+				fmt.Sprintf("statefulset.kubernetes.io/pod-name=proxy-%d", i),
+				"proxy-", "proxy")
+		}
+	})
+})
 
-			for _, v := range e2eutil.GetFileList() {
-				url := e2eutil.GetFileURL(v)
-				fmt.Println("download url " + url)
+func singleDfgetTest(name, ns, label, podNamePrefix, container string) {
+	It(name, func() {
+		out, err := e2eutil.KubeCtlCommand("-n", ns, "get", "pod", "-l", label,
+			"-o", "jsonpath='{range .items[*]}{.metadata.name}{end}'").CombinedOutput()
+		podName := strings.Trim(string(out), "'")
+		Expect(err).NotTo(HaveOccurred())
+		fmt.Println("test in pod: " + podName)
+		Expect(strings.HasPrefix(podName, podNamePrefix)).Should(BeTrue())
+		pod := e2eutil.NewPodExec(ns, podName, container)
 
-				// get original file digest
-				out, err = e2eutil.DockerCommand("sha256sum", v).CombinedOutput()
-				fmt.Println(string(out))
-				Expect(err).NotTo(HaveOccurred())
-				sha256sum1 := strings.Split(string(out), " ")[0]
+		var urls = map[string]string{}
+		for _, v := range e2eutil.GetFileList() {
+			urls[e2eutil.GetFileURL(v)] = v
+			urls[e2eutil.GetNoContentLengthFileURL(v)] = v
+		}
 
-				// download file
-				out, err = pod.Command("dfget", "-O", "/tmp/d7y.out", url).CombinedOutput()
-				fmt.Println(string(out))
-				Expect(err).NotTo(HaveOccurred())
+		for url, path := range urls {
+			fmt.Println("download url: " + url)
+			// get original file digest
+			out, err = e2eutil.DockerCommand("sha256sum", path).CombinedOutput()
+			fmt.Println("original sha256sum: " + string(out))
+			Expect(err).NotTo(HaveOccurred())
+			sha256sum1 := strings.Split(string(out), " ")[0]
 
-				// get downloaded file digest
-				out, err = pod.Command("sha256sum", "/tmp/d7y.out").CombinedOutput()
-				fmt.Println(string(out))
-				Expect(err).NotTo(HaveOccurred())
-				sha256sum2 := strings.Split(string(out), " ")[0]
+			var (
+				start time.Time
+				end   time.Time
+			)
+			// download file via dfget
+			start = time.Now()
+			out, err = pod.Command("dfget", "-O", "/tmp/d7y.out", url).CombinedOutput()
+			end = time.Now()
+			fmt.Println(string(out))
+			Expect(err).NotTo(HaveOccurred())
+
+			// get dfget downloaded file digest
+			out, err = pod.Command("sha256sum", "/tmp/d7y.out").CombinedOutput()
+			fmt.Println("dfget sha256sum: " + string(out))
+			Expect(err).NotTo(HaveOccurred())
+			sha256sum2 := strings.Split(string(out), " ")[0]
+			Expect(sha256sum1).To(Equal(sha256sum2))
 
-				Expect(sha256sum1).To(Equal(sha256sum2))
+			// slow download
+			Expect(end.Sub(start).Seconds() < 30.0).To(Equal(true))
+
+			// skip dfdaemon
+			if ns == dragonflyNamespace {
+				continue
 			}
-		})
+			// download file via proxy
+			start = time.Now()
+			out, err = pod.Command("sh", "-c", fmt.Sprintf(`
+              export http_proxy=http://127.0.0.1:65001
+              wget -O /tmp/wget.out %s`, url)).CombinedOutput()
+			end = time.Now()
+			fmt.Println(string(out))
+			Expect(err).NotTo(HaveOccurred())
+
+			// get proxy downloaded file digest
+			out, err = pod.Command("sha256sum", "/tmp/wget.out").CombinedOutput()
+			fmt.Println("wget sha256sum: " + string(out))
+			Expect(err).NotTo(HaveOccurred())
+			sha256sum3 := strings.Split(string(out), " ")[0]
+			Expect(sha256sum1).To(Equal(sha256sum3))
+
+			// slow download
+			Expect(end.Sub(start).Seconds() < 30.0).To(Equal(true))
+		}
 	})
-})
+}

test/e2e/e2e_test.go

@@ -30,9 +30,10 @@ import (
 )
 
 const (
-	proxy              = "localhost:65001"
-	hostnameFilePath   = "/etc/hostname"
-	dragonflyNamespace = "dragonfly-system"
+	proxy                 = "localhost:65001"
+	hostnameFilePath      = "/etc/hostname"
+	dragonflyNamespace    = "dragonfly-system"
+	dragonflyE2ENamespace = "dragonfly-e2e"
 )
 
 const (

test/e2e/e2eutil/file_server.go

@@ -40,3 +40,8 @@ func GetFileURL(filePath string) string {
 	baseURL := "http://file-server.dragonfly-e2e.svc/kind"
 	return fmt.Sprintf("%s%s", baseURL, filePath)
 }
+
+func GetNoContentLengthFileURL(filePath string) string {
+	baseURL := "http://file-server-no-content-length.dragonfly-e2e.svc/kind"
+	return fmt.Sprintf("%s%s", baseURL, filePath)
+}

test/e2e/manager/preheat.go

@@ -48,11 +48,11 @@ var _ = Describe("Preheat with manager", func() {
 
 			for _, v := range e2eutil.GetFileList() {
 				url := e2eutil.GetFileURL(v)
-				fmt.Println("download url " + url)
+				fmt.Println("download url: " + url)
 
 				// get original file digest
 				out, err := e2eutil.DockerCommand("sha256sum", v).CombinedOutput()
-				fmt.Println(string(out))
+				fmt.Println("original sha256sum: " + string(out))
 				Expect(err).NotTo(HaveOccurred())
 				sha256sum1 := strings.Split(string(out), " ")[0]
 
@@ -92,7 +92,7 @@ var _ = Describe("Preheat with manager", func() {
 
 		It("preheat image should be ok", func() {
 			url := "https://registry-1.docker.io/v2/library/alpine/manifests/3.14"
-			fmt.Println("download image " + url)
+			fmt.Println("download image: " + url)
 
 			var (
 				cdnTaskIDs = []string{
@@ -145,7 +145,7 @@ var _ = Describe("Preheat with manager", func() {
 		It("concurrency 100 preheat should be ok", func() {
 			// generate the data file
 			url := e2eutil.GetFileURL(hostnameFilePath)
-			fmt.Println("download url " + url)
+			fmt.Println("download url: " + url)
 			dataFilePath := "post_data"
 			fd, err := os.Create(dataFilePath)
 			Expect(err).NotTo(HaveOccurred())
@@ -162,7 +162,7 @@ var _ = Describe("Preheat with manager", func() {
 
 			// get original file digest
 			out, err = e2eutil.DockerCommand("sha256sum", hostnameFilePath).CombinedOutput()
-			fmt.Println(string(out))
+			fmt.Println("original sha256sum: " + string(out))
 			Expect(err).NotTo(HaveOccurred())
 			sha256sum1 := strings.Split(string(out), " ")[0]
 
@@ -259,10 +259,9 @@ func checkPreheatResult(cdnPods [3]*e2eutil.PodExec, cdnTaskID string) string {
 
 		// calculate digest of downloaded file
 		out, err = cdn.Command("sha256sum", fmt.Sprintf("%s/%s/%s", cdnCachePath, dir, file)).CombinedOutput()
-		fmt.Println(string(out))
+		fmt.Println("preheat sha256sum: " + string(out))
 		Expect(err).NotTo(HaveOccurred())
 		sha256sum2 = strings.Split(string(out), " ")[0]
-		fmt.Println(string(sha256sum2))
 		break
 	}
 	return sha256sum2