Enhance KinD environment (apache#56)

commands/run/run.go

@@ -110,6 +110,7 @@ func doCleanup(stopAction func()) {
 	if stopAction != nil {
 		stopAction()
 	}
+	setup.DoStopSetup()
 	if err := cleanup.DoCleanupAccordingE2E(); err != nil {
 		logger.Log.Errorf("cleanup part error: %s", err)
 	} else {

commands/setup/setup.go

@@ -20,6 +20,7 @@ package setup
 
 import (
 	"fmt"
+	"sync"
 
 	"github.com/apache/skywalking-infra-e2e/internal/components/setup"
 	"github.com/apache/skywalking-infra-e2e/internal/config"
@@ -40,6 +41,15 @@ var Setup = &cobra.Command{
 		if err := DoSetupAccordingE2E(); err != nil {
 			return fmt.Errorf("[Setup] %s", err)
 		}
+
+		if config.GlobalConfig.E2EConfig.Setup.Env == constant.Kind && setup.KindShouldWaitSignal() {
+			wg := sync.WaitGroup{}
+			wg.Add(1)
+			util.AddShutDownHook(wg.Done)
+			wg.Wait()
+
+			setup.KindCleanNotify()
+		}
 		return nil
 	},
 }
@@ -68,3 +78,8 @@ func DoSetupAccordingE2E() error {
 
 	return nil
 }
+
+func DoStopSetup() {
+	// notify clean up
+	setup.KindCleanNotify()
+}

docs/en/setup/Configuration-File.md

@@ -24,24 +24,66 @@ Support two kinds of the environment to set up the system.
 ```yaml
 setup:
   env: kind
-  file: path/to/kind.yaml  # Specified kinD manifest file path
-  timeout: 1200            # timeout second
-  steps:                   # customize steps for prepare the environment
-    - name: customize setups        # step name
+  file: path/to/kind.yaml               # Specified kinD manifest file path
+  timeout: 1200                         # timeout second
+  init-system-environment: path/to/env  # Import environment file
+  steps:                                # customize steps for prepare the environment
+    - name: customize setups            # step name
       # one of command line or kinD manifest file
-      command: command lines        # use command line to setup 
-      path: /path/to/manifest.yaml  # the manifest file path
-      wait:                         # how to verify the manifest is set up finish
-        - namespace:                # The pod namespace
-          resource:                 # The pod resource name
-          label-selector:           # The resource label selector
-          for:                      # The wait condition 
+      command: command lines            # use command line to setup 
+      path: /path/to/manifest.yaml      # the manifest file path
+      wait:                             # how to verify the manifest is set up finish
+        - namespace:                    # The pod namespace
+          resource:                     # The pod resource name
+          label-selector:               # The resource label selector
+          for:                          # The wait condition
+  kind:
+     import-images:                     # import docker images to KinD
+        - image:version                 # support using env to expand image, such as `${env_key}` or `$env_key`
+     expose-ports:                      # Expose resource for host access
+        - namespace:                    # The resource namespace
+          resource:                     # The resource name, such as `pod/foo` or `service/foo`
+          port:                         # Want to expose port from resource
 ```
 
 The `KinD` environment follow these steps:
-1. Start the `KinD` cluster according to the config file.
+1. Start the `KinD` cluster according to the config file, expose `KUBECONFIG` to environment for help execute `kubectl` in the steps.
+1. Load docker images from `kind.import-images` if needed.
 1. Apply the resources files (`--manifests`) or/and run the custom init command (`--commands`) by steps.
 1. Wait until all steps are finished and all services are ready with the timeout(second).
+1. Expose all resource ports for host access.
+
+#### Import docker image
+
+If you want to import docker image from private registries, there are several ways to do this:
+1. Using `imagePullSecrets` to pull images, [please take reference from document](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#registry-secret-existing-credentials).
+2. Using `kind.import-images` to load images from host.
+   ```yaml
+   kind:
+      import-images:
+        - skywalking/oap:${OAP_HASH} # support using environment to expand the image name
+   ```
+
+#### Resource Export
+
+If you want to access the resource from host, should follow these steps:
+1. Declare which resource and ports need to be accessible from host.
+   ```yaml
+   setup:
+      kind:
+         expose-ports:
+           - namespace: default  # Need to expose resource namespace
+             resource: pod/foo   # Resource description, such as `pod/foo` or `service/foo`
+             port: 8080          # Resource port want to expose, support `<resource_port>`, `<bind_to_host_port>:<resource_port>`
+   ```
+2. Follow this format to get the host and port mapping by the environment, and it's available in steps(trigger, verify).
+   ```yaml
+   trigger:
+      # trigger with specified mapped port, the resource name replace all `/` or `-` as `_`
+      # host format: <resource_name>_host
+      # port format: <resource_name>_<container_port>
+      url: http://${pod_foo_host}:${pod_foo_8080}/
+   ```
 
 ### Compose
 
@@ -78,7 +120,7 @@ If you want to get the service host and port mapping, should follow these steps:
    ```yaml
    trigger:
       # trigger with specified mappinged port
-      url: http://${oap.host}:${oap_8080}/
+      url: http://${oap_host}:${oap_8080}/
    ```
 
 ## Trigger

examples/kind/e2e.yaml

@@ -42,6 +42,11 @@ setup:
         - namespace: default
           resource: deployment/nginx1
           for: condition=Available
+  kind:
+    expose-ports:
+      - namespace: default
+        resource: deployment/nginx1
+        port: 80
   timeout: 1200
 
 cleanup:
@@ -52,7 +57,7 @@ trigger:
   action: http
   interval: 3s
   times: 5
-  url: http://localhost:9090/user
+  url: http://${deployment_nginx1_host}:${deployment_nginx1_80}/
   method: GET
 
 verify:
@@ -67,7 +72,3 @@ verify:
       expected: ../../test/verify/1.expected.yaml
     - actual: ../../test/verify/2.actual.yaml
       expected: ../../test/verify/2.expected.yaml
-    - actual: ../../test/verify/1.actual.yaml
-      expected: ../../test/verify/2.expected.yaml
-    - query: swctl --display yaml service ls
-      expected: ../../test/verify/3.expected.yaml

go.sum

@@ -247,6 +247,7 @@ github.com/evanphx/json-patch/v5 v5.1.0 h1:B0aXl1o/1cP8NbviYiBMkcHBtUjIJ1/Ccg6b+
 github.com/evanphx/json-patch/v5 v5.1.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=

internal/components/cleanup/kind.go

@@ -21,15 +21,13 @@ package cleanup
 import (
 	"io/ioutil"
 	"os"
-
-	"github.com/apache/skywalking-infra-e2e/internal/config"
+	"strings"
 
 	"gopkg.in/yaml.v2"
 	kind "sigs.k8s.io/kind/cmd/kind/app"
 	kindcmd "sigs.k8s.io/kind/pkg/cmd"
 
-	"strings"
-
+	"github.com/apache/skywalking-infra-e2e/internal/config"
 	"github.com/apache/skywalking-infra-e2e/internal/constant"
 	"github.com/apache/skywalking-infra-e2e/internal/logger"
 )