Support importing stargz snapshotter as a plugin

This commit introduces `github.com/containerd/stargz-snapshotter/service/plugin` pkg which can be importd as a *builtin* stargz snapshotter plugin. This also adds integration tests for builtin stargz snapshotter. Signed-off-by: Kohei Tokunaga <[email protected]>
wswsmao · Feb 15, 2021 · db7a2f2 · db7a2f2
1 parent bdc018f
commit db7a2f2
Show file tree

Hide file tree

Showing 15 changed files with 404 additions and 120 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -38,13 +38,18 @@ jobs:
       fail-fast: false
       matrix:
         buildargs: ["", "--build-arg=CONTAINERD_VERSION=master"] # released version & master version
+        builtin: ["true", "false"]
+        exclude:
+        - buildargs: ""
+          builtin: "true"
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
     - uses: actions/checkout@v2
     - name: Run integration test
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
+        BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
       run: make integration
 
   test-optimize:
@@ -70,13 +75,18 @@ jobs:
       fail-fast: false
       matrix:
         buildargs: ["", "--build-arg=CONTAINERD_VERSION=master"] # released version & master version
+        builtin: ["true", "false"]
+        exclude:
+        - buildargs: ""
+          builtin: "true"
     steps:
     - name: Install htpasswd for setting up private registry
       run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils
     - uses: actions/checkout@v2
     - name: Run test for pulling image from private registry on Kubernetes
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
+        BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
       run: make test-pullsecrets
 
   test-cri:
@@ -86,11 +96,16 @@ jobs:
       fail-fast: false
       matrix:
         buildargs: ["", "--build-arg=CONTAINERD_VERSION=master"] # released version & master version
+        builtin: ["true", "false"]
+        exclude:
+        - buildargs: ""
+          builtin: "true"
     steps:
     - uses: actions/checkout@v2
     - name: Varidate the runtime through CRI
       env:
         DOCKER_BUILD_ARGS: ${{ matrix.buildargs }}
+        BUILTIN_SNAPSHOTTER: ${{ matrix.builtin }}
       run: make test-cri
 
   #

diff --git a/Dockerfile b/Dockerfile
@@ -30,6 +30,20 @@ RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \
     cd $GOPATH/src/github.com/containerd/containerd && \
     GO111MODULE=off make && DESTDIR=/out/ make install
 
+# Build containerd with builtin stargz snapshotter
+FROM golang-base AS containerd-snapshotter-dev
+ARG CONTAINERD_VERSION
+COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter
+RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \
+    git clone -b ${CONTAINERD_VERSION} --depth 1 \
+              https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \
+    echo 'require github.com/containerd/stargz-snapshotter v0.0.0\nreplace github.com/containerd/stargz-snapshotter => '$GOPATH'/src/github.com/containerd/stargz-snapshotter' \
+      >> $GOPATH/src/github.com/containerd/containerd/go.mod && \
+    echo 'package main \nimport _ "github.com/containerd/stargz-snapshotter/service/plugin"' \
+      > $GOPATH/src/github.com/containerd/containerd/cmd/containerd/builtins_stargz_snapshotter.go && \
+    cd $GOPATH/src/github.com/containerd/containerd && \
+    make vendor && make && DESTDIR=/out/ make install
+
 # Build runc
 FROM golang-base AS runc-dev
 ARG RUNC_VERSION
@@ -73,6 +87,21 @@ FROM containerd-base AS snapshotter-base
 COPY --from=snapshotter-dev /out/* /usr/local/bin/
 RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr
 
+# Base image which contains containerd with builtin stargz snapshotter
+# `docker-ce-cli` is used only for users to `docker login` to registries (e.g. DockerHub)
+# with configuring ~/.docker/config.json
+FROM golang-base AS containerd-snapshotter-base
+RUN apt-get update -y && apt-get --no-install-recommends install -y fuse \
+                                 apt-transport-https gnupg2 software-properties-common && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - && \
+    add-apt-repository \
+      "deb [arch=${TARGETARCH:-amd64}] https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \
+    apt-get update -y && apt-get --no-install-recommends install -y docker-ce-cli
+COPY --from=containerd-snapshotter-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/
+COPY --from=runc-dev /out/sbin/* /usr/local/sbin/
+COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/
+RUN ln -s /usr/local/bin/ctr-remote /usr/local/bin/ctr
+
 # Image which can be used as all-in-one single node demo environment
 FROM snapshotter-base AS cind
 COPY ./script/config/ /
@@ -95,6 +124,14 @@ RUN apt-get update && apt-get install -y iptables && \
     mkdir -p /opt/cni/bin && \
     curl -Ls https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH:-amd64}-${CNI_PLUGINS_VERSION}.tgz | tar xzv -C /opt/cni/bin
 
+# Image which can be used as a node image for KinD (containerd with builtin snapshotter)
+FROM kindest/node:v1.20.0 AS kind-builtin-snapshotter
+COPY --from=containerd-snapshotter-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/
+COPY --from=snapshotter-dev /out/ctr-remote /usr/local/bin/
+COPY ./script/config/ /
+RUN apt-get update -y && apt-get install --no-install-recommends -y fuse
+ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
+
 # Image which can be used as a node image for KinD
 FROM kindest/node:v1.20.0
 COPY --from=containerd-dev /out/bin/containerd /out/bin/containerd-shim-runc-v2 /usr/local/bin/

diff --git a/cmd/containerd-stargz-grpc/main.go b/cmd/containerd-stargz-grpc/main.go
@@ -21,7 +21,6 @@ import (
 	"flag"
 	golog "log"
 	"net"
-	"net/http"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -30,12 +29,7 @@ import (
 	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
 	"github.com/containerd/containerd/contrib/snapshotservice"
 	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/remotes/docker"
-	"github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/keychain"
-	stargzfs "github.com/containerd/stargz-snapshotter/fs"
-	"github.com/containerd/stargz-snapshotter/fs/source"
-	snbase "github.com/containerd/stargz-snapshotter/snapshot"
-	"github.com/hashicorp/go-multierror"
+	"github.com/containerd/stargz-snapshotter/service"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 )
@@ -67,7 +61,7 @@ func main() {
 
 	var (
 		ctx    = log.WithLogger(context.Background(), log.L)
-		config Config
+		config service.Config
 	)
 	// Streams log of standard lib (go-fuse uses this) into debug log
 	// Snapshotter should use "github.com/containerd/containerd/log" otherwize
@@ -79,31 +73,7 @@ func main() {
 		log.G(ctx).WithError(err).Fatalf("failed to load config file %q", *configPath)
 	}
 
-	// Prepare kubeconfig-based keychain if required
-	credsFuncs := []func(string) (string, string, error){keychain.NewDockerconfigKeychain(ctx)}
-	if config.KubeconfigKeychainConfig.EnableKeychain {
-		var opts []keychain.KubeconfigOption
-		if kcp := config.KubeconfigKeychainConfig.KubeconfigPath; kcp != "" {
-			opts = append(opts, keychain.WithKubeconfigPath(kcp))
-		}
-		credsFuncs = append(credsFuncs, keychain.NewKubeconfigKeychain(ctx, opts...))
-	}
-
-	// Use RegistryHosts based on ResolverConfig and keychain
-	hosts := hostsFromConfig(config.ResolverConfig, credsFuncs...)
-
-	// Configure filesystem and snapshotter
-	fs, err := stargzfs.NewFilesystem(filepath.Join(*rootDir, "stargz"),
-		config.Config,
-		stargzfs.WithGetSources(sources(
-			sourceFromCRILabels(hosts),      // provides source info based on CRI labels
-			source.FromDefaultLabels(hosts), // provides source info based on default labels
-		)),
-	)
-	if err != nil {
-		log.G(ctx).WithError(err).Fatalf("failed to configure filesystem")
-	}
-	rs, err := snbase.NewSnapshotter(ctx, filepath.Join(*rootDir, "snapshotter"), fs, snbase.AsynchronousRemove)
+	rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config)
 	if err != nil {
 		log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
 	}
@@ -117,10 +87,10 @@ func main() {
 	rpc := grpc.NewServer()
 
 	// Convert the snapshotter to a gRPC service,
-	service := snapshotservice.FromSnapshotter(rs)
+	snsvc := snapshotservice.FromSnapshotter(rs)
 
 	// Register the service with the gRPC server
-	snapshotsapi.RegisterSnapshotsServer(rpc, service)
+	snapshotsapi.RegisterSnapshotsServer(rpc, snsvc)
 
 	// Prepare the directory for the socket
 	if err := os.MkdirAll(filepath.Dir(*address), 0700); err != nil {
@@ -151,53 +121,3 @@ func waitForSIGINT() {
 	signal.Notify(c, os.Interrupt)
 	<-c
 }
-
-func hostsFromConfig(cfg ResolverConfig, credsFuncs ...func(string) (string, string, error)) docker.RegistryHosts {
-	return func(host string) (hosts []docker.RegistryHost, _ error) {
-		for _, h := range append(cfg.Host[host].Mirrors, MirrorConfig{
-			Host: host,
-		}) {
-			tr := &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}
-			config := docker.RegistryHost{
-				Client:       tr,
-				Host:         h.Host,
-				Scheme:       "https",
-				Path:         "/v2",
-				Capabilities: docker.HostCapabilityPull,
-				Authorizer: docker.NewDockerAuthorizer(
-					docker.WithAuthClient(tr),
-					docker.WithAuthCreds(func(host string) (string, string, error) {
-						for _, f := range credsFuncs {
-							if username, secret, err := f(host); err != nil {
-								return "", "", err
-							} else if !(username == "" && secret == "") {
-								return username, secret, nil
-							}
-						}
-						return "", "", nil
-					})),
-			}
-			if localhost, _ := docker.MatchLocalhost(config.Host); localhost || h.Insecure {
-				config.Scheme = "http"
-			}
-			if config.Host == "docker.io" {
-				config.Host = "registry-1.docker.io"
-			}
-			hosts = append(hosts, config)
-		}
-		return
-	}
-}
-
-func sources(ps ...source.GetSources) source.GetSources {
-	return func(labels map[string]string) (source []source.Source, allErr error) {
-		for _, p := range ps {
-			src, err := p(labels)
-			if err == nil {
-				return src, nil
-			}
-			allErr = multierror.Append(allErr, err)
-		}
-		return
-	}
-}
diff --git a/script/cri/test-stargz.sh b/script/cri/test-stargz.sh
@@ -120,11 +120,19 @@ cat "${IMAGE_LIST}" | sed -E 's/^([^/]*).*/\1/g' | sort | uniq | while read DOMA
 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${DOMAIN}"]
 endpoint = ["http://${REGISTRY_HOST}:5000"]
 EOF
-    cat <<EOF >> "${SNAPSHOTTER_CONFIG}"
+    if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
+        cat <<EOF >> "${CONTAINERD_CONFIG}"
+[[plugins."io.containerd.snapshotter.v1.stargz".resolver.host."${DOMAIN}".mirrors]]
+host = "${REGISTRY_HOST}:5000"
+insecure = true
+EOF
+    else
+        cat <<EOF >> "${SNAPSHOTTER_CONFIG}"
 [[resolver.host."${DOMAIN}".mirrors]]
 host = "${REGISTRY_HOST}:5000"
 insecure = true
 EOF
+    fi
 done
 echo "==== Containerd config ===="
 cat "${CONTAINERD_CONFIG}"
@@ -153,7 +161,9 @@ docker exec "${TEST_NODE_NAME}" /bin/bash -c \
        "cd /go/src/github.com/kubernetes-sigs/cri-tools && make critest && make install-critest -e BINDIR=/go/bin"
 
 # Varidate the runtime through CRI
-docker exec "${TEST_NODE_NAME}" systemctl restart stargz-snapshotter
+if [ "${BUILTIN_SNAPSHOTTER:-}" != "true" ] ; then
+    docker exec "${TEST_NODE_NAME}" systemctl restart stargz-snapshotter
+fi
 docker exec "${TEST_NODE_NAME}" systemctl restart containerd
 CONNECTED=
 for i in $(seq 100) ; do
@@ -174,7 +184,7 @@ docker exec "${TEST_NODE_NAME}" containerd --version
 echo "==============================="
 docker exec "${TEST_NODE_NAME}" /go/bin/critest --runtime-endpoint=${CONTAINERD_SOCK}
 
-# Check if stargz snapshotter is working
+echo "Check if stargz snapshotter is working"
 docker exec "${TEST_NODE_NAME}" \
        ctr-remote --namespace=k8s.io snapshot --snapshotter=stargz ls \
     | sed -E '1d' > "${TMPFILE}"
@@ -183,10 +193,16 @@ if ! [ -s "${TMPFILE}" ] ; then
     exit 1
 fi
 
-# Check all remote snapshots are created successfully
-docker exec "${TEST_NODE_NAME}" journalctl -u stargz-snapshotter \
-    | grep "${LOG_REMOTE_SNAPSHOT}" \
-    | sed -E 's/^[^\{]*(\{.*)$/\1/g' > "${LOG_FILE}"
+echo "Check all remote snapshots are created successfully"
+if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
+    docker exec "${TEST_NODE_NAME}" journalctl -u containerd \
+        | grep "${LOG_REMOTE_SNAPSHOT}" \
+        | sed -E 's/^[^\{]*(\{.*)$/\1/g' > "${LOG_FILE}"
+else
+    docker exec "${TEST_NODE_NAME}" journalctl -u stargz-snapshotter \
+        | grep "${LOG_REMOTE_SNAPSHOT}" \
+        | sed -E 's/^[^\{]*(\{.*)$/\1/g' > "${LOG_FILE}"
+fi
 check_remote_snapshots "${LOG_FILE}"
 
 exit 0
diff --git a/script/cri/test.sh b/script/cri/test.sh
@@ -23,7 +23,13 @@ source "${CONTEXT}/const.sh"
 
 if [ "${CRI_NO_RECREATE:-}" != "true" ] ; then
     echo "Preparing node image..."
-    docker build ${DOCKER_BUILD_ARGS:-} -t "${NODE_BASE_IMAGE_NAME}" "${REPO}"
+
+    TARGET_STAGE=
+    if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
+        TARGET_STAGE="--target kind-builtin-snapshotter"
+    fi
+
+    docker build ${DOCKER_BUILD_ARGS:-} -t "${NODE_BASE_IMAGE_NAME}" ${TARGET_STAGE} "${REPO}"
     docker build ${DOCKER_BUILD_ARGS:-} -t "${PREPARE_NODE_IMAGE}" --target containerd-base "${REPO}"
 fi
 
@@ -37,6 +43,27 @@ function cleanup {
 }
 trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM
 
+BUILTIN_HACK_INST=
+if [ "${BUILTIN_SNAPSHOTTER:-}" == "true" ] ; then
+    # Special configuration for CRI containerd + builtin stargz snapshotter
+    cat <<EOF > "${TMP_CONTEXT}/containerd.hack.toml"
+version = 2
+
+[debug]
+  format = "json"
+  level = "debug"
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  default_runtime_name = "runc"
+  snapshotter = "stargz"
+  disable_snapshot_annotations = false
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler]
+  runtime_type = "io.containerd.runc.v2"
+EOF
+    BUILTIN_HACK_INST="COPY containerd.hack.toml /etc/containerd/config.toml"
+fi
+
 # Prepare the testing node
 cat <<EOF > "${TMP_CONTEXT}/Dockerfile"
 # Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg.
@@ -62,6 +89,8 @@ RUN apt install -y --no-install-recommends git make gcc build-essential jq && \
     NOSUDO=true ./hack/install/install-cni-config.sh && \
     systemctl disable kubelet
 
+${BUILTIN_HACK_INST}
+
 ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
 EOF
 docker build -t "${NODE_TEST_IMAGE_NAME}" ${DOCKER_BUILD_ARGS:-} "${TMP_CONTEXT}"

diff --git a/script/integration/containerd/config.containerd.toml b/script/integration/containerd/config.containerd.toml
@@ -1,4 +1,10 @@
-[proxy_plugins]
-  [proxy_plugins.stargz]
-    type = "snapshot"
-    address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+[plugins.stargz]
+root_path = "/var/lib/containerd-stargz-grpc/"
+disable_verification = false
+
+[plugins.stargz.blob]
+check_always = true
+
+[[plugins.stargz.resolver.host."registry-integration:5000".mirrors]]
+host = "registry-alt:5000"
+insecure = true