feature: framework of cni

Signed-off-by: YaoZengzeng <[email protected]>

cri/config.go

@@ -0,0 +1,11 @@
+package cri
+
+// Config defines the CRI configuration.
+type Config struct {
+	// Listen is the listening address which servers CRI.
+	Listen string
+	// NetworkPluginBinDir is the directory in which the binaries for the plugin is kept.
+	NetworkPluginBinDir string
+	// NetworkPluginConfDir is the directory in which the admin places a CNI conf.
+	NetworkPluginConfDir string
+}

cri/cri.go → cri/service/cri.go

@@ -1,4 +1,4 @@
-package cri
+package service
 
 import (
 	"net"
@@ -38,11 +38,11 @@ func NewService(cfg config.Config, criMgr mgr.CriMgr) (*Service, error) {
 // Serve starts grpc server.
 func (s *Service) Serve() error {
 	// Unlink to cleanup the previous socket file.
-	if err := syscall.Unlink(s.config.ListenCRI); err != nil && !os.IsNotExist(err) {
+	if err := syscall.Unlink(s.config.CriConfig.Listen); err != nil && !os.IsNotExist(err) {
 		return err
 	}
 
-	l, err := net.Listen("unix", s.config.ListenCRI)
+	l, err := net.Listen("unix", s.config.CriConfig.Listen)
 	if err != nil {
 		return err
 	}

daemon/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"github.com/alibaba/pouch/cri"
 	"github.com/alibaba/pouch/network"
 	"github.com/alibaba/pouch/pkg/utils"
 	"github.com/alibaba/pouch/volume"
@@ -14,12 +15,12 @@ type Config struct {
 	// Network config
 	NetworkConfg network.Config
 
+	// CRI config.
+	CriConfig cri.Config
+
 	// Server listening address.
 	Listen []string
 
-	// ListenCRI is the listening address which serves CRI.
-	ListenCRI string
-
 	// Debug refers to the log mode.
 	Debug bool
 

daemon/daemon.go

@@ -6,7 +6,7 @@ import (
 	"reflect"
 
 	"github.com/alibaba/pouch/apis/server"
-	"github.com/alibaba/pouch/cri"
+	cri "github.com/alibaba/pouch/cri/service"
 	"github.com/alibaba/pouch/ctrd"
 	"github.com/alibaba/pouch/daemon/config"
 	"github.com/alibaba/pouch/daemon/meta"

daemon/mgr/cri.go

@@ -7,9 +7,12 @@ import (
 
 	apitypes "github.com/alibaba/pouch/apis/types"
 	"github.com/alibaba/pouch/ctrd"
+	"github.com/alibaba/pouch/daemon/config"
 	"github.com/alibaba/pouch/pkg/reference"
 	"github.com/alibaba/pouch/version"
 
+	"github.com/sirupsen/logrus"
+
 	// NOTE: "golang.org/x/net/context" is compatible with standard "context" in golang1.7+.
 	"golang.org/x/net/context"
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
@@ -60,15 +63,21 @@ type CriMgr interface {
 type CriManager struct {
 	ContainerMgr ContainerMgr
 	ImageMgr     ImageMgr
+	CniMgr       CniMgr
 }
 
 // NewCriManager creates a brand new cri manager.
-func NewCriManager(ctrMgr ContainerMgr, imgMgr ImageMgr) (*CriManager, error) {
-	c := &CriManager{
+func NewCriManager(config *config.Config, ctrMgr ContainerMgr, imgMgr ImageMgr) (*CriManager, error) {
+	cniMgr, err := NewCniManager(&config.CriConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new cni manager: %v", err)
+	}
+
+	return &CriManager{
 		ContainerMgr: ctrMgr,
 		ImageMgr:     imgMgr,
-	}
-	return c, nil
+		CniMgr:       cniMgr,
+	}, nil
 }
 
 // TODO: Move the underlying functions to their respective files in the future.
@@ -110,16 +119,30 @@ func (c *CriManager) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 	if err != nil {
 		return nil, fmt.Errorf("failed to create a sandbox for pod %q: %v", config.Metadata.Name, err)
 	}
+	id := createResp.ID
 
 	// Step 3: Start the sandbox container.
-	err = c.ContainerMgr.Start(ctx, createResp.ID, "")
+	err = c.ContainerMgr.Start(ctx, id, "")
 	if err != nil {
 		return nil, fmt.Errorf("failed to start sandbox container for pod %q: %v", config.Metadata.Name, err)
 	}
 
-	// TODO: setup networking for the sandbox.
+	// Step 4: Setup networking for the sandbox.
+	container, err := c.ContainerMgr.Get(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+	netnsPath, err := containerNetns(container)
+	if err != nil {
+		return nil, err
+	}
+	err = c.CniMgr.SetUpPodNetwork(config, id, netnsPath)
+	if err != nil {
+		// If setup network failed, don't break now.
+		logrus.Errorf("failed to setup network for sandbox %q: %v", id, err)
+	}
 
-	return &runtime.RunPodSandboxResponse{PodSandboxId: createResp.ID}, nil
+	return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil
 }
 
 // StopPodSandbox stops the sandbox. If there are any running containers in the

daemon/mgr/cri_network.go

@@ -0,0 +1,114 @@
+package mgr
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/alibaba/pouch/cri"
+
+	"github.com/cri-o/ocicni/pkg/ocicni"
+	"github.com/sirupsen/logrus"
+	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// CniMgr as an interface defines all operations against CNI.
+type CniMgr interface {
+	// Name returns the plugin's name. This will be used when searching
+	// for a plugin by name, e.g.
+	Name() string
+
+	// SetUpPodNetwork is the method called after the sandbox container of the
+	// pod has been created but before the other containers of the pod
+	// are launched.
+	SetUpPodNetwork(config *runtime.PodSandboxConfig, id string, netnsPath string) error
+
+	// TearDownPodNetwork is the method called before a pod's sandbox container will be deleted.
+	TearDownPodNetwork(config *runtime.PodSandboxConfig) error
+
+	// GetPodNetworkStatus is the method called to obtain the ipv4 or ipv6 addresses of the pod sandbox.
+	GetPodNetworkStatus(config *runtime.PodSandboxConfig) (string, error)
+
+	// Status returns error if the network plugin is in error state.
+	Status() error
+}
+
+// CniManager is an implementation of interface CniMgr.
+type CniManager struct {
+	// plugin is used to setup and teardown network when run/stop pod sandbox.
+	plugin ocicni.CNIPlugin
+}
+
+// NewCniManager initializes a brand new cni manager.
+func NewCniManager(cfg *cri.Config) (*CniManager, error) {
+	networkPluginBinDir := cfg.NetworkPluginBinDir
+	networkPluginConfDir := cfg.NetworkPluginConfDir
+
+	// Create CNI configuration directory if it doesn't exist to avoid breaking.
+	_, err := os.Stat(networkPluginConfDir)
+	if err != nil && os.IsNotExist(err) {
+		err = os.MkdirAll(networkPluginConfDir, 0666)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create configuration directory for CNI: %v", err)
+		}
+	}
+
+	plugin, err := ocicni.InitCNI(networkPluginConfDir, networkPluginBinDir)
+	if err != nil {
+		return nil, err
+	}
+
+	return &CniManager{
+		plugin: plugin,
+	}, nil
+}
+
+// Name returns the plugin's name. This will be used when searching
+// for a plugin by name, e.g.
+func (c *CniManager) Name() string {
+	return c.plugin.Name()
+}
+
+// SetUpPodNetwork is the method called after the sandbox container of the
+// pod has been created but before the other containers of the pod
+// are launched.
+func (c *CniManager) SetUpPodNetwork(config *runtime.PodSandboxConfig, id string, netnsPath string) error {
+	podNetwork := ocicni.PodNetwork{
+		Name:      config.GetMetadata().GetName(),
+		Namespace: config.GetMetadata().GetNamespace(),
+		ID:        id,
+		NetNS:     netnsPath,
+		// TODO: portmapping.
+	}
+
+	_, err := c.plugin.SetUpPod(podNetwork)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			// Teardown network if an error returned.
+			err := c.plugin.TearDownPod(podNetwork)
+			if err != nil {
+				logrus.Errorf("failed to detroy network for sandbox %q: %v", id, err)
+			}
+		}
+	}()
+
+	return nil
+}
+
+// TearDownPodNetwork is the method called before a pod's sandbox container will be deleted.
+func (c *CniManager) TearDownPodNetwork(config *runtime.PodSandboxConfig) error {
+	return fmt.Errorf("TearDownPodNetwork Not Implemented Yet")
+}
+
+// GetPodNetworkStatus is the method called to obtain the ipv4 or ipv6 addresses of the pod sandbox.
+func (c *CniManager) GetPodNetworkStatus(config *runtime.PodSandboxConfig) (string, error) {
+	return "", fmt.Errorf("GetPodNetworkStatus Not Implemented Yet")
+}
+
+// Status returns error if the network plugin is in error state.
+func (c *CniManager) Status() error {
+	return fmt.Errorf("Status Not Implemented Yet")
+}

daemon/mgr/cri_utils.go

@@ -443,6 +443,17 @@ func filterCRIContainers(containers []*runtime.Container, filter *runtime.Contai
 	return filtered
 }
 
+// containerNetns returns the network namespace of the given container.
+func containerNetns(container *ContainerMeta) (string, error) {
+	pid := container.State.Pid
+	if pid == 0 {
+		// Pouch reports pid 0 for an exited container.
+		return "", fmt.Errorf("can't find network namespace for the terminated container %q", container.ID)
+	}
+
+	return fmt.Sprintf("/proc/%v/ns/net", pid), nil
+}
+
 // Image related tool functions.
 
 // imageToCriImage converts pouch image API to CRI image API.

hack/make.sh

@@ -43,6 +43,21 @@ function install_pouch ()
 	else
 		sh -x $DIR/hack/install_lxcfs_on_centos.sh
 	fi
+
+	# install nsenter
+	echo "Install nsenter"
+	if grep -qi "ubuntu" /etc/issue ; then
+		apt-get install libncurses5-dev libslang2-dev gettext zlib1g-dev libselinux1-dev debhelper lsb-release pkg-config po-debconf autoconf automake autopoint libtool
+		wget https://www.kernel.org/pub/linux/utils/util-linux/v2.24/util-linux-2.24.1.tar.gz -P $TMP
+		tar xf $TMP/util-linux-2.24.1.tar.gz -C $TMP && cd $TMP/util-linux-2.24.1
+		./autogen.sh
+		autoreconf -vfi
+		./configure && make 
+		cp ./nsenter /usr/local/bin
+		cd $DIR/
+	else
+		yum install -y util-linux
+	fi
 }
 
 function target()

internal/generator.go

@@ -47,5 +47,5 @@ func GenNetworkMgr(cfg *config.Config, d DaemonProvider) (mgr.NetworkMgr, error)
 
 // GenCriMgr generates a CriMgr instance.
 func GenCriMgr(d DaemonProvider) (mgr.CriMgr, error) {
-	return mgr.NewCriManager(d.CtrMgr(), d.ImgMgr())
+	return mgr.NewCriManager(d.Config(), d.CtrMgr(), d.ImgMgr())
 }

main.go

@@ -55,7 +55,9 @@ func setupFlags(cmd *cobra.Command) {
 
 	flagSet.StringVar(&cfg.HomeDir, "home-dir", "/var/lib/pouch", "Specify root dir of pouchd")
 	flagSet.StringArrayVarP(&cfg.Listen, "listen", "l", []string{"unix:///var/run/pouchd.sock"}, "Specify listening addresses of Pouchd")
-	flagSet.StringVar(&cfg.ListenCRI, "listen-cri", "/var/run/pouchcri.sock", "Specify listening address of CRI")
+	flagSet.StringVar(&cfg.CriConfig.Listen, "listen-cri", "/var/run/pouchcri.sock", "Specify listening address of CRI")
+	flagSet.StringVar(&cfg.CriConfig.NetworkPluginBinDir, "cni-bin-dir", "/opt/cni/bin", "The directory for putting cni plugin binaries.")
+	flagSet.StringVar(&cfg.CriConfig.NetworkPluginConfDir, "cni-conf-dir", "/etc/cni/net.d", "The directory for putting cni plugin configuration files.")
 	flagSet.BoolVarP(&cfg.Debug, "debug", "D", false, "Switch daemon log level to DEBUG mode")
 	flagSet.StringVarP(&cfg.ContainerdAddr, "containerd", "c", "/var/run/containerd.sock", "Specify listening address of containerd")
 	flagSet.StringVar(&cfg.ContainerdPath, "containerd-path", "", "Specify the path of containerd binary")