feature: cri support update runtime config

pouch write the runtime-config to a file

Signed-off-by: zhuangqh <[email protected]>

cri/config/config.go

@@ -25,4 +25,6 @@ type Config struct {
 	CriStatsCollectPeriod int `json:"cri-stats-collect-period,omitempty"`
 	// EnableCriStatsCollect specify whether cri collect stats from containerd.
 	EnableCriStatsCollect bool `json:"enable-cri-stats-collect,omitempty"`
+	// RuntimeConfigFile is a file to make the runtime config persistent.
+	RuntimeConfigFile string `json:"runtime-config-file"`
 }

cri/ocicni/cni_manager.go

@@ -1,8 +1,12 @@
 package ocicni
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net"
 	"os"
+	"sync"
 
 	"github.com/alibaba/pouch/cri/config"
 
@@ -14,23 +18,37 @@ import (
 
 // CniManager is an implementation of interface CniMgr.
 type CniManager struct {
+	sync.RWMutex
 	// plugin is used to setup and teardown network when run/stop pod sandbox.
 	plugin ocicni.CNIPlugin
+	// runtimeConfigFile is a file to make the runtime config persistent.
+	runtimeConfigFile string
+	// defaultRuntimeConfig is configuration specific to the default pod network interface.
+	defaultRuntimeConfig ocicni.RuntimeConfig
 }
 
 // NewCniManager initializes a brand new cni manager.
-// If initialize failed, return NoopCniManager, we should not make pouchd creashed
-// because of the failure of cni manager.
 func NewCniManager(cfg *config.Config) (CniMgr, error) {
 	networkPluginBinDir := cfg.NetworkPluginBinDir
 	networkPluginConfDir := cfg.NetworkPluginConfDir
 
 	// Create CNI configuration directory if it doesn't exist to avoid breaking.
-	_, err := os.Stat(networkPluginConfDir)
-	if err != nil && os.IsNotExist(err) {
-		err = os.MkdirAll(networkPluginConfDir, 0666)
+	if err := os.MkdirAll(networkPluginConfDir, 0755); err != nil {
+		return nil, err
+	}
+
+	// load runtime config
+	runtimeConfig := ocicni.RuntimeConfig{}
+
+	data, err := ioutil.ReadFile(cfg.RuntimeConfigFile)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("failed to open runtime config file %s: %v", cfg.RuntimeConfigFile, err)
+	}
+
+	if len(data) > 0 {
+		err = json.Unmarshal(data, &runtimeConfig)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to read from runtime config file %s: %v", cfg.RuntimeConfigFile, err)
 		}
 	}
 
@@ -41,7 +59,9 @@ func NewCniManager(cfg *config.Config) (CniMgr, error) {
 	}
 
 	return &CniManager{
-		plugin: plugin,
+		plugin:               plugin,
+		defaultRuntimeConfig: runtimeConfig,
+		runtimeConfigFile:    cfg.RuntimeConfigFile,
 	}, nil
 }
 
@@ -51,10 +71,19 @@ func (c *CniManager) Name() string {
 	return c.plugin.Name()
 }
 
+// GetDefaultNetworkName returns the name of the plugin's default network.
+func (c *CniManager) GetDefaultNetworkName() string {
+	return c.plugin.GetDefaultNetworkName()
+}
+
 // SetUpPodNetwork is the method called after the sandbox container of the
 // pod has been created but before the other containers of the pod
 // are launched.
 func (c *CniManager) SetUpPodNetwork(podNetwork *ocicni.PodNetwork) error {
+	c.RLock()
+	c.updateDefaultRuntimeConfig(podNetwork)
+	c.RUnlock()
+
 	_, err := c.plugin.SetUpPod(*podNetwork)
 
 	defer func() {
@@ -74,8 +103,36 @@ func (c *CniManager) SetUpPodNetwork(podNetwork *ocicni.PodNetwork) error {
 	return nil
 }
 
+// updateDefaultRuntimeConfig set some config of the pod default network interface.
+// only set podCIDR now.
+func (c *CniManager) updateDefaultRuntimeConfig(podNetwork *ocicni.PodNetwork) {
+	if len(c.defaultRuntimeConfig.IpRanges) == 0 {
+		return
+	}
+
+	if podNetwork.RuntimeConfig == nil {
+		podNetwork.RuntimeConfig = make(map[string]ocicni.RuntimeConfig)
+	}
+
+	defaultNetworkName := c.GetDefaultNetworkName()
+
+	if _, exist := podNetwork.RuntimeConfig[defaultNetworkName]; !exist {
+		podNetwork.RuntimeConfig[defaultNetworkName] = ocicni.RuntimeConfig{}
+	}
+
+	if len(podNetwork.RuntimeConfig[defaultNetworkName].IpRanges) == 0 {
+		conf := podNetwork.RuntimeConfig[defaultNetworkName]
+		conf.IpRanges = c.defaultRuntimeConfig.IpRanges
+		podNetwork.RuntimeConfig[defaultNetworkName] = conf
+	}
+}
+
 // TearDownPodNetwork is the method called before a pod's sandbox container will be deleted.
 func (c *CniManager) TearDownPodNetwork(podNetwork *ocicni.PodNetwork) error {
+	c.RLock()
+	c.updateDefaultRuntimeConfig(podNetwork)
+	c.RUnlock()
+
 	// perform the teardown network operation whatever to
 	// give CNI Plugin a chance to perform some operations
 	err := c.plugin.TearDownPod(*podNetwork)
@@ -128,3 +185,54 @@ func (c *CniManager) GetPodNetworkStatus(netnsPath string) (string, error) {
 func (c *CniManager) Status() error {
 	return c.plugin.Status()
 }
+
+const (
+	// CNIChangeEventPodCIDR is a subject type to update the podCIDR for the default pod network.
+	CNIChangeEventPodCIDR = "pod-cidr-change"
+)
+
+// Event handle the changes of CNI.
+// only support updatePodCIDR now.
+func (c *CniManager) Event(subject string, detail interface{}) error {
+	c.Lock()
+	defer c.Unlock()
+
+	var err error
+	switch subject {
+	case CNIChangeEventPodCIDR:
+		err = c.updatePodCIDR(detail)
+	default:
+		err = fmt.Errorf("unknown event subject: %s", subject)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	// save the runtime config
+	data, err := json.Marshal(c.defaultRuntimeConfig)
+	if err != nil {
+		return fmt.Errorf("failed to marshal runtime config: %v", err)
+	}
+
+	if err := ioutil.WriteFile(c.runtimeConfigFile, data, 0644); err != nil {
+		return fmt.Errorf("failed to write config to file %s: %v", c.runtimeConfigFile, err)
+	}
+
+	return nil
+}
+
+func (c *CniManager) updatePodCIDR(val interface{}) error {
+	podCIDR, ok := val.(string)
+	if !ok {
+		return fmt.Errorf("podCIDR expected string type")
+	}
+
+	if _, _, err := net.ParseCIDR(podCIDR); err != nil {
+		return fmt.Errorf("\"%s\" is not a valid CIDR value", podCIDR)
+	}
+
+	c.defaultRuntimeConfig.IpRanges = [][]ocicni.IpRange{{{Subnet: podCIDR}}}
+
+	return nil
+}

cri/ocicni/interface.go

@@ -8,6 +8,10 @@ type CniMgr interface {
 	// for a plugin by name, e.g.
 	Name() string
 
+	// GetDefaultNetworkName returns the name of the plugin's default
+	// network.
+	GetDefaultNetworkName() string
+
 	// SetUpPodNetwork is the method called after the sandbox container of the
 	// pod has been created but before the other containers of the pod
 	// are launched.
@@ -22,6 +26,9 @@ type CniMgr interface {
 	// Status returns error if the network plugin is in error state.
 	Status() error
 
+	// Event handle the changes of CNI.
+	Event(subject string, detail interface{}) error
+
 	// NewNetNS creates a new persistent network namespace and returns the
 	// namespace path, without switching to it
 	NewNetNS() (string, error)

cri/v1alpha2/cri.go

@@ -1203,7 +1203,18 @@ func (c *CriManager) PortForward(ctx context.Context, r *runtime.PortForwardRequ
 
 // UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates.
 func (c *CriManager) UpdateRuntimeConfig(ctx context.Context, r *runtime.UpdateRuntimeConfigRequest) (*runtime.UpdateRuntimeConfigResponse, error) {
-	return nil, fmt.Errorf("UpdateRuntimeConfig Not Implemented Yet")
+	podCIDR := r.GetRuntimeConfig().GetNetworkConfig().GetPodCidr()
+	if podCIDR == "" {
+		return &runtime.UpdateRuntimeConfigResponse{}, nil
+	}
+
+	if err := c.CniMgr.Event(cni.CNIChangeEventPodCIDR, podCIDR); err != nil {
+		return nil, fmt.Errorf("failed to update podCIDR: %v", err)
+	}
+
+	logrus.Infof("CNI default PodCIDR change to \"%v\"", podCIDR)
+
+	return &runtime.UpdateRuntimeConfigResponse{}, nil
 }
 
 // Status returns the status of the runtime.

cri/v1alpha2/cri_utils.go

@@ -552,23 +552,31 @@ func setupSandboxFiles(sandboxRootDir string, config *runtime.PodSandboxConfig)
 // and do nothing when networkNamespaceMode equals runtime.NamespaceMode_NODE.
 func (c *CriManager) setupPodNetwork(id, netnsPath string, config *runtime.PodSandboxConfig) error {
 	return c.CniMgr.SetUpPodNetwork(&ocicni.PodNetwork{
-		Name:         config.GetMetadata().GetName(),
-		Namespace:    config.GetMetadata().GetNamespace(),
-		ID:           id,
-		NetNS:        netnsPath,
-		PortMappings: toCNIPortMappings(config.GetPortMappings()),
+		Name:      config.GetMetadata().GetName(),
+		Namespace: config.GetMetadata().GetNamespace(),
+		ID:        id,
+		NetNS:     netnsPath,
+		RuntimeConfig: map[string]ocicni.RuntimeConfig{
+			c.CniMgr.GetDefaultNetworkName(): {
+				PortMappings: toCNIPortMappings(config.GetPortMappings()),
+			},
+		},
 	})
 }
 
 // teardownNetwork teardown the network of PodSandbox.
 // and do nothing when networkNamespaceMode equals runtime.NamespaceMode_NODE.
 func (c *CriManager) teardownNetwork(id, netnsPath string, config *runtime.PodSandboxConfig) error {
 	return c.CniMgr.TearDownPodNetwork(&ocicni.PodNetwork{
-		Name:         config.GetMetadata().GetName(),
-		Namespace:    config.GetMetadata().GetNamespace(),
-		ID:           id,
-		NetNS:        netnsPath,
-		PortMappings: toCNIPortMappings(config.GetPortMappings()),
+		Name:      config.GetMetadata().GetName(),
+		Namespace: config.GetMetadata().GetNamespace(),
+		ID:        id,
+		NetNS:     netnsPath,
+		RuntimeConfig: map[string]ocicni.RuntimeConfig{
+			c.CniMgr.GetDefaultNetworkName(): {
+				PortMappings: toCNIPortMappings(config.GetPortMappings()),
+			},
+		},
 	})
 }
 

main.go

@@ -84,6 +84,7 @@ func setupFlags(cmd *cobra.Command) {
 	flagSet.BoolVar(&cfg.CriConfig.StreamServerReusePort, "stream-server-reuse-port", false, "Specify whether cri stream server share port with pouchd. If this is true, the listen option of pouchd should specify a tcp socket and its port should be same with stream-server-port.")
 	flagSet.IntVar(&cfg.CriConfig.CriStatsCollectPeriod, "cri-stats-collect-period", 10, "The time duration (in time.Second) cri collect stats from containerd.")
 	flagSet.BoolVar(&cfg.CriConfig.EnableCriStatsCollect, "enable-cri-stats-collect", false, "Specify whether cri collect stats from containerd. If this is true, option CriStatsCollectPeriod will take effect.")
+	flagSet.StringVar(&cfg.CriConfig.RuntimeConfigFile, "cni-runtime-config", "/etc/pouch/cni-runtime-config.json", "A config file to make the cni runtime config persistent.")
 	flagSet.BoolVarP(&cfg.Debug, "debug", "D", false, "Switch daemon log level to DEBUG mode")
 	flagSet.StringVarP(&cfg.ContainerdAddr, "containerd", "c", "/var/run/containerd.sock", "Specify listening address of containerd")
 	flagSet.StringVar(&cfg.ContainerdPath, "containerd-path", "", "Specify the path of containerd binary")