decouple node ipam controller from k/k

cmd/cloud-controller-manager/app/config/config.go

@@ -24,6 +24,8 @@ import (
 	"k8s.io/client-go/tools/record"
 	cloudprovider "k8s.io/cloud-provider"
 	ccmconfig "k8s.io/cloud-provider/app/apis/config"
+
+	nodeipamconfig "sigs.k8s.io/cloud-provider-azure/pkg/nodeipam/config"
 )
 
 // Config is the main context object for the cloud controller manager.
@@ -39,6 +41,8 @@ type Config struct {
 	Authentication  apiserver.AuthenticationInfo
 	Authorization   apiserver.AuthorizationInfo
 
+	NodeIPAMControllerConfig nodeipamconfig.NodeIPAMControllerConfiguration
+
 	// the general kube client
 	Client *clientset.Clientset
 

cmd/cloud-controller-manager/app/controllermanager.go

@@ -236,7 +236,7 @@ func startControllers(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan st
 		klog.V(1).Infof("Starting %q", controllerName)
 		_, started, err := initFn(c, cloud, stopCh)
 		if err != nil {
-			klog.Errorf("Error starting %q", controllerName)
+			klog.Errorf("Error starting %q: %s", controllerName, err.Error())
 			return err
 		}
 		if !started {
@@ -281,5 +281,6 @@ func newControllerInitializers() map[string]initFunc {
 	controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController
 	controllers["service"] = startServiceController
 	controllers["route"] = startRouteController
+	controllers["node-ipam"] = startNodeIpamController
 	return controllers
 }

cmd/cloud-controller-manager/app/core.go

@@ -21,6 +21,7 @@ limitations under the License.
 package app
 
 import (
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -37,6 +38,10 @@ import (
 	netutils "k8s.io/utils/net"
 
 	cloudcontrollerconfig "sigs.k8s.io/cloud-provider-azure/cmd/cloud-controller-manager/app/config"
+	cloudcontrolleroptions "sigs.k8s.io/cloud-provider-azure/cmd/cloud-controller-manager/app/options"
+	nodeipamcontroller "sigs.k8s.io/cloud-provider-azure/pkg/nodeipam"
+	nodeipamconfig "sigs.k8s.io/cloud-provider-azure/pkg/nodeipam/config"
+	"sigs.k8s.io/cloud-provider-azure/pkg/nodeipam/ipam"
 )
 
 const (
@@ -152,6 +157,151 @@ func startRouteController(ctx *cloudcontrollerconfig.CompletedConfig, cloud clou
 	return nil, true, nil
 }
 
+func startNodeIpamController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
+	var serviceCIDR *net.IPNet
+	var secondaryServiceCIDR *net.IPNet
+
+	// should we start nodeIPAM
+	if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs {
+		return nil, false, nil
+	}
+
+	// failure: bad cidrs in config
+	clusterCIDRs, dualStack, err := processCIDRs(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// failure: more than one cidr and dual stack is not enabled
+	if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) {
+		return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack feature is not enabled", len(clusterCIDRs))
+	}
+
+	// failure: more than one cidr but they are not configured as dual stack
+	if len(clusterCIDRs) > 1 && !dualStack {
+		return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs))
+	}
+
+	// failure: more than cidrs is not allowed even with dual stack
+	if len(clusterCIDRs) > 2 {
+		return nil, false, fmt.Errorf("len of clusters is:%v > more than max allowed of 2", len(clusterCIDRs))
+	}
+
+	// service cidr processing
+	if len(strings.TrimSpace(ctx.NodeIPAMControllerConfig.ServiceCIDR)) != 0 {
+		_, serviceCIDR, err = net.ParseCIDR(ctx.NodeIPAMControllerConfig.ServiceCIDR)
+		if err != nil {
+			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.NodeIPAMControllerConfig.ServiceCIDR, err)
+		}
+	}
+
+	if len(strings.TrimSpace(ctx.NodeIPAMControllerConfig.SecondaryServiceCIDR)) != 0 {
+		_, secondaryServiceCIDR, err = net.ParseCIDR(ctx.NodeIPAMControllerConfig.SecondaryServiceCIDR)
+		if err != nil {
+			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.NodeIPAMControllerConfig.SecondaryServiceCIDR, err)
+		}
+	}
+
+	// the following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided
+	if serviceCIDR != nil && secondaryServiceCIDR != nil {
+		// should have dual stack flag enabled
+		if !utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) {
+			return nil, false, fmt.Errorf("secondary service cidr is provided and IPv6DualStack feature is not enabled")
+		}
+
+		// should be dual stack (from different IPFamilies)
+		dualstackServiceCIDR, err := netutils.IsDualStackCIDRs([]*net.IPNet{serviceCIDR, secondaryServiceCIDR})
+		if err != nil {
+			return nil, false, fmt.Errorf("failed to perform dualstack check on serviceCIDR and secondaryServiceCIDR error:%v", err)
+		}
+		if !dualstackServiceCIDR {
+			return nil, false, fmt.Errorf("serviceCIDR and secondaryServiceCIDR are not dualstack (from different IPfamiles)")
+		}
+	}
+
+	var nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6 int
+	if utilfeature.DefaultFeatureGate.Enabled(IPv6DualStack) {
+		// only --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 supported with dual stack clusters.
+		// --node-cidr-mask-size flag is incompatible with dual stack clusters.
+		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(ctx.NodeIPAMControllerConfig)
+	} else {
+		// only --node-cidr-mask-size supported with single stack clusters.
+		// --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 flags are incompatible with dual stack clusters.
+		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(ctx.NodeIPAMControllerConfig)
+	}
+
+	if err != nil {
+		return nil, false, err
+	}
+
+	// get list of node cidr mask sizes
+	nodeCIDRMaskSizes := getNodeCIDRMaskSizes(clusterCIDRs, nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6)
+
+	nodeIpamController, err := nodeipamcontroller.NewNodeIpamController(
+		ctx.SharedInformers.Core().V1().Nodes(),
+		cloud,
+		ctx.ClientBuilder.ClientOrDie("node-controller"),
+		clusterCIDRs,
+		serviceCIDR,
+		secondaryServiceCIDR,
+		nodeCIDRMaskSizes,
+		ipam.CIDRAllocatorType(ctx.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
+	)
+	if err != nil {
+		return nil, true, err
+	}
+	go nodeIpamController.Run(stopCh)
+	return nil, true, nil
+}
+
+// setNodeCIDRMaskSizes returns the IPv4 and IPv6 node cidr mask sizes.
+// If --node-cidr-mask-size not set, then it will return default IPv4 and IPv6 cidr mask sizes.
+func setNodeCIDRMaskSizes(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) {
+	ipv4Mask, ipv6Mask := cloudcontrolleroptions.DefaultNodeMaskCIDRIPv4, cloudcontrolleroptions.DefaultNodeMaskCIDRIPv6
+	// NodeCIDRMaskSizeIPv4 and NodeCIDRMaskSizeIPv6 can be used only for dual-stack clusters
+	if cfg.NodeCIDRMaskSizeIPv4 != 0 || cfg.NodeCIDRMaskSizeIPv6 != 0 {
+		return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 are not allowed with non dual-stack clusters")
+	}
+	if cfg.NodeCIDRMaskSize != 0 {
+		ipv4Mask = int(cfg.NodeCIDRMaskSize)
+		ipv6Mask = int(cfg.NodeCIDRMaskSize)
+	}
+	return ipv4Mask, ipv6Mask, nil
+}
+
+// setNodeCIDRMaskSizesDualStack returns the IPv4 and IPv6 node cidr mask sizes to the value provided
+// for --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 respectively. If value not provided,
+// then it will return default IPv4 and IPv6 cidr mask sizes.
+func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) {
+	ipv4Mask, ipv6Mask := cloudcontrolleroptions.DefaultNodeMaskCIDRIPv4, cloudcontrolleroptions.DefaultNodeMaskCIDRIPv6
+	// NodeCIDRMaskSize can be used only for single stack clusters
+	if cfg.NodeCIDRMaskSize != 0 {
+		return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size is not allowed with dual-stack clusters")
+	}
+	if cfg.NodeCIDRMaskSizeIPv4 != 0 {
+		ipv4Mask = int(cfg.NodeCIDRMaskSizeIPv4)
+	}
+	if cfg.NodeCIDRMaskSizeIPv6 != 0 {
+		ipv6Mask = int(cfg.NodeCIDRMaskSizeIPv6)
+	}
+	return ipv4Mask, ipv6Mask, nil
+}
+
+// getNodeCIDRMaskSizes is a helper function that helps the generate the node cidr mask
+// sizes slice based on the cluster cidr slice
+func getNodeCIDRMaskSizes(clusterCIDRs []*net.IPNet, maskSizeIPv4, maskSizeIPv6 int) []int {
+	nodeMaskCIDRs := make([]int, len(clusterCIDRs))
+
+	for idx, clusterCIDR := range clusterCIDRs {
+		if netutils.IsIPv6CIDR(clusterCIDR) {
+			nodeMaskCIDRs[idx] = maskSizeIPv6
+		} else {
+			nodeMaskCIDRs[idx] = maskSizeIPv4
+		}
+	}
+	return nodeMaskCIDRs
+}
+
 // processCIDRs is a helper function that works on a comma separated cidrs and returns
 // a list of typed cidrs
 // a flag if cidrs represents a dual stack

cmd/cloud-controller-manager/app/options/nodeipamcontroller.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	nodeipamconfig "sigs.k8s.io/cloud-provider-azure/pkg/nodeipam/config"
+)
+
+const (
+	// DefaultNodeMaskCIDRIPv4 is default mask size for IPv4 node cidr
+	DefaultNodeMaskCIDRIPv4 = 24
+	// DefaultNodeMaskCIDRIPv6 is default mask size for IPv6 node cidr
+	DefaultNodeMaskCIDRIPv6 = 64
+	// DefaultNodeCIDRMaskSize is the default mask size for node cidr
+	DefaultNodeCIDRMaskSize = 24
+)
+
+// NodeIPAMControllerOptions holds the NodeIpamController options.
+type NodeIPAMControllerOptions struct {
+	*nodeipamconfig.NodeIPAMControllerConfiguration
+}
+
+// AddFlags adds flags related to NodeIpamController for controller manager to the specified FlagSet.
+func (o *NodeIPAMControllerOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+	fs.StringVar(&o.ServiceCIDR, "service-cluster-ip-range", "", "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true")
+	fs.Int32Var(&o.NodeCIDRMaskSize, "node-cidr-mask-size", DefaultNodeCIDRMaskSize, "Mask size for node cidr in cluster. Default is 24 for IPv4 and 64 for IPv6.")
+	fs.Int32Var(&o.NodeCIDRMaskSizeIPv4, "node-cidr-mask-size-ipv4", 0, "Mask size for IPv4 node cidr in dual-stack cluster. Default is 24.")
+	fs.Int32Var(&o.NodeCIDRMaskSizeIPv6, "node-cidr-mask-size-ipv6", 0, "Mask size for IPv6 node cidr in dual-stack cluster. Default is 64.")
+}
+
+// ApplyTo fills up NodeIpamController config with options.
+func (o *NodeIPAMControllerOptions) ApplyTo(cfg *nodeipamconfig.NodeIPAMControllerConfiguration) error {
+	if o == nil {
+		return nil
+	}
+
+	// split the cidrs list and assign primary and secondary
+	serviceCIDRList := strings.Split(o.ServiceCIDR, ",")
+	if len(serviceCIDRList) > 0 {
+		cfg.ServiceCIDR = serviceCIDRList[0]
+	}
+	if len(serviceCIDRList) > 1 {
+		cfg.SecondaryServiceCIDR = serviceCIDRList[1]
+	}
+
+	cfg.NodeCIDRMaskSize = o.NodeCIDRMaskSize
+	cfg.NodeCIDRMaskSizeIPv4 = o.NodeCIDRMaskSizeIPv4
+	cfg.NodeCIDRMaskSizeIPv6 = o.NodeCIDRMaskSizeIPv6
+
+	return nil
+}
+
+// Validate checks validation of NodeIPAMControllerOptions.
+func (o *NodeIPAMControllerOptions) Validate() []error {
+	if o == nil {
+		return nil
+	}
+	errs := make([]error, 0)
+
+	serviceCIDRList := strings.Split(o.ServiceCIDR, ",")
+	if len(serviceCIDRList) > 2 {
+		errs = append(errs, fmt.Errorf("--service-cluster-ip-range can not contain more than two entries"))
+	}
+
+	return errs
+}
+
+func defaultNodeIPAMControllerOptions() *NodeIPAMControllerOptions {
+	return &NodeIPAMControllerOptions{
+		&nodeipamconfig.NodeIPAMControllerConfiguration{
+			ServiceCIDR:          "",
+			NodeCIDRMaskSize:     DefaultNodeCIDRMaskSize,
+			NodeCIDRMaskSizeIPv4: 0,
+			NodeCIDRMaskSizeIPv6: 0,
+		},
+	}
+}

cmd/cloud-controller-manager/app/options/options.go

@@ -64,9 +64,10 @@ const (
 
 // CloudControllerManagerOptions is the main context object for the controller manager.
 type CloudControllerManagerOptions struct {
-	Generic           *cmoptions.GenericControllerManagerConfigurationOptions
-	KubeCloudShared   *cpoptions.KubeCloudSharedOptions
-	ServiceController *cpoptions.ServiceControllerOptions
+	Generic            *cmoptions.GenericControllerManagerConfigurationOptions
+	KubeCloudShared    *cpoptions.KubeCloudSharedOptions
+	ServiceController  *cpoptions.ServiceControllerOptions
+	NodeIPAMController *NodeIPAMControllerOptions
 
 	SecureServing *apiserveroptions.SecureServingOptionsWithLoopback
 	// TODO: remove insecure serving mode
@@ -94,7 +95,8 @@ func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error)
 		ServiceController: &cpoptions.ServiceControllerOptions{
 			ServiceControllerConfiguration: &componentConfig.ServiceController,
 		},
-		SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
+		NodeIPAMController: defaultNodeIPAMControllerOptions(),
+		SecureServing:      apiserveroptions.NewSecureServingOptions().WithLoopback(),
 		InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
 			BindAddress: net.ParseIP(componentConfig.Generic.Address),
 			BindPort:    int(componentConfig.Generic.Port),
@@ -142,6 +144,7 @@ func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultC
 	o.Generic.AddFlags(&fss, allControllers, disabledByDefaultControllers)
 	o.KubeCloudShared.AddFlags(fss.FlagSet("generic"))
 	o.ServiceController.AddFlags(fss.FlagSet("service controller"))
+	o.NodeIPAMController.AddFlags(fss.FlagSet("node ipam controller"))
 
 	o.SecureServing.AddFlags(fss.FlagSet("secure serving"))
 	o.InsecureServing.AddUnqualifiedFlags(fss.FlagSet("insecure serving"))
@@ -170,6 +173,9 @@ func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config,
 	if err = o.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil {
 		return err
 	}
+	if err = o.NodeIPAMController.ApplyTo(&c.NodeIPAMControllerConfig); err != nil {
+		return err
+	}
 	if err = o.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil {
 		return err
 	}
@@ -239,6 +245,7 @@ func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefau
 	errors = append(errors, o.Generic.Validate(allControllers, disabledByDefaultControllers)...)
 	errors = append(errors, o.KubeCloudShared.Validate()...)
 	errors = append(errors, o.ServiceController.Validate()...)
+	errors = append(errors, o.NodeIPAMController.Validate()...)
 	errors = append(errors, o.SecureServing.Validate()...)
 	errors = append(errors, o.InsecureServing.Validate()...)
 	errors = append(errors, o.Authentication.Validate()...)

cmd/cloud-controller-manager/app/options/options_test.go

@@ -34,6 +34,7 @@ import (
 	componentbaseconfig "k8s.io/component-base/config"
 	kubectrlmgrconfig "k8s.io/controller-manager/config"
 	cmoptions "k8s.io/controller-manager/options"
+	"sigs.k8s.io/cloud-provider-azure/pkg/nodeipam/config"
 )
 
 func TestDefaultFlags(t *testing.T) {
@@ -91,6 +92,11 @@ func TestDefaultFlags(t *testing.T) {
 				ConcurrentServiceSyncs: 1,
 			},
 		},
+		NodeIPAMController: &NodeIPAMControllerOptions{
+			NodeIPAMControllerConfiguration: &config.NodeIPAMControllerConfiguration{
+				NodeCIDRMaskSize: DefaultNodeCIDRMaskSize,
+			},
+		},
 		SecureServing: (&apiserveroptions.SecureServingOptions{
 			BindPort:    10258,
 			BindAddress: net.ParseIP("0.0.0.0"),
@@ -229,6 +235,11 @@ func TestAddFlags(t *testing.T) {
 				ConcurrentServiceSyncs: 1,
 			},
 		},
+		NodeIPAMController: &NodeIPAMControllerOptions{
+			NodeIPAMControllerConfiguration: &config.NodeIPAMControllerConfiguration{
+				NodeCIDRMaskSize: DefaultNodeCIDRMaskSize,
+			},
+		},
 		SecureServing: (&apiserveroptions.SecureServingOptions{
 			BindPort:    10001,
 			BindAddress: net.ParseIP("192.168.4.21"),