feat: allow dual-stack support with bootkube wrapper

Handle dual-stack configurations with the bootkube wrapper.  This uses
the new PodCIDRs and ServiceCIDRs `asset.Config` parameters in bootkube.
It also relies on the bootkube-plugin features for manipulating
kube-proxy config and installing the dual-stack DNS service.

Fixes siderolabs#2055

Signed-off-by: Seán C McCord <[email protected]>

docs/website/content/v0.4/en/configuration/v1alpha1.md

@@ -520,6 +520,22 @@ controllerManager:
 
 ```
 
+#### proxy
+
+Kube-proxy server-specific configuration options
+
+Type: `ProxyConfig`
+
+Examples:
+
+```yaml
+proxy:
+  mode: ipvs
+  extraArgs:
+    key: value
+
+```
+
 #### scheduler
 
 Scheduler server specific configuration options.
@@ -969,6 +985,23 @@ Type: `map`
 
 ---
 
+### ProxyConfig
+
+#### mode
+
+proxy mode of kube-proxy.
+By default, this is 'iptables'.
+
+Type: `string`
+
+#### extraArgs
+
+Extra arguments to supply to kube-proxy.
+
+Type: `map`
+
+---
+
 ### SchedulerConfig
 
 #### image

go.mod

@@ -60,7 +60,7 @@ require (
 	github.com/spf13/cobra v0.0.5
 	github.com/stretchr/testify v1.5.1
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2
-	github.com/talos-systems/bootkube-plugin v0.0.0-20200418011639-b642f0b2921d
+	github.com/talos-systems/bootkube-plugin v0.0.0-20200428220847-2d8b58bf3941
 	github.com/talos-systems/go-procfs v0.0.0-20200219015357-57c7311fdd45
 	github.com/talos-systems/go-smbios v0.0.0-20200219201045-94b8c4e489ee
 	github.com/talos-systems/grpc-proxy v0.2.0

go.sum

@@ -589,8 +589,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/talos-systems/bootkube-plugin v0.0.0-20200418011639-b642f0b2921d h1:UMQx8f7FqcQhwrKpBVGxxF7Kakjr+r7OQ941HmO1cgA=
-github.com/talos-systems/bootkube-plugin v0.0.0-20200418011639-b642f0b2921d/go.mod h1:AbdJAgHK5rJNDPUN3msPTfQJSR9b4DKb5xNN07uG8/Y=
+github.com/talos-systems/bootkube-plugin v0.0.0-20200428220847-2d8b58bf3941 h1:+YwUn+yPyjFl2x0J9fLaDdV1mmg/H1Lcia+1D8t3Grc=
+github.com/talos-systems/bootkube-plugin v0.0.0-20200428220847-2d8b58bf3941/go.mod h1:AbdJAgHK5rJNDPUN3msPTfQJSR9b4DKb5xNN07uG8/Y=
 github.com/talos-systems/go-procfs v0.0.0-20200219015357-57c7311fdd45 h1:FND/LgzFHTBdJBOeZVzdO6B47kxQZvSIzb9AMIXYotg=
 github.com/talos-systems/go-procfs v0.0.0-20200219015357-57c7311fdd45/go.mod h1:ATyUGFQIW8OnbnmvqefZWVPgL9g+CAmXHfkgny21xX8=
 github.com/talos-systems/go-smbios v0.0.0-20200219201045-94b8c4e489ee h1:9i0ZFsjZ0wY8UUn/tk2MQshLBC0PNFJe3+84AUqzzyw=

internal/app/bootkube/assets.go

@@ -17,6 +17,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 
 	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/go-multierror"
@@ -92,14 +93,14 @@ func generateAssets(config runtime.Configurator) (err error) {
 		return err
 	}
 
-	_, podCIDR, err := net.ParseCIDR(config.Cluster().Network().PodCIDR())
+	podCIDRs, err := splitCIDRs(config.Cluster().Network().PodCIDR())
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to process Pod CIDRs: %w", err)
 	}
 
-	_, serviceCIDR, err := net.ParseCIDR(config.Cluster().Network().ServiceCIDR())
+	serviceCIDRs, err := splitCIDRs(config.Cluster().Network().ServiceCIDR())
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to process Service CIDRs: %w", err)
 	}
 
 	urls := []string{config.Cluster().Endpoint().Hostname()}
@@ -116,14 +117,14 @@ func generateAssets(config runtime.Configurator) (err error) {
 		return fmt.Errorf("failed to parse Kubernetes key: %w", err)
 	}
 
-	apiServiceIP, err := tnet.NthIPInNetwork(serviceCIDR, 1)
+	apiServiceIPs, err := nthIPInCIDRSet(serviceCIDRs, 1)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to calculate API service IP: %w", err)
 	}
 
-	dnsServiceIP, err := tnet.NthIPInNetwork(serviceCIDR, 10)
+	dnsServiceIPs, err := nthIPInCIDRSet(serviceCIDRs, 10)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to calculate DNS service IP: %w", err)
 	}
 
 	images := asset.DefaultImages
@@ -138,6 +139,8 @@ func generateAssets(config runtime.Configurator) (err error) {
 		ClusterName:                config.Cluster().Name(),
 		APIServerExtraArgs:         config.Cluster().APIServer().ExtraArgs(),
 		ControllerManagerExtraArgs: config.Cluster().ControllerManager().ExtraArgs(),
+		ProxyMode:                  config.Cluster().Proxy().Mode(),
+		ProxyExtraArgs:             config.Cluster().Proxy().ExtraArgs(),
 		SchedulerExtraArgs:         config.Cluster().Scheduler().ExtraArgs(),
 		CACert:                     k8sCA,
 		CAPrivKey:                  k8sKey,
@@ -148,10 +151,10 @@ func generateAssets(config runtime.Configurator) (err error) {
 		EtcdUseTLS:                 true,
 		ControlPlaneEndpoint:       config.Cluster().Endpoint(),
 		LocalAPIServerPort:         config.Cluster().LocalAPIServerPort(),
-		APIServiceIP:               apiServiceIP,
-		DNSServiceIP:               dnsServiceIP,
-		PodCIDR:                    podCIDR,
-		ServiceCIDR:                serviceCIDR,
+		APIServiceIPs:              apiServiceIPs,
+		DNSServiceIPs:              dnsServiceIPs,
+		PodCIDRs:                   podCIDRs,
+		ServiceCIDRs:               serviceCIDRs,
 		NetworkProvider:            config.Cluster().Network().CNI().Name(),
 		AltNames:                   altNames,
 		Images:                     images,
@@ -249,3 +252,29 @@ func fetchManifests(urls []string, headers map[string]string) error {
 
 	return result.ErrorOrNil()
 }
+
+func splitCIDRs(cidrList string) (out []*net.IPNet, err error) {
+	for _, podCIDR := range strings.Split(cidrList, ",") {
+		_, cidr, err := net.ParseCIDR(podCIDR)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %q as a CIDR: %w", podCIDR, err)
+		}
+
+		out = append(out, cidr)
+	}
+
+	return out, nil
+}
+
+func nthIPInCIDRSet(cidrList []*net.IPNet, offset int) (out []net.IP, err error) {
+	for _, cidr := range cidrList {
+		ip, err := tnet.NthIPInNetwork(cidr, offset)
+		if err != nil {
+			return nil, fmt.Errorf("failed to calculate offset %d from CIDR %q: %w", offset, cidr, err)
+		}
+
+		out = append(out, ip)
+	}
+
+	return out, nil
+}

internal/app/machined/pkg/runtime/configurator.go

@@ -337,6 +337,7 @@ type ClusterConfig interface {
 	Name() string
 	APIServer() APIServer
 	ControllerManager() ControllerManager
+	Proxy() Proxy
 	Scheduler() Scheduler
 	Endpoint() *url.URL
 	Token() Token
@@ -383,6 +384,17 @@ type ControllerManager interface {
 	ExtraArgs() map[string]string
 }
 
+// Proxy defines the requirements for a config that pertains to the kube-proxy
+// options.
+type Proxy interface {
+
+	// Mode indicates the proxy mode for kube-proxy.  By default, this is `iptables`.  Other options include `ipvs`.
+	Mode() string
+
+	// ExtraArgs describe an additional set of arguments to be supplied to the execution of `kube-proxy`
+	ExtraArgs() map[string]string
+}
+
 // Scheduler defines the requirements for a config that pertains to scheduler related
 // options.
 type Scheduler interface {

pkg/config/types/v1alpha1/generate/init.go

@@ -50,6 +50,7 @@ func initUd(in *Input) (*v1alpha1.Config, error) {
 			CertSANs: certSANs,
 		},
 		ControllerManagerConfig: &v1alpha1.ControllerManagerConfig{},
+		ProxyConfig:             &v1alpha1.ProxyConfig{},
 		SchedulerConfig:         &v1alpha1.SchedulerConfig{},
 		EtcdConfig: &v1alpha1.EtcdConfig{
 			RootCA: in.Certs.Etcd,

pkg/config/types/v1alpha1/v1alpha1_configurator.go

@@ -281,6 +281,29 @@ func (c *ControllerManagerConfig) ExtraArgs() map[string]string {
 	return c.ExtraArgsConfig
 }
 
+// Proxy implements the Configurator interface
+func (c *ClusterConfig) Proxy() runtime.Proxy {
+	if c.ProxyConfig == nil {
+		return &ProxyConfig{}
+	}
+
+	return c.ProxyConfig
+}
+
+// Mode implements the Proxy interface
+func (p *ProxyConfig) Mode() string {
+	if p.ModeConfig == "" {
+		return "iptables"
+	}
+
+	return p.ModeConfig
+}
+
+// ExtraArgs implements the Proxy interface.
+func (p *ProxyConfig) ExtraArgs() map[string]string {
+	return p.ExtraArgsConfig
+}
+
 // Scheduler implements the Configurator interface.
 func (c *ClusterConfig) Scheduler() runtime.Scheduler {
 	if c.SchedulerConfig == nil {

pkg/config/types/v1alpha1/v1alpha1_types.go

@@ -301,6 +301,15 @@ type ClusterConfig struct {
 	//           key: value
 	ControllerManagerConfig *ControllerManagerConfig `yaml:"controllerManager,omitempty"`
 	//   description: |
+	//     Kube-proxy server-specific configuration options
+	//   examples:
+	//     - |
+	//       proxy:
+	//         mode: ipvs
+	//         extraArgs:
+	//           key: value
+	ProxyConfig *ProxyConfig `yaml:"proxy,omitempty"`
+	//   description: |
 	//     Scheduler server specific configuration options.
 	//   examples:
 	//     - |
@@ -608,6 +617,17 @@ type ControllerManagerConfig struct {
 	ExtraArgsConfig map[string]string `yaml:"extraArgs,omitempty"`
 }
 
+// ProxyConfig represents the kube proxy configuration values
+type ProxyConfig struct {
+	//   description: |
+	//     proxy mode of kube-proxy.
+	//     By default, this is 'iptables'.
+	ModeConfig string `yaml:"mode,omitempty"`
+	//   description: |
+	//     Extra arguments to supply to kube-proxy.
+	ExtraArgsConfig map[string]string `yaml:"extraArgs,omitempty"`
+}
+
 // SchedulerConfig represents kube scheduler config vals.
 type SchedulerConfig struct {
 	//   description: |