Add initial support for Hetzner Cloud

Makefile

@@ -43,7 +43,7 @@ UPLOAD_CMD=$(KOPS_ROOT)/hack/upload ${UPLOAD_ARGS}
 # Unexport environment variables that can affect tests and are not used in builds
 unexport AWS_ACCESS_KEY_ID AWS_REGION AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN CNI_VERSION_URL DNS_IGNORE_NS_CHECK DNSCONTROLLER_IMAGE DO_ACCESS_TOKEN GOOGLE_APPLICATION_CREDENTIALS
 unexport KOPS_BASE_URL KOPS_CLUSTER_NAME KOPS_RUN_OBSOLETE_VERSION KOPS_STATE_STORE KOPS_STATE_S3_ACL KUBE_API_VERSIONS NODEUP_URL OPENSTACK_CREDENTIAL_FILE SKIP_PACKAGE_UPDATE
-unexport SKIP_REGION_CHECK S3_ACCESS_KEY_ID S3_ENDPOINT S3_REGION S3_SECRET_ACCESS_KEY
+unexport SKIP_REGION_CHECK S3_ACCESS_KEY_ID S3_ENDPOINT S3_REGION S3_SECRET_ACCESS_KEY HCLOUD_TOKEN
 
 
 VERSION=$(shell tools/get_version.sh | grep VERSION | awk '{print $$2}')

cmd/kops-controller/main.go

@@ -37,6 +37,7 @@ import (
 	nodeidentityazure "k8s.io/kops/pkg/nodeidentity/azure"
 	nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do"
 	nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce"
+	nodeidentityhetzner "k8s.io/kops/pkg/nodeidentity/hetzner"
 	nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier"
@@ -206,6 +207,12 @@ func addNodeController(mgr manager.Manager, opt *config.Options) error {
 			return fmt.Errorf("error building identifier: %v", err)
 		}
 
+	case "hetzner":
+		identifier, err = nodeidentityhetzner.New(opt.CacheNodeidentityInfo)
+		if err != nil {
+			return fmt.Errorf("error building identifier: %w", err)
+		}
+
 	case "azure":
 		identifier, err = nodeidentityazure.New(opt.CacheNodeidentityInfo)
 		if err != nil {

go.mod

@@ -61,6 +61,7 @@ require (
 	github.com/gophercloud/gophercloud v0.24.0
 	github.com/hashicorp/hcl/v2 v2.12.0
 	github.com/hashicorp/vault/api v1.5.0
+	github.com/hetznercloud/hcloud-go v1.33.2
 	github.com/jacksontj/memberlistmesh v0.0.0-20190905163944-93462b9d2bb7
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pelletier/go-toml v1.9.5

go.sum

@@ -822,6 +822,8 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/heketi/heketi v10.3.0+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
 github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4=
+github.com/hetznercloud/hcloud-go v1.33.2 h1:ptWKVYLW7YtjXzsqTFKFxwpVo3iM9UMkVPBYQE4teLU=
+github.com/hetznercloud/hcloud-go v1.33.2/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=

hack/update-expected.sh

@@ -27,7 +27,7 @@ PKG="${1:-./...}"
 
 # Don't override variables that are commonly used in dev, but shouldn't be in our tests
 unset KOPS_BASE_URL DNSCONTROLLER_IMAGE KOPSCONTROLLER_IMAGE KUBE_APISERVER_HEALTHCHECK_IMAGE KOPS_FEATURE_FLAGS KOPS_ARCH
-unset AWS_ACCESS_KEY_ID AWS_REGION AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN CNI_VERSION_URL DNS_IGNORE_NS_CHECK DO_ACCESS_TOKEN GOOGLE_APPLICATION_CREDENTIALS
+unset AWS_ACCESS_KEY_ID AWS_REGION AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN CNI_VERSION_URL DNS_IGNORE_NS_CHECK DO_ACCESS_TOKEN GOOGLE_APPLICATION_CREDENTIALS HCLOUD_TOKEN
 unset KOPS_CLUSTER_NAME KOPS_RUN_OBSOLETE_VERSION KOPS_STATE_STORE KOPS_STATE_S3_ACL KUBE_API_VERSIONS NODEUP_URL OPENSTACK_CREDENTIAL_FILE PROTOKUBE_IMAGE SKIP_PACKAGE_UPDATE
 unset SKIP_REGION_CHECK S3_ACCESS_KEY_ID S3_ENDPOINT S3_REGION S3_SECRET_ACCESS_KEY
 

nodeup/pkg/bootstrap/install.go

@@ -136,6 +136,10 @@ func (i *Installation) buildEnvFile() *nodetasks.File {
 		envVars["DIGITALOCEAN_ACCESS_TOKEN"] = os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
 	}
 
+	if os.Getenv("HCLOUD_TOKEN") != "" {
+		envVars["HCLOUD_TOKEN"] = os.Getenv("HCLOUD_TOKEN")
+	}
+
 	if os.Getenv("OSS_REGION") != "" {
 		envVars["OSS_REGION"] = os.Getenv("OSS_REGION")
 	}

nodeup/pkg/model/context.go

@@ -18,14 +18,17 @@ package model
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/blang/semver/v4"
+	hcloudmetadata "github.com/hetznercloud/hcloud-go/hcloud/metadata"
 	"k8s.io/klog/v2"
-	"k8s.io/mount-utils"
-
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/apis/kops/util"
@@ -36,8 +39,8 @@ import (
 	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/distributions"
 	"k8s.io/kops/util/pkg/vfs"
-
-	"github.com/blang/semver/v4"
+	"k8s.io/mount-utils"
+	"sigs.k8s.io/yaml"
 )
 
 const (
@@ -372,8 +375,10 @@ func (c *NodeupModelContext) UseKopsControllerForNodeBootstrap() bool {
 
 // UsesSecondaryIP checks if the CNI in use attaches secondary interfaces to the host.
 func (c *NodeupModelContext) UsesSecondaryIP() bool {
-	return (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) || c.Cluster.Spec.Networking.AmazonVPC != nil ||
-		(c.Cluster.Spec.Networking.Cilium != nil && c.Cluster.Spec.Networking.Cilium.IPAM == kops.CiliumIpamEni)
+	return (c.Cluster.Spec.Networking.CNI != nil && c.Cluster.Spec.Networking.CNI.UsesSecondaryIP) ||
+		c.Cluster.Spec.Networking.AmazonVPC != nil ||
+		(c.Cluster.Spec.Networking.Cilium != nil && c.Cluster.Spec.Networking.Cilium.IPAM == kops.CiliumIpamEni) ||
+		c.CloudProvider == kops.CloudProviderHetzner
 }
 
 // UseBootstrapTokens checks if we are using bootstrap tokens
@@ -617,3 +622,51 @@ func (c *NodeupModelContext) InstallNvidiaRuntime() bool {
 func (c *NodeupModelContext) RunningOnGCE() bool {
 	return c.CloudProvider == kops.CloudProviderGCE
 }
+
+// GetMetadataLocalIP returns the local IP address read from metadata
+func (c *NodeupModelContext) GetMetadataLocalIP() (string, error) {
+	var internalIP string
+
+	switch c.CloudProvider {
+	case kops.CloudProviderAWS:
+		sess := session.Must(session.NewSession())
+		metadata := ec2metadata.New(sess)
+		localIPv4, err := metadata.GetMetadata("local-ipv4")
+		if err != nil {
+			return "", fmt.Errorf("failed to get local-ipv4 address from ec2 metadata: %w", err)
+		}
+		internalIP = localIPv4
+
+	case kops.CloudProviderHetzner:
+		client := hcloudmetadata.NewClient()
+		privateNetworksYaml, err := client.PrivateNetworks()
+		if err != nil {
+			return "", fmt.Errorf("failed to get private networks from hetzner cloud metadata: %w", err)
+		}
+		var privateNetworks []struct {
+			IP           net.IP   `json:"ip"`
+			AliasIPs     []net.IP `json:"alias_ips"`
+			InterfaceNum int      `json:"interface_num"`
+			MACAddress   string   `json:"mac_address"`
+			NetworkID    int      `json:"network_id"`
+			NetworkName  string   `json:"network_name"`
+			Network      string   `json:"network"`
+			Subnet       string   `json:"subnet"`
+			Gateway      net.IP   `json:"gateway"`
+		}
+		err = yaml.Unmarshal([]byte(privateNetworksYaml), &privateNetworks)
+		if err != nil {
+			return "", fmt.Errorf("failed to convert private networks to object: %w", err)
+		}
+		for _, privateNetwork := range privateNetworks {
+			if privateNetwork.InterfaceNum == 1 {
+				internalIP = privateNetwork.IP.String()
+			}
+		}
+
+	default:
+		return "", fmt.Errorf("getting local IP from metadata is not supported for cloud provider: %q", c.CloudProvider)
+	}
+
+	return internalIP, nil
+}

nodeup/pkg/model/kube_apiserver.go

@@ -64,6 +64,16 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 		kubeAPIServer = *b.NodeupConfig.APIServerConfig.KubeAPIServer
 	}
 
+	if b.CloudProvider == kops.CloudProviderHetzner {
+		localIP, err := b.GetMetadataLocalIP()
+		if err != nil {
+			return err
+		}
+		if localIP != "" {
+			kubeAPIServer.AdvertiseAddress = localIP
+		}
+	}
+
 	if err := b.writeAuthenticationConfig(c, &kubeAPIServer); err != nil {
 		return err
 	}

nodeup/pkg/model/kubelet.go

@@ -25,28 +25,24 @@ import (
 	"path/filepath"
 	"strings"
 
-	"k8s.io/kops/pkg/model/components"
-
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
-
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/klog/v2"
-	kubelet "k8s.io/kubelet/config/v1beta1"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
-
+	"k8s.io/klog/v2"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/flagbuilder"
+	"k8s.io/kops/pkg/model/components"
 	"k8s.io/kops/pkg/nodelabels"
 	"k8s.io/kops/pkg/rbac"
 	"k8s.io/kops/pkg/systemd"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/util/pkg/distributions"
+	kubelet "k8s.io/kubelet/config/v1beta1"
 )
 
 const (
@@ -295,13 +291,13 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
 	}
 
 	if b.UsesSecondaryIP() {
-		sess := session.Must(session.NewSession())
-		metadata := ec2metadata.New(sess)
-		localIpv4, err := metadata.GetMetadata("local-ipv4")
+		localIP, err := b.GetMetadataLocalIP()
 		if err != nil {
-			return nil, fmt.Errorf("error fetching the local-ipv4 address from the ec2 meta-data: %v", err)
+			return nil, err
+		}
+		if localIP != "" {
+			flags += " --node-ip=" + localIP
 		}
-		flags += " --node-ip=" + localIpv4
 	}
 
 	if b.usesContainerizedMounter() {

nodeup/pkg/model/protokube.go

@@ -334,6 +334,10 @@ func (t *ProtokubeBuilder) buildEnvFile() (*nodetasks.File, error) {
 		envVars["DIGITALOCEAN_ACCESS_TOKEN"] = os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
 	}
 
+	if os.Getenv("HCLOUD_TOKEN") != "" {
+		envVars["HCLOUD_TOKEN"] = os.Getenv("HCLOUD_TOKEN")
+	}
+
 	if os.Getenv("OSS_REGION") != "" {
 		envVars["OSS_REGION"] = os.Getenv("OSS_REGION")
 	}

pkg/apis/kops/channel.go

@@ -289,6 +289,7 @@ const (
 	CloudProviderAWS       CloudProviderID = "aws"
 	CloudProviderDO        CloudProviderID = "digitalocean"
 	CloudProviderGCE       CloudProviderID = "gce"
+	CloudProviderHetzner   CloudProviderID = "hetzner"
 	CloudProviderOpenstack CloudProviderID = "openstack"
 	CloudProviderAzure     CloudProviderID = "azure"
 )

pkg/apis/kops/cluster.go

@@ -236,6 +236,8 @@ type CloudProviderSpec struct {
 	DO *DOSpec `json:"do,omitempty"`
 	// GCE configures the GCE cloud provider.
 	GCE *GCESpec `json:"gce,omitempty"`
+	// Hetzner configures the Hetzner cloud provider.
+	Hetzner *HetznerSpec `json:"hetzner,omitempty"`
 	// Openstack configures the Openstack cloud provider.
 	Openstack *OpenstackSpec `json:"openstack,omitempty"`
 }
@@ -252,6 +254,10 @@ type DOSpec struct {
 type GCESpec struct {
 }
 
+// HetznerSpec configures the Hetzner cloud provider.
+type HetznerSpec struct {
+}
+
 type KarpenterConfig struct {
 	Enabled bool `json:"enabled,omitempty"`
 }
@@ -896,6 +902,8 @@ func (c *ClusterSpec) GetCloudProvider() CloudProviderID {
 		return CloudProviderDO
 	} else if c.CloudProvider.GCE != nil {
 		return CloudProviderGCE
+	} else if c.CloudProvider.Hetzner != nil {
+		return CloudProviderHetzner
 	} else if c.CloudProvider.Openstack != nil {
 		return CloudProviderOpenstack
 	}

pkg/apis/kops/v1alpha2/conversion.go

@@ -89,6 +89,8 @@ func Convert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *kops
 		out.CloudProvider.DO = &kops.DOSpec{}
 	case kops.CloudProviderGCE:
 		out.CloudProvider.GCE = &kops.GCESpec{}
+	case kops.CloudProviderHetzner:
+		out.CloudProvider.Hetzner = &kops.HetznerSpec{}
 	case kops.CloudProviderOpenstack:
 		out.CloudProvider.Openstack = &kops.OpenstackSpec{}
 		if in.CloudConfig != nil && in.CloudConfig.Openstack != nil {
@@ -103,6 +105,7 @@ func Convert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *kops
 			string(kops.CloudProviderDO),
 			string(kops.CloudProviderAzure),
 			string(kops.CloudProviderAWS),
+			string(kops.CloudProviderHetzner),
 			string(kops.CloudProviderOpenstack),
 		})
 	}

pkg/apis/kops/v1alpha3/cluster.go

@@ -233,6 +233,8 @@ type CloudProviderSpec struct {
 	DO *DOSpec `json:"do,omitempty"`
 	// GCE configures the GCE cloud provider.
 	GCE *GCESpec `json:"gce,omitempty"`
+	// Hetzner configures the Hetzner cloud provider.
+	Hetzner *HetznerSpec `json:"hetzner,omitempty"`
 	// Openstack configures the Openstack cloud provider.
 	Openstack *OpenstackSpec `json:"openstack,omitempty"`
 }
@@ -249,6 +251,10 @@ type DOSpec struct {
 type GCESpec struct {
 }
 
+// HetznerSpec configures the Hetzner cloud provider.
+type HetznerSpec struct {
+}
+
 type KarpenterConfig struct {
 	Enabled bool `json:"enabled,omitempty"`
 }