Affinity defaulting for mongodb (kubedb#490)

* Affinity defaulting for mongodb Signed-off-by: the-redback <[email protected]> * Set anti-affinity against pods in same namespace Signed-off-by: Tamal Saha <[email protected]> * Generate default affinity rules based on topology Signed-off-by: Tamal Saha <[email protected]> * Add ClusterTopology to Controller Signed-off-by: Tamal Saha <[email protected]> Co-authored-by: Tamal Saha <[email protected]>
andersosthus · Jan 25, 2020 · 118d374 · 118d374
1 parent de5af66
commit 118d374
Show file tree

Hide file tree

Showing 8 changed files with 211 additions and 17 deletions.
diff --git a/apis/kubedb/v1alpha1/mongodb_helpers.go b/apis/kubedb/v1alpha1/mongodb_helpers.go
@@ -30,6 +30,8 @@ import (
 	apps "k8s.io/api/apps/v1"
 	core "k8s.io/api/core/v1"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	core_util "kmodules.xyz/client-go/core/v1"
 	v1 "kmodules.xyz/client-go/core/v1"
 	meta_util "kmodules.xyz/client-go/meta"
 	appcat "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
@@ -71,7 +73,7 @@ func (m MongoDB) ShardNodeTemplate() string {
 	if m.Spec.ShardTopology == nil {
 		return ""
 	}
-	return fmt.Sprintf("%v${%v}", m.ShardCommonNodeName(), ShardAffinityTemplateVar)
+	return fmt.Sprintf("%s${%s}", m.ShardCommonNodeName(), ShardAffinityTemplateVar)
 }
 
 func (m MongoDB) ShardCommonNodeName() string {
@@ -308,7 +310,7 @@ func (m *MongoDB) GetMonitoringVendor() string {
 	return ""
 }
 
-func (m *MongoDB) SetDefaults(mgVersion *v1alpha1.MongoDBVersion) {
+func (m *MongoDB) SetDefaults(mgVersion *v1alpha1.MongoDBVersion, topology *core_util.Topology) {
 	if m == nil {
 		return
 	}
@@ -364,6 +366,19 @@ func (m *MongoDB) SetDefaults(mgVersion *v1alpha1.MongoDBVersion) {
 		m.setDefaultProbes(&m.Spec.ShardTopology.Shard.PodTemplate, mgVersion)
 		m.setDefaultProbes(&m.Spec.ShardTopology.ConfigServer.PodTemplate, mgVersion)
 		m.setDefaultProbes(&m.Spec.ShardTopology.Mongos.PodTemplate, mgVersion)
+
+		// set default affinity (PodAntiAffinity)
+		shardLabels := m.OffshootSelectors()
+		shardLabels[MongoDBShardLabelKey] = m.ShardNodeTemplate()
+		m.setDefaultAffinity(&m.Spec.ShardTopology.Shard.PodTemplate, shardLabels, topology, int(m.Spec.ShardTopology.Shard.Replicas)*int(m.Spec.ShardTopology.Shard.Shards))
+
+		configServerLabels := m.OffshootSelectors()
+		configServerLabels[MongoDBConfigLabelKey] = m.ConfigSvrNodeName()
+		m.setDefaultAffinity(&m.Spec.ShardTopology.ConfigServer.PodTemplate, configServerLabels, topology, int(m.Spec.ShardTopology.ConfigServer.Replicas))
+
+		mongosLabels := m.OffshootSelectors()
+		mongosLabels[MongoDBMongosLabelKey] = m.MongosNodeName()
+		m.setDefaultAffinity(&m.Spec.ShardTopology.Mongos.PodTemplate, mongosLabels, topology, int(m.Spec.ShardTopology.Mongos.Replicas))
 	} else {
 		if m.Spec.Replicas == nil {
 			m.Spec.Replicas = types.Int32P(1)
@@ -378,6 +393,8 @@ func (m *MongoDB) SetDefaults(mgVersion *v1alpha1.MongoDBVersion) {
 
 		// set default probes
 		m.setDefaultProbes(m.Spec.PodTemplate, mgVersion)
+		// set default affinity (PodAntiAffinity)
+		m.setDefaultAffinity(m.Spec.PodTemplate, m.OffshootSelectors(), topology, int(*m.Spec.Replicas))
 	}
 }
 
@@ -450,6 +467,46 @@ func (m *MongoDB) setDefaultProbes(podTemplate *ofst.PodTemplateSpec, mgVersion
 	}
 }
 
+// setDefaultAffinity
+func (m *MongoDB) setDefaultAffinity(podTemplate *ofst.PodTemplateSpec, labels map[string]string, topology *core_util.Topology, totalPods int) {
+	if podTemplate == nil || podTemplate.Spec.Affinity != nil {
+		// Update topologyKey fields according to Kubernetes version
+		topology.ConvertAffinity(podTemplate.Spec.Affinity)
+		return
+	}
+
+	podTemplate.Spec.Affinity = &core.Affinity{
+		PodAntiAffinity: &core.PodAntiAffinity{
+			// Prefer to not schedule multiple pods on the node with same zone
+			PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{
+				{
+					Weight: 100,
+					PodAffinityTerm: core.PodAffinityTerm{
+						Namespaces: []string{m.Namespace},
+						LabelSelector: &metav1.LabelSelector{
+							MatchLabels: labels,
+						},
+						TopologyKey: topology.LabelZone,
+					},
+				},
+			},
+		},
+	}
+
+	// If there are more nodes than pods, don't schedule multiple pods on the same node
+	if topology.TotalNodes > totalPods {
+		podTemplate.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = []core.PodAffinityTerm{
+			{
+				Namespaces: []string{m.Namespace},
+				LabelSelector: &metav1.LabelSelector{
+					MatchLabels: labels,
+				},
+				TopologyKey: core.LabelHostname,
+			},
+		}
+	}
+}
+
 // setSecurityContext will set default PodSecurityContext.
 // These values will be applied only to newly created objects.
 // These defaultings should not be applied to DBs or dormantDBs,

diff --git a/apis/kubedb/v1alpha1/mongodb_helpers_test.go b/apis/kubedb/v1alpha1/mongodb_helpers_test.go
@@ -25,8 +25,22 @@ import (
 	core "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	core_util "kmodules.xyz/client-go/core/v1"
 )
 
+var testTopology = &core_util.Topology{
+	Regions: map[string][]string{
+		"us-east-1": {"us-east-1a", "us-east-1b", "us-east-1c"},
+	},
+	TotalNodes: 100,
+	InstanceTypes: map[string]int{
+		"n1-standard-4": 100,
+	},
+	LabelZone:         core.LabelZoneFailureDomain,
+	LabelRegion:       core.LabelZoneRegion,
+	LabelInstanceType: core.LabelInstanceType,
+}
+
 func TestMongoDB_HostAddress(t *testing.T) {
 	mongodb := &MongoDB{
 		ObjectMeta: metav1.ObjectMeta{
@@ -75,6 +89,8 @@ func TestMongoDB_HostAddress(t *testing.T) {
 		},
 	}
 
+	mongodb.SetDefaults(&v1alpha1.MongoDBVersion{}, testTopology)
+
 	shardDSN := mongodb.HostAddress()
 	t.Log(shardDSN)
 
@@ -140,7 +156,7 @@ func TestMongoDB_ShardDSN(t *testing.T) {
 	shardDSN := mongodb.ShardDSN(0)
 	t.Log(shardDSN)
 
-	mongodb.SetDefaults(&v1alpha1.MongoDBVersion{})
+	mongodb.SetDefaults(&v1alpha1.MongoDBVersion{}, testTopology)
 }
 
 func TestMongoDB_ConfigSvrDSN(t *testing.T) {
@@ -214,5 +230,5 @@ func TestMongoDB_SetDefaults(t *testing.T) {
 		},
 	}
 
-	mongodb.SetDefaults(&v1alpha1.MongoDBVersion{})
+	mongodb.SetDefaults(&v1alpha1.MongoDBVersion{}, testTopology)
 }
diff --git a/go.mod b/go.mod
@@ -22,10 +22,10 @@ require (
 	k8s.io/apimachinery v0.16.5-beta.1
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a
-	kmodules.xyz/client-go v0.0.0-20200125095840-f95182274a65
+	kmodules.xyz/client-go v0.0.0-20200125212626-a094b2ba24c6
 	kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891
 	kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419
-	kmodules.xyz/monitoring-agent-api v0.0.0-20200125130554-3ed41c0ceff4
+	kmodules.xyz/monitoring-agent-api v0.0.0-20200125202117-d3b3e33ce41f
 	kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0
 	kmodules.xyz/offshoot-api v0.0.0-20200103145223-2c4f520520d2
 	kmodules.xyz/webhook-runtime v0.0.0-20191127075323-d4bfdee6974d

diff --git a/go.sum b/go.sum
@@ -778,15 +778,15 @@ kmodules.xyz/client-go v0.0.0-20191127054604-26981530831d/go.mod h1:OFxuKCiVR+MY
 kmodules.xyz/client-go v0.0.0-20191211192817-f1dcd02124ba/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI=
 kmodules.xyz/client-go v0.0.0-20200116162153-e083ae16abca h1:WErv3VRRePgp4oszIHgrFnxIaIHyfP9+930B99B8BBM=
 kmodules.xyz/client-go v0.0.0-20200116162153-e083ae16abca/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI=
-kmodules.xyz/client-go v0.0.0-20200125095840-f95182274a65 h1:GOCyBO2SahVAgRkKl2FNZ95MuMhWpArraUjCFR8heR0=
-kmodules.xyz/client-go v0.0.0-20200125095840-f95182274a65/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI=
+kmodules.xyz/client-go v0.0.0-20200125212626-a094b2ba24c6 h1:pZvnFFQOJpSdYhm9h+83tfLgHZkfVmiuHHV8XtWDVSo=
+kmodules.xyz/client-go v0.0.0-20200125212626-a094b2ba24c6/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI=
 kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY=
 kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891 h1:2W/fqLbAurvupIZL3TEFrlb7DnOIhlOXpNgzgcFFSmA=
 kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891/go.mod h1:9NXNZ4xhqof0WngtIuo4vl+WoCQpLJEJcyuEo3mPpiM=
 kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419 h1:o6KD8XMxdyRR3rqScTsWvcufFDT7vQBnYXpHUp6UtRg=
 kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419/go.mod h1:EksiAQK3p3bVU5cNBjLT0z5V7er1tvWLBWILTuf6G/0=
-kmodules.xyz/monitoring-agent-api v0.0.0-20200125130554-3ed41c0ceff4 h1:JoAMMESbYxoY+1GFIbIyRRda835uvnLc6vSeiGwqJck=
-kmodules.xyz/monitoring-agent-api v0.0.0-20200125130554-3ed41c0ceff4/go.mod h1:q5Adit2fxEoY1jSkYiK1t856Ombl8cunYJBmgL43wwc=
+kmodules.xyz/monitoring-agent-api v0.0.0-20200125202117-d3b3e33ce41f h1:BcVQqfRGPGZATwMLabMR6cFytqssE/wh+ZnfYSHIPwQ=
+kmodules.xyz/monitoring-agent-api v0.0.0-20200125202117-d3b3e33ce41f/go.mod h1:q5Adit2fxEoY1jSkYiK1t856Ombl8cunYJBmgL43wwc=
 kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0 h1:ilgkGU/bteKRvH99piXxU8b2c1E9lwxb0sDpLLhy4iY=
 kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0/go.mod h1:AE1rz+T/3dlUwH5pLgCNghWeeFfeVBnBXKJVpsLHlmc=
 kmodules.xyz/offshoot-api v0.0.0-20200103145223-2c4f520520d2 h1:c/wxNy8KgaS8v/htruqtL0YgrKciUKroK1zXPVZHuqg=

diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -36,6 +36,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
+	core_util "kmodules.xyz/client-go/core/v1"
 	"kmodules.xyz/client-go/tools/queue"
 	appcat_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
 	appcat_in "kmodules.xyz/custom-resources/client/informers/externalversions"
@@ -61,6 +62,8 @@ type Controller struct {
 	CertManagerClient cm.Interface
 	// externalClient for crd
 	ExternalClient ext_cs.Interface
+	// Cluster topology when the operator started
+	ClusterTopology *core_util.Topology
 }
 
 type Config struct {

diff --git a/vendor/kmodules.xyz/client-go/core/v1/node.go b/vendor/kmodules.xyz/client-go/core/v1/node.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"gomodules.xyz/version"
 	core "k8s.io/api/core/v1"
 	kerr "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -121,16 +122,114 @@ func IsMaster(node core.Node) bool {
 	return ok17 || (ok16 && role16 == "master")
 }
 
-func Topology(kc kubernetes.Interface) (regions map[string][]string, instances map[string]int, err error) {
+type Topology struct {
+	Regions       map[string][]string
+	TotalNodes    int
+	InstanceTypes map[string]int
+
+	LabelZone         string
+	LabelRegion       string
+	LabelInstanceType string
+
+	// https://github.com/kubernetes/kubernetes/blob/v1.17.2/staging/src/k8s.io/api/core/v1/well_known_labels.go
+
+	//LabelHostname = "kubernetes.io/hostname"
+	//
+	//LabelZoneFailureDomain       = "failure-domain.beta.kubernetes.io/zone"
+	//LabelZoneRegion              = "failure-domain.beta.kubernetes.io/region"
+	//LabelZoneFailureDomainStable = "topology.kubernetes.io/zone"
+	//LabelZoneRegionStable        = "topology.kubernetes.io/region"
+	//
+	//LabelInstanceType       = "beta.kubernetes.io/instance-type"
+	//LabelInstanceTypeStable = "node.kubernetes.io/instance-type"
+}
+
+func (t Topology) ConvertAffinity(affinity *core.Affinity) {
+	if affinity == nil {
+		return
+	}
+
+	t.convertPodAffinityTerm(affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
+	t.convertWeightedPodAffinityTerm(affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
+
+	t.convertPodAffinityTerm(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
+	t.convertWeightedPodAffinityTerm(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution)
+}
+
+func isZoneKey(key string) bool {
+	return key == core.LabelZoneFailureDomain || key == "topology.kubernetes.io/zone"
+}
+
+func isRegionKey(key string) bool {
+	return key == core.LabelZoneRegion || key == "topology.kubernetes.io/region"
+}
+
+func isInstanceTypeKey(key string) bool {
+	return key == core.LabelInstanceType || key == "node.kubernetes.io/instance-type"
+}
+
+func (t Topology) convertPodAffinityTerm(terms []core.PodAffinityTerm) {
+	for i := range terms {
+		if isZoneKey(terms[i].TopologyKey) {
+			terms[i].TopologyKey = t.LabelZone
+		}
+		if isRegionKey(terms[i].TopologyKey) {
+			terms[i].TopologyKey = t.LabelRegion
+		}
+		if isInstanceTypeKey(terms[i].TopologyKey) {
+			terms[i].TopologyKey = t.LabelInstanceType
+		}
+	}
+}
+
+func (t Topology) convertWeightedPodAffinityTerm(terms []core.WeightedPodAffinityTerm) {
+	for i := range terms {
+		if isZoneKey(terms[i].PodAffinityTerm.TopologyKey) {
+			terms[i].PodAffinityTerm.TopologyKey = t.LabelZone
+		}
+		if isRegionKey(terms[i].PodAffinityTerm.TopologyKey) {
+			terms[i].PodAffinityTerm.TopologyKey = t.LabelRegion
+		}
+		if isInstanceTypeKey(terms[i].PodAffinityTerm.TopologyKey) {
+			terms[i].PodAffinityTerm.TopologyKey = t.LabelInstanceType
+		}
+	}
+}
+
+func DetectTopology(kc kubernetes.Interface) (*Topology, error) {
 	// TODO: Use https://github.com/kubernetes/client-go/blob/kubernetes-1.17.0/metadata/interface.go once upgraded to 1.17
 
+	var topology Topology
+
+	info, err := kc.Discovery().ServerVersion()
+	if err != nil {
+		return nil, err
+	}
+	ver, err := version.NewVersion(info.GitVersion)
+	if err != nil {
+		return nil, err
+	}
+	ver = ver.ToMutator().ResetPrerelease().ResetMetadata().Done()
+	if ver.Major() >= 1 && ver.Minor() >= 17 {
+		topology.LabelZone = "topology.kubernetes.io/zone"
+		topology.LabelRegion = "topology.kubernetes.io/region"
+		topology.LabelInstanceType = "node.kubernetes.io/instance-type"
+	} else {
+		topology.LabelZone = core.LabelZoneFailureDomain
+		topology.LabelRegion = core.LabelZoneRegion
+		topology.LabelInstanceType = core.LabelInstanceType
+	}
+	topology.TotalNodes = 0
+
 	mapRegion := make(map[string]sets.String)
-	instances = make(map[string]int)
+	instances := make(map[string]int)
 
 	lister := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
 		return kc.CoreV1().Nodes().List(opts)
 	}))
 	err = lister.EachListItem(context.Background(), metav1.ListOptions{Limit: 100}, func(obj runtime.Object) error {
+		topology.TotalNodes++
+
 		m, err := meta.Accessor(obj)
 		if err != nil {
 			return err
@@ -164,12 +263,15 @@ func Topology(kc kubernetes.Interface) (regions map[string][]string, instances m
 		return nil
 	})
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	regions = make(map[string][]string)
+	regions := make(map[string][]string)
 	for k, v := range mapRegion {
 		regions[k] = v.List()
 	}
-	return regions, instances, nil
+	topology.Regions = regions
+	topology.InstanceTypes = instances
+
+	return &topology, nil
 }
diff --git a/vendor/kmodules.xyz/monitoring-agent-api/api/v1/helpers.go b/vendor/kmodules.xyz/monitoring-agent-api/api/v1/helpers.go
@@ -1,3 +1,19 @@
+/*
+Copyright The Kmodules Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package v1
 
 import (

diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -784,7 +784,7 @@ k8s.io/utils/integer
 k8s.io/utils/path
 k8s.io/utils/pointer
 k8s.io/utils/trace
-# kmodules.xyz/client-go v0.0.0-20200125095840-f95182274a65
+# kmodules.xyz/client-go v0.0.0-20200125212626-a094b2ba24c6
 kmodules.xyz/client-go
 kmodules.xyz/client-go/core/v1
 kmodules.xyz/client-go/discovery
@@ -806,7 +806,7 @@ kmodules.xyz/custom-resources/client/informers/externalversions/appcatalog
 kmodules.xyz/custom-resources/client/informers/externalversions/appcatalog/v1alpha1
 kmodules.xyz/custom-resources/client/informers/externalversions/internalinterfaces
 kmodules.xyz/custom-resources/client/listers/appcatalog/v1alpha1
-# kmodules.xyz/monitoring-agent-api v0.0.0-20200125130554-3ed41c0ceff4
+# kmodules.xyz/monitoring-agent-api v0.0.0-20200125202117-d3b3e33ce41f
 kmodules.xyz/monitoring-agent-api/api/v1
 # kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0
 kmodules.xyz/objectstore-api/api/v1